From d3c20b2e57494ab5fc828c797359da8d816c7148 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 7 Dec 2021 18:50:16 +0200 Subject: [PATCH] Alarm Query performance improvements --- .../main/data/upgrade/3.3.2/schema_update.sql | 29 ++++++ .../install/ThingsboardInstallService.java | 4 + .../install/SqlDatabaseUpgradeService.java | 22 +++++ .../server/common/data/alarm/EntityAlarm.java | 40 ++++++++ .../data/relation/RelationTypeGroup.java | 1 - .../server/dao/alarm/AlarmDao.java | 5 + .../server/dao/alarm/BaseAlarmService.java | 27 ++--- .../dao/entityview/EntityViewServiceImpl.java | 1 - .../server/dao/model/ModelConstants.java | 1 + .../model/sql/EntityAlarmCompositeKey.java | 42 ++++++++ .../dao/model/sql/EntityAlarmEntity.java | 99 +++++++++++++++++++ .../server/dao/sql/alarm/AlarmRepository.java | 55 ++++------- .../dao/sql/alarm/EntityAlarmRepository.java | 29 ++++++ .../server/dao/sql/alarm/JpaAlarmDao.java | 16 ++- .../query/DefaultAlarmQueryRepository.java | 21 ++-- .../sql/query/DefaultQueryLogComponent.java | 2 +- .../resources/sql/schema-entities-hsql.sql | 12 +++ .../resources/sql/schema-entities-idx.sql | 4 + .../main/resources/sql/schema-entities.sql | 12 +++ .../dao/service/BaseAlarmServiceTest.java | 19 ---- .../resources/sql/hsql/drop-all-tables.sql | 1 + .../resources/sql/psql/drop-all-tables.sql | 1 + .../sql/timescale/drop-all-tables.sql | 1 + 23 files changed, 366 insertions(+), 78 deletions(-) create mode 100644 application/src/main/data/upgrade/3.3.2/schema_update.sql create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/alarm/EntityAlarm.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/model/sql/EntityAlarmCompositeKey.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/model/sql/EntityAlarmEntity.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/sql/alarm/EntityAlarmRepository.java diff --git a/application/src/main/data/upgrade/3.3.2/schema_update.sql b/application/src/main/data/upgrade/3.3.2/schema_update.sql new file mode 100644 index 0000000000..99d2294129 --- /dev/null +++ b/application/src/main/data/upgrade/3.3.2/schema_update.sql @@ -0,0 +1,29 @@ +-- +-- Copyright © 2016-2021 The Thingsboard Authors +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +CREATE TABLE IF NOT EXISTS entity_alarm ( + tenant_id uuid NOT NULL, + entity_id uuid NOT NULL, + created_time bigint NOT NULL, + type varchar(255) NOT NULL, + customer_id uuid, + alarm_id uuid + CONSTRAINT entity_alarm_pkey PRIMARY KEY(entity_id, alarm_id), + CONSTRAINT fk_entity_alarm_id FOREIGN KEY (alarm_id) REFERENCES alarm(id) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_alarm_tenant_status_created_time ON alarm(tenant_id, status, created_time DESC); +CREATE INDEX IF NOT EXISTS idx_entity_alarm_created_time ON entity_alarm(tenant_id, entity_id, created_time DESC); diff --git a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java index ccdac8b363..5b1bee8538 100644 --- a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java +++ b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java @@ -210,6 +210,10 @@ public class ThingsboardInstallService { log.info("Upgrading ThingsBoard from version 3.3.0 to 3.3.1 ..."); case "3.3.1": log.info("Upgrading ThingsBoard from version 3.3.1 to 3.3.2 ..."); + break; + case "3.3.2": + log.info("Upgrading ThingsBoard from version 3.3.2 to 3.3.3 ..."); + databaseEntitiesUpgradeService.upgradeDatabase("3.3.2"); log.info("Updating system data..."); systemDataLoaderService.updateSystemWidgets(); break; diff --git a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java index fcb9632020..3f9bec2d07 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java @@ -469,6 +469,28 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService log.error("Failed updating schema!!!", e); } break; + case "3.3.2": + try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) { + log.info("Updating schema ..."); + schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "3.3.2", SCHEMA_UPDATE_SQL); + loadSql(schemaUpdateFile, conn); + try { + conn.createStatement().execute("insert into entity_alarm(tenant_id, entity_id, created_time, type, customer_id, alarm_id, status, severity)" + + " select tenant_id, originator_id, created_time, type, customer_id, id, status, severity from alarm;"); + conn.createStatement().execute("insert into entity_alarm(tenant_id, entity_id, created_time, type, customer_id, alarm_id, status, severity)" + + " select a.tenant_id, r.from_id, created_time, type, customer_id, id, status, severity" + + " from alarm a inner join relation r on r.relation_type_group = 'ALARM' and r.relation_type = 'ANY' and a.id = r.to_id ON CONFLICT DO NOTHING;"); + conn.createStatement().execute("delete from relation r where r.relation_type_group = 'ALARM';"); + } catch (Exception e) { + log.error("Failed to update alarm relations!!!", e); + } + log.info("Updating schema settings..."); + conn.createStatement().execute("UPDATE tb_schema_settings SET schema_version = 3003003;"); + log.info("Schema updated."); + } catch (Exception e) { + log.error("Failed updating schema!!!", e); + } + break; default: throw new RuntimeException("Unable to upgrade SQL database, unsupported fromVersion: " + fromVersion); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/EntityAlarm.java b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/EntityAlarm.java new file mode 100644 index 0000000000..f6d4a0e2c7 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/EntityAlarm.java @@ -0,0 +1,40 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.alarm; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.thingsboard.server.common.data.HasTenantId; +import org.thingsboard.server.common.data.id.AlarmId; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; + +@Data +@NoArgsConstructor +@AllArgsConstructor +public class EntityAlarm implements HasTenantId { + + private TenantId tenantId; + private EntityId entityId; + private long createdTime; + private String alarmType; + + private CustomerId customerId; + private AlarmId alarmId; + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/relation/RelationTypeGroup.java b/common/data/src/main/java/org/thingsboard/server/common/data/relation/RelationTypeGroup.java index c6ca10088d..c35a1dfd1a 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/relation/RelationTypeGroup.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/relation/RelationTypeGroup.java @@ -18,7 +18,6 @@ package org.thingsboard.server.common.data.relation; public enum RelationTypeGroup { COMMON, - ALARM, DASHBOARD, RULE_CHAIN, RULE_NODE, diff --git a/dao/src/main/java/org/thingsboard/server/dao/alarm/AlarmDao.java b/dao/src/main/java/org/thingsboard/server/dao/alarm/AlarmDao.java index 3b10f624d7..8918b7cc1f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/alarm/AlarmDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/alarm/AlarmDao.java @@ -21,6 +21,7 @@ import org.thingsboard.server.common.data.alarm.AlarmInfo; import org.thingsboard.server.common.data.alarm.AlarmQuery; import org.thingsboard.server.common.data.alarm.AlarmSeverity; import org.thingsboard.server.common.data.alarm.AlarmStatus; +import org.thingsboard.server.common.data.alarm.EntityAlarm; import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; @@ -32,6 +33,7 @@ import org.thingsboard.server.common.data.query.AlarmDataQuery; import org.thingsboard.server.dao.Dao; import java.util.Collection; +import java.util.List; import java.util.Set; import java.util.UUID; @@ -59,4 +61,7 @@ public interface AlarmDao extends Dao { PageData findAlarmsIdsByEndTsBeforeAndTenantId(Long time, TenantId tenantId, PageLink pageLink); + void createEntityAlarmRecord(EntityAlarm entityAlarm); + + List findEntityAlarmRecords(TenantId tenantId, AlarmId id); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java b/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java index 90d988c20b..64874eff7d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java @@ -34,6 +34,7 @@ import org.thingsboard.server.common.data.alarm.AlarmQuery; import org.thingsboard.server.common.data.alarm.AlarmSearchStatus; import org.thingsboard.server.common.data.alarm.AlarmSeverity; import org.thingsboard.server.common.data.alarm.AlarmStatus; +import org.thingsboard.server.common.data.alarm.EntityAlarm; import org.thingsboard.server.common.data.exception.ApiUsageLimitsExceededException; import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.id.CustomerId; @@ -166,23 +167,24 @@ public class BaseAlarmService extends AbstractEntityService implements AlarmServ private AlarmOperationResult createAlarm(Alarm alarm) throws InterruptedException, ExecutionException { log.debug("New Alarm : {}", alarm); Alarm saved = alarmDao.save(alarm.getTenantId(), alarm); - List propagatedEntitiesList = createAlarmRelations(saved); + List propagatedEntitiesList = createEntityAlarmRecords(saved); return new AlarmOperationResult(saved, true, true, propagatedEntitiesList); } - private List createAlarmRelations(Alarm alarm) throws InterruptedException, ExecutionException { + private List createEntityAlarmRecords(Alarm alarm) throws InterruptedException, ExecutionException { List propagatedEntitiesList; if (alarm.isPropagate()) { Set parentEntities = getParentEntities(alarm); propagatedEntitiesList = new ArrayList<>(parentEntities.size() + 1); for (EntityId parentId : parentEntities) { propagatedEntitiesList.add(parentId); - createAlarmRelation(alarm.getTenantId(), parentId, alarm.getId()); + createEntityAlarmRecord(alarm.getTenantId(), parentId, alarm); } propagatedEntitiesList.add(alarm.getOriginator()); } else { propagatedEntitiesList = Collections.singletonList(alarm.getOriginator()); } + createEntityAlarmRecord(alarm.getTenantId(), alarm.getOriginator(), alarm); return propagatedEntitiesList; } @@ -221,7 +223,7 @@ public class BaseAlarmService extends AbstractEntityService implements AlarmServ List propagatedEntitiesList; if (!oldPropagate && newPropagate) { try { - propagatedEntitiesList = createAlarmRelations(result); + propagatedEntitiesList = createEntityAlarmRecords(result); } catch (InterruptedException | ExecutionException e) { log.warn("Failed to update alarm relations [{}]", result, e); throw new RuntimeException(e); @@ -382,17 +384,20 @@ public class BaseAlarmService extends AbstractEntityService implements AlarmServ private Set getPropagationEntityIds(Alarm alarm) { if (alarm.isPropagate()) { - List relations = relationService.findByTo(alarm.getTenantId(), alarm.getId(), RelationTypeGroup.ALARM); - Set propagationEntityIds = relations.stream().map(EntityRelation::getFrom).collect(Collectors.toSet()); - propagationEntityIds.add(alarm.getOriginator()); - return propagationEntityIds; + List entityAlarms = alarmDao.findEntityAlarmRecords(alarm.getTenantId(), alarm.getId()); + return entityAlarms.stream().map(EntityAlarm::getEntityId).collect(Collectors.toSet()); } else { return Collections.singleton(alarm.getOriginator()); } } - private void createAlarmRelation(TenantId tenantId, EntityId entityId, EntityId alarmId) { - createRelation(tenantId, new EntityRelation(entityId, alarmId, AlarmSearchStatus.ANY.name(), RelationTypeGroup.ALARM)); + private void createEntityAlarmRecord(TenantId tenantId, EntityId entityId, Alarm alarm) { + EntityAlarm entityAlarm = new EntityAlarm(tenantId, entityId, alarm.getCreatedTime(), alarm.getType(), alarm.getCustomerId(), alarm.getId()); + try { + alarmDao.createEntityAlarmRecord(entityAlarm); + } catch (Exception e) { + log.warn("[{}] Failed to create entity alarm record: {}", tenantId, entityAlarm, e); + } } private ListenableFuture getAndUpdate(TenantId tenantId, AlarmId alarmId, Function function) { @@ -402,7 +407,7 @@ public class BaseAlarmService extends AbstractEntityService implements AlarmServ } private DataValidator alarmDataValidator = - new DataValidator() { + new DataValidator<>() { @Override protected void validateDataImpl(TenantId tenantId, Alarm alarm) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java index 304b04a33d..33ea3d635f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java @@ -44,7 +44,6 @@ import org.thingsboard.server.common.data.id.EntityViewId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; -import org.thingsboard.server.common.data.page.TimePageLink; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.EntitySearchDirection; import org.thingsboard.server.common.data.relation.RelationTypeGroup; diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java index e8e90f2d3c..6f7a3c2567 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java @@ -257,6 +257,7 @@ public class ModelConstants { /** * Cassandra alarm constants. */ + public static final String ENTITY_ALARM_COLUMN_FAMILY_NAME = "entity_alarm"; public static final String ALARM_COLUMN_FAMILY_NAME = "alarm"; public static final String ALARM_TENANT_ID_PROPERTY = TENANT_ID_PROPERTY; public static final String ALARM_CUSTOMER_ID_PROPERTY = CUSTOMER_ID_PROPERTY; diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/EntityAlarmCompositeKey.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/EntityAlarmCompositeKey.java new file mode 100644 index 0000000000..69152418b8 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/EntityAlarmCompositeKey.java @@ -0,0 +1,42 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.model.sql; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.thingsboard.server.common.data.alarm.EntityAlarm; + +import javax.persistence.Transient; +import java.io.Serializable; +import java.util.UUID; + +@NoArgsConstructor +@AllArgsConstructor +@Data +public class EntityAlarmCompositeKey implements Serializable { + + @Transient + private static final long serialVersionUID = -245388185894468450L; + + private UUID entityId; + private UUID alarmId; + + public EntityAlarmCompositeKey(EntityAlarm entityAlarm) { + this.entityId = entityAlarm.getEntityId().getId(); + this.alarmId = entityAlarm.getAlarmId().getId(); + } +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/EntityAlarmEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/EntityAlarmEntity.java new file mode 100644 index 0000000000..50d57cf61c --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/EntityAlarmEntity.java @@ -0,0 +1,99 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.model.sql; + +import lombok.Data; +import org.thingsboard.server.common.data.alarm.EntityAlarm; +import org.thingsboard.server.common.data.id.AlarmId; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.model.ToData; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.Id; +import javax.persistence.IdClass; +import javax.persistence.Table; +import java.util.UUID; + +import static org.thingsboard.server.dao.model.ModelConstants.CREATED_TIME_PROPERTY; +import static org.thingsboard.server.dao.model.ModelConstants.CUSTOMER_ID_PROPERTY; +import static org.thingsboard.server.dao.model.ModelConstants.ENTITY_ALARM_COLUMN_FAMILY_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.ENTITY_ID_COLUMN; +import static org.thingsboard.server.dao.model.ModelConstants.ENTITY_TYPE_COLUMN; +import static org.thingsboard.server.dao.model.ModelConstants.TENANT_ID_COLUMN; + +@Data +@Entity +@Table(name = ENTITY_ALARM_COLUMN_FAMILY_NAME) +@IdClass(EntityAlarmCompositeKey.class) +public final class EntityAlarmEntity implements ToData { + + @Column(name = TENANT_ID_COLUMN, columnDefinition = "uuid") + private UUID tenantId; + + @Column(name = ENTITY_TYPE_COLUMN) + private String entityType; + + @Id + @Column(name = ENTITY_ID_COLUMN, columnDefinition = "uuid") + private UUID entityId; + + @Id + @Column(name = "alarm_id", columnDefinition = "uuid") + private UUID alarmId; + + @Column(name = CREATED_TIME_PROPERTY) + private long createdTime; + + @Column(name = "alarm_type") + private String alarmType; + + @Column(name = CUSTOMER_ID_PROPERTY, columnDefinition = "uuid") + private UUID customerId; + + public EntityAlarmEntity() { + super(); + } + + public EntityAlarmEntity(EntityAlarm entityAlarm) { + tenantId = entityAlarm.getTenantId().getId(); + entityId = entityAlarm.getEntityId().getId(); + entityType = entityAlarm.getEntityId().getEntityType().name(); + alarmId = entityAlarm.getAlarmId().getId(); + alarmType = entityAlarm.getAlarmType(); + createdTime = entityAlarm.getCreatedTime(); + if (entityAlarm.getCustomerId() != null) { + customerId = entityAlarm.getCustomerId().getId(); + } + } + + @Override + public EntityAlarm toData() { + EntityAlarm result = new EntityAlarm(); + result.setTenantId(new TenantId(tenantId)); + result.setEntityId(EntityIdFactory.getByTypeAndUuid(entityType, entityId)); + result.setAlarmId(new AlarmId(alarmId)); + result.setAlarmType(alarmType); + result.setCreatedTime(createdTime); + if (customerId != null) { + result.setCustomerId(new CustomerId(customerId)); + } + return result; + } + +} \ No newline at end of file diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmRepository.java index a67e29bac0..0152ef058d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmRepository.java @@ -42,47 +42,28 @@ public interface AlarmRepository extends CrudRepository { Pageable pageable); @Query(value = "SELECT new org.thingsboard.server.dao.model.sql.AlarmInfoEntity(a) FROM AlarmEntity a " + - "LEFT JOIN RelationEntity re ON a.id = re.toId " + - "AND re.relationTypeGroup = 'ALARM' " + - "AND re.toType = 'ALARM' " + - "AND re.fromId = :affectedEntityId " + - "AND re.fromType = :affectedEntityType " + + "LEFT JOIN EntityAlarmEntity ea ON a.id = ea.alarmId " + "WHERE a.tenantId = :tenantId " + - "AND (a.originatorId = :affectedEntityId or re.fromId IS NOT NULL) " + - "AND (:startTime IS NULL OR a.createdTime >= :startTime) " + - "AND (:endTime IS NULL OR a.createdTime <= :endTime) " + + "AND ea.tenantId = :tenantId " + + "AND ea.entityId = :affectedEntityId " + + "AND ea.entityType = :affectedEntityType " + + "AND (:startTime IS NULL OR (a.createdTime >= :startTime AND ea.createdTime >= :startTime)) " + + "AND (:endTime IS NULL OR (a.createdTime <= :endTime AND ea.createdTime <= :endTime)) " + "AND ((:alarmStatuses) IS NULL OR a.status in (:alarmStatuses)) " + "AND (LOWER(a.type) LIKE LOWER(CONCAT('%', :searchText, '%')) " + " OR LOWER(a.severity) LIKE LOWER(CONCAT('%', :searchText, '%')) " + " OR LOWER(a.status) LIKE LOWER(CONCAT('%', :searchText, '%'))) " , countQuery = "" + - "SELECT count(a) + " + //alarms with relations only - " (SELECT count(a) FROM AlarmEntity a " + //alarms WITHOUT any relations - " LEFT JOIN RelationEntity re ON a.id = re.toId " + - " AND re.relationTypeGroup = 'ALARM' " + - " AND re.toType = 'ALARM' " + - " AND re.fromId = :affectedEntityId " + - " AND re.fromType = :affectedEntityType " + - " WHERE a.tenantId = :tenantId " + - " AND (a.originatorId = :affectedEntityId) " + - " AND (re.fromId IS NULL) " + //anti join - " AND (:startTime IS NULL OR a.createdTime >= :startTime) " + - " AND (:endTime IS NULL OR a.createdTime <= :endTime) " + - " AND ((:alarmStatuses) IS NULL OR a.status in (:alarmStatuses)) " + - " AND (LOWER(a.type) LIKE LOWER(CONCAT('%', :searchText, '%')) " + - " OR LOWER(a.severity) LIKE LOWER(CONCAT('%', :searchText, '%')) " + - " OR LOWER(a.status) LIKE LOWER(CONCAT('%', :searchText, '%'))) " + - " )" + + "SELECT count(a) " + //alarms with relations only "FROM AlarmEntity a " + - "INNER JOIN RelationEntity re ON a.id = re.toId " + - "AND re.relationTypeGroup = 'ALARM' " + - "AND re.toType = 'ALARM' " + - "AND re.fromId = :affectedEntityId " + - "AND re.fromType = :affectedEntityType " + + "LEFT JOIN EntityAlarmEntity ea ON a.id = ea.alarmId " + "WHERE a.tenantId = :tenantId " + - "AND (:startTime IS NULL OR a.createdTime >= :startTime) " + - "AND (:endTime IS NULL OR a.createdTime <= :endTime) " + + "AND ea.tenantId = :tenantId " + + "AND ea.entityId = :affectedEntityId " + + "AND ea.entityType = :affectedEntityType " + + "AND (:startTime IS NULL OR (a.createdTime >= :startTime AND ea.createdTime >= :startTime)) " + + "AND (:endTime IS NULL OR (a.createdTime <= :endTime AND ea.createdTime <= :endTime)) " + "AND ((:alarmStatuses) IS NULL OR a.status in (:alarmStatuses)) " + "AND (LOWER(a.type) LIKE LOWER(CONCAT('%', :searchText, '%')) " + " OR LOWER(a.severity) LIKE LOWER(CONCAT('%', :searchText, '%')) " + @@ -149,13 +130,11 @@ public interface AlarmRepository extends CrudRepository { Pageable pageable); @Query(value = "SELECT a.severity FROM AlarmEntity a " + - "LEFT JOIN RelationEntity re ON a.id = re.toId " + - "AND re.relationTypeGroup = 'ALARM' " + - "AND re.toType = 'ALARM' " + - "AND re.fromId = :affectedEntityId " + - "AND re.fromType = :affectedEntityType " + + "LEFT JOIN EntityAlarmEntity ea ON a.id = ea.alarmId " + "WHERE a.tenantId = :tenantId " + - "AND (a.originatorId = :affectedEntityId or re.fromId IS NOT NULL) " + + "AND ea.tenantId = :tenantId " + + "AND ea.entityId = :affectedEntityId " + + "AND ea.entityType = :affectedEntityType " + "AND ((:alarmStatuses) IS NULL OR a.status in (:alarmStatuses))") Set findAlarmSeverities(@Param("tenantId") UUID tenantId, @Param("affectedEntityId") UUID affectedEntityId, diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/EntityAlarmRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/EntityAlarmRepository.java new file mode 100644 index 0000000000..ad67e9ba1d --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/EntityAlarmRepository.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.alarm; + +import org.springframework.data.repository.CrudRepository; +import org.thingsboard.server.dao.model.sql.EntityAlarmCompositeKey; +import org.thingsboard.server.dao.model.sql.EntityAlarmEntity; + +import java.util.List; +import java.util.UUID; + +public interface EntityAlarmRepository extends CrudRepository { + + List findAllByAlarmId(UUID alarmId); + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDao.java index 3216a23eee..73e7a39b9a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDao.java @@ -26,6 +26,7 @@ import org.thingsboard.server.common.data.alarm.AlarmInfo; import org.thingsboard.server.common.data.alarm.AlarmQuery; import org.thingsboard.server.common.data.alarm.AlarmSeverity; import org.thingsboard.server.common.data.alarm.AlarmStatus; +import org.thingsboard.server.common.data.alarm.EntityAlarm; import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; @@ -37,6 +38,7 @@ import org.thingsboard.server.common.data.query.AlarmDataQuery; import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.alarm.AlarmDao; import org.thingsboard.server.dao.model.sql.AlarmEntity; +import org.thingsboard.server.dao.model.sql.EntityAlarmEntity; import org.thingsboard.server.dao.relation.RelationDao; import org.thingsboard.server.dao.sql.JpaAbstractDao; import org.thingsboard.server.dao.sql.query.AlarmQueryRepository; @@ -62,7 +64,7 @@ public class JpaAlarmDao extends JpaAbstractDao implements A private AlarmQueryRepository alarmQueryRepository; @Autowired - private RelationDao relationDao; + private EntityAlarmRepository entityAlarmRepository; @Override protected Class getEntityClass() { @@ -169,4 +171,16 @@ public class JpaAlarmDao extends JpaAbstractDao implements A return DaoUtil.pageToPageData(alarmRepository.findAlarmsIdsByEndTsBeforeAndTenantId(time, tenantId.getId(), DaoUtil.toPageable(pageLink))) .mapData(AlarmId::new); } + + @Override + public void createEntityAlarmRecord(EntityAlarm entityAlarm) { + log.debug("Saving entity {}", entityAlarm); + entityAlarmRepository.save(new EntityAlarmEntity(entityAlarm)); + } + + @Override + public List findEntityAlarmRecords(TenantId tenantId, AlarmId id) { + log.trace("[{}] Try to find entity alarm records using [{}]", tenantId, id); + return DaoUtil.convertDataList(entityAlarmRepository.findAllByAlarmId(id.getId())); + } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultAlarmQueryRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultAlarmQueryRepository.java index 400b12fcab..3fee0f6bef 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultAlarmQueryRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultAlarmQueryRepository.java @@ -105,7 +105,7 @@ public class DefaultAlarmQueryRepository implements AlarmQueryRepository { " a.propagate_relation_types as propagate_relation_types, " + " a.type as type," + SELECT_ORIGINATOR_NAME + ", "; - private static final String JOIN_RELATIONS = "left join relation r on r.relation_type_group = 'ALARM' and r.relation_type = 'ANY' and a.id = r.to_id and r.from_id in (:entity_ids)"; + private static final String JOIN_ENTITY_ALARMS = "inner join entity_alarm ea on a.id = ea.alarm_id"; protected final NamedParameterJdbcTemplate jdbcTemplate; private final TransactionTemplate transactionTemplate; @@ -132,8 +132,8 @@ public class DefaultAlarmQueryRepository implements AlarmQueryRepository { StringBuilder joinPart = new StringBuilder(); boolean addAnd = false; if (pageLink.isSearchPropagatedAlarms()) { - selectPart.append(" CASE WHEN r.from_id IS NULL THEN a.originator_id ELSE r.from_id END as entity_id "); - fromPart.append(JOIN_RELATIONS); + selectPart.append(" ea.entity_id as entity_id "); + fromPart.append(JOIN_ENTITY_ALARMS); wherePart.append(buildPermissionsQuery(tenantId, customerId, ctx)); addAnd = true; } else { @@ -145,7 +145,7 @@ public class DefaultAlarmQueryRepository implements AlarmQueryRepository { sortPart.append(alarmFieldColumnMap.getOrDefault(sortOrderKey, sortOrderKey)) .append(" ").append(sortOrder.getDirection().name()); if (pageLink.isSearchPropagatedAlarms()) { - wherePart.append(" and (a.originator_id in (:entity_ids) or r.from_id IS NOT NULL)"); + wherePart.append(" and ea.entity_id in (:entity_ids)"); } else { addAndIfNeeded(wherePart, addAnd); addAnd = true; @@ -166,7 +166,7 @@ public class DefaultAlarmQueryRepository implements AlarmQueryRepository { } joinPart.append(" as e(id, priority)) e "); if (pageLink.isSearchPropagatedAlarms()) { - joinPart.append("on (r.from_id IS NULL and a.originator_id = e.id) or (r.from_id IS NOT NULL and r.from_id = e.id)"); + joinPart.append("on ea.entity_id = e.id"); } else { joinPart.append("on a.originator_id = e.id"); } @@ -188,6 +188,9 @@ public class DefaultAlarmQueryRepository implements AlarmQueryRepository { addAnd = true; ctx.addLongParameter("startTime", startTs); wherePart.append("a.created_time >= :startTime"); + if (pageLink.isSearchPropagatedAlarms()) { + wherePart.append(" and ea.created_time >= :startTime"); + } } if (endTs > 0) { @@ -195,6 +198,9 @@ public class DefaultAlarmQueryRepository implements AlarmQueryRepository { addAnd = true; ctx.addLongParameter("endTime", endTs); wherePart.append("a.created_time <= :endTime"); + if (pageLink.isSearchPropagatedAlarms()) { + wherePart.append(" and ea.created_time <= :endTime"); + } } if (pageLink.getTypeList() != null && !pageLink.getTypeList().isEmpty()) { @@ -202,6 +208,9 @@ public class DefaultAlarmQueryRepository implements AlarmQueryRepository { addAnd = true; ctx.addStringListParameter("alarmTypes", pageLink.getTypeList()); wherePart.append("a.type in (:alarmTypes)"); + if (pageLink.isSearchPropagatedAlarms()) { + wherePart.append(" and ea.alarm_type in (:alarmTypes)"); + } } if (pageLink.getSeverityList() != null && !pageLink.getSeverityList().isEmpty()) { @@ -279,7 +288,7 @@ public class DefaultAlarmQueryRepository implements AlarmQueryRepository { private String buildPermissionsQuery(TenantId tenantId, CustomerId customerId, QueryContext ctx) { StringBuilder permissionsQuery = new StringBuilder(); ctx.addUuidParameter("permissions_tenant_id", tenantId.getId()); - permissionsQuery.append(" a.tenant_id = :permissions_tenant_id "); + permissionsQuery.append(" a.tenant_id = :permissions_tenant_id and ea.tenant_id = :permissions_tenant_id "); /* No need to check the customer id, because we already use entity id list that passed security check when we were evaluating the data query. */ diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponent.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponent.java index 89bf4d1ac1..737c148d49 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponent.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponent.java @@ -33,7 +33,7 @@ public class DefaultQueryLogComponent implements QueryLogComponent { @Override public void logQuery(QueryContext ctx, String query, long duration) { if (logSqlQueries && duration > logQueriesThreshold) { - log.info("QUERY: {} took {}ms", query, duration); + log.info("QUERY: {} took {} ms", query, duration); Arrays.asList(ctx.getParameterNames()).forEach(param -> log.info("QUERY PARAM: {} -> {}", param, ctx.getValue(param))); } } diff --git a/dao/src/main/resources/sql/schema-entities-hsql.sql b/dao/src/main/resources/sql/schema-entities-hsql.sql index e58ca10459..c5931a8964 100644 --- a/dao/src/main/resources/sql/schema-entities-hsql.sql +++ b/dao/src/main/resources/sql/schema-entities-hsql.sql @@ -43,6 +43,18 @@ CREATE TABLE IF NOT EXISTS alarm ( type varchar(255) ); +CREATE TABLE IF NOT EXISTS entity_alarm ( + tenant_id uuid NOT NULL, + entity_type varchar(32), + entity_id uuid NOT NULL, + created_time bigint NOT NULL, + alarm_type varchar(255) NOT NULL, + customer_id uuid, + alarm_id uuid, + CONSTRAINT entity_alarm_pkey PRIMARY KEY(entity_id, alarm_id), + CONSTRAINT fk_entity_alarm_id FOREIGN KEY (alarm_id) REFERENCES alarm(id) ON DELETE CASCADE +); + CREATE TABLE IF NOT EXISTS asset ( id uuid NOT NULL CONSTRAINT asset_pkey PRIMARY KEY, created_time bigint NOT NULL, diff --git a/dao/src/main/resources/sql/schema-entities-idx.sql b/dao/src/main/resources/sql/schema-entities-idx.sql index 78b1eda905..1238b5ee12 100644 --- a/dao/src/main/resources/sql/schema-entities-idx.sql +++ b/dao/src/main/resources/sql/schema-entities-idx.sql @@ -20,8 +20,12 @@ CREATE INDEX IF NOT EXISTS idx_alarm_originator_created_time ON alarm(originator CREATE INDEX IF NOT EXISTS idx_alarm_tenant_created_time ON alarm(tenant_id, created_time DESC); +CREATE INDEX IF NOT EXISTS idx_alarm_tenant_status_created_time ON alarm(tenant_id, status, created_time DESC); + CREATE INDEX IF NOT EXISTS idx_alarm_tenant_alarm_type_created_time ON alarm(tenant_id, type, created_time DESC); +CREATE INDEX IF NOT EXISTS idx_entity_alarm_created_time ON entity_alarm(tenant_id, entity_id, created_time DESC); + CREATE INDEX IF NOT EXISTS idx_relation_to_id ON relation(relation_type_group, to_type, to_id); CREATE INDEX IF NOT EXISTS idx_relation_from_id ON relation(relation_type_group, from_type, from_id); diff --git a/dao/src/main/resources/sql/schema-entities.sql b/dao/src/main/resources/sql/schema-entities.sql index 69760c377a..de27c8d554 100644 --- a/dao/src/main/resources/sql/schema-entities.sql +++ b/dao/src/main/resources/sql/schema-entities.sql @@ -58,6 +58,18 @@ CREATE TABLE IF NOT EXISTS alarm ( type varchar(255) ); +CREATE TABLE IF NOT EXISTS entity_alarm ( + tenant_id uuid NOT NULL, + entity_type varchar(32), + entity_id uuid NOT NULL, + created_time bigint NOT NULL, + alarm_type varchar(255) NOT NULL, + customer_id uuid, + alarm_id uuid, + CONSTRAINT entity_alarm_pkey PRIMARY KEY(entity_id, alarm_id), + CONSTRAINT fk_entity_alarm_id FOREIGN KEY (alarm_id) REFERENCES alarm(id) ON DELETE CASCADE +); + CREATE TABLE IF NOT EXISTS asset ( id uuid NOT NULL CONSTRAINT asset_pkey PRIMARY KEY, created_time bigint NOT NULL, diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/BaseAlarmServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/BaseAlarmServiceTest.java index a798133ad4..2d3f21545f 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/BaseAlarmServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/BaseAlarmServiceTest.java @@ -612,16 +612,6 @@ public abstract class BaseAlarmServiceTest extends AbstractServiceTest { Assert.assertEquals(1, alarms.getData().size()); Assert.assertEquals(created, alarms.getData().get(0)); - List toAlarmRelations = relationService.findByTo(tenantId, created.getId(), RelationTypeGroup.ALARM); - Assert.assertEquals(1, toAlarmRelations.size()); - - List fromChildRelations = relationService.findByFrom(tenantId, childId, RelationTypeGroup.ALARM); - Assert.assertEquals(0, fromChildRelations.size()); - - List fromParentRelations = relationService.findByFrom(tenantId, parentId, RelationTypeGroup.ALARM); - Assert.assertEquals(1, fromParentRelations.size()); - - Assert.assertTrue("Alarm was not deleted when expected", alarmService.deleteAlarm(tenantId, created.getId()).isSuccessful()); Alarm fetched = alarmService.findAlarmByIdAsync(tenantId, created.getId()).get(); @@ -647,14 +637,5 @@ public abstract class BaseAlarmServiceTest extends AbstractServiceTest { Assert.assertNotNull(alarms.getData()); Assert.assertEquals(0, alarms.getData().size()); - toAlarmRelations = relationService.findByTo(tenantId, created.getId(), RelationTypeGroup.ALARM); - Assert.assertEquals(0, toAlarmRelations.size()); - - fromChildRelations = relationService.findByFrom(tenantId, childId, RelationTypeGroup.ALARM); - Assert.assertEquals(0, fromChildRelations.size()); - - fromParentRelations = relationService.findByFrom(tenantId, childId, RelationTypeGroup.ALARM); - Assert.assertEquals(0, fromParentRelations.size()); - } } diff --git a/dao/src/test/resources/sql/hsql/drop-all-tables.sql b/dao/src/test/resources/sql/hsql/drop-all-tables.sql index 2cd7e26b48..095a38533e 100644 --- a/dao/src/test/resources/sql/hsql/drop-all-tables.sql +++ b/dao/src/test/resources/sql/hsql/drop-all-tables.sql @@ -1,4 +1,5 @@ DROP TABLE IF EXISTS admin_settings; +DROP TABLE IF EXISTS entity_alarm; DROP TABLE IF EXISTS alarm; DROP TABLE IF EXISTS asset; DROP TABLE IF EXISTS audit_log; diff --git a/dao/src/test/resources/sql/psql/drop-all-tables.sql b/dao/src/test/resources/sql/psql/drop-all-tables.sql index 0a71953032..3d1d38a0e7 100644 --- a/dao/src/test/resources/sql/psql/drop-all-tables.sql +++ b/dao/src/test/resources/sql/psql/drop-all-tables.sql @@ -1,4 +1,5 @@ DROP TABLE IF EXISTS admin_settings; +DROP TABLE IF EXISTS entity_alarm; DROP TABLE IF EXISTS alarm; DROP TABLE IF EXISTS asset; DROP TABLE IF EXISTS audit_log; diff --git a/dao/src/test/resources/sql/timescale/drop-all-tables.sql b/dao/src/test/resources/sql/timescale/drop-all-tables.sql index 53da64aff2..f113ec5277 100644 --- a/dao/src/test/resources/sql/timescale/drop-all-tables.sql +++ b/dao/src/test/resources/sql/timescale/drop-all-tables.sql @@ -1,4 +1,5 @@ DROP TABLE IF EXISTS admin_settings; +DROP TABLE IF EXISTS entity_alarm; DROP TABLE IF EXISTS alarm; DROP TABLE IF EXISTS asset; DROP TABLE IF EXISTS audit_log;