From e369218a1fe386b153530bf9af4744d88c58c0bc Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 12 Sep 2023 12:17:54 +0300 Subject: [PATCH] Fix rule node query performance --- .../server/dao/rule/RuleNodeDao.java | 2 +- .../server/dao/sql/rule/JpaRuleNodeDao.java | 4 +- .../dao/sql/rule/RuleNodeRepository.java | 13 +- .../dao/sql/rule/JpaRuleNodeDaoTest.java | 158 ++++++++++++++++++ 4 files changed, 169 insertions(+), 8 deletions(-) create mode 100644 dao/src/test/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeDaoTest.java diff --git a/dao/src/main/java/org/thingsboard/server/dao/rule/RuleNodeDao.java b/dao/src/main/java/org/thingsboard/server/dao/rule/RuleNodeDao.java index 7df394ddd4..cd8fbb0a3c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/rule/RuleNodeDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/rule/RuleNodeDao.java @@ -30,7 +30,7 @@ import java.util.List; */ public interface RuleNodeDao extends Dao { - List findRuleNodesByTenantIdAndType(TenantId tenantId, String type, String search); + List findRuleNodesByTenantIdAndType(TenantId tenantId, String type, String configurationSearch); PageData findAllRuleNodesByType(String type, PageLink pageLink); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeDao.java index 7adf8aa0c8..827b73a0c1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeDao.java @@ -56,8 +56,8 @@ public class JpaRuleNodeDao extends JpaAbstractDao imp } @Override - public List findRuleNodesByTenantIdAndType(TenantId tenantId, String type, String search) { - return DaoUtil.convertDataList(ruleNodeRepository.findRuleNodesByTenantIdAndType(tenantId.getId(), type, search)); + public List findRuleNodesByTenantIdAndType(TenantId tenantId, String type, String configurationSearch) { + return DaoUtil.convertDataList(ruleNodeRepository.findRuleNodesByTenantIdAndType(tenantId.getId(), type, configurationSearch)); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleNodeRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleNodeRepository.java index e016462255..c8ecb8000b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleNodeRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleNodeRepository.java @@ -29,19 +29,22 @@ import java.util.UUID; public interface RuleNodeRepository extends JpaRepository { - @Query("SELECT r FROM RuleNodeEntity r WHERE r.ruleChainId in " + - "(select id from RuleChainEntity rc WHERE rc.tenantId = :tenantId) " + - "AND r.type = :ruleType AND LOWER(r.configuration) LIKE LOWER(CONCAT('%', :searchText, '%')) ") + @Query(nativeQuery = true, value = "SELECT * FROM rule_node r WHERE r.rule_chain_id in " + + "(select id from rule_chain rc WHERE rc.tenant_id = :tenantId) AND r.type = :ruleType " + + " AND (:searchText IS NULL OR r.configuration ILIKE CONCAT('%', :searchText, '%'))") List findRuleNodesByTenantIdAndType(@Param("tenantId") UUID tenantId, @Param("ruleType") String ruleType, @Param("searchText") String searchText); - @Query("SELECT r FROM RuleNodeEntity r WHERE r.type = :ruleType AND LOWER(r.configuration) LIKE LOWER(CONCAT('%', :searchText, '%')) ") + @Query(nativeQuery = true, value = "SELECT * FROM rule_node r WHERE r.type = :ruleType " + + " AND (:searchText IS NULL OR r.configuration ILIKE CONCAT('%', :searchText, '%'))") Page findAllRuleNodesByType(@Param("ruleType") String ruleType, @Param("searchText") String searchText, Pageable pageable); - @Query("SELECT r FROM RuleNodeEntity r WHERE r.type = :ruleType AND r.configurationVersion < :version AND LOWER(r.configuration) LIKE LOWER(CONCAT('%', :searchText, '%')) ") + @Query(nativeQuery = true, value = "SELECT * FROM rule_node r WHERE r.type = :ruleType " + + " AND configuration_version < :version " + + " AND (:searchText IS NULL OR r.configuration ILIKE CONCAT('%', :searchText, '%'))") Page findAllRuleNodesByTypeAndVersionLessThan(@Param("ruleType") String ruleType, @Param("version") int version, @Param("searchText") String searchText, diff --git a/dao/src/test/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeDaoTest.java b/dao/src/test/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeDaoTest.java new file mode 100644 index 0000000000..6e77927233 --- /dev/null +++ b/dao/src/test/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeDaoTest.java @@ -0,0 +1,158 @@ +/** + * Copyright © 2016-2023 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.rule; + +import com.datastax.oss.driver.api.core.uuid.Uuids; +import com.google.common.util.concurrent.ListeningExecutorService; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.id.RuleChainId; +import org.thingsboard.server.common.data.id.RuleNodeId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.common.data.rule.RuleChain; +import org.thingsboard.server.common.data.rule.RuleNode; +import org.thingsboard.server.dao.AbstractJpaDaoTest; +import org.thingsboard.server.dao.rule.RuleChainDao; +import org.thingsboard.server.dao.rule.RuleNodeDao; + +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +public class JpaRuleNodeDaoTest extends AbstractJpaDaoTest { + + public static final int COUNT = 40; + public static final String PREFIX_FOR_RULE_NODE_NAME = "SEARCH_TEXT_"; + List ruleNodeIds; + TenantId tenantId1; + TenantId tenantId2; + RuleChainId ruleChainId1; + RuleChainId ruleChainId2; + + @Autowired + private RuleChainDao ruleChainDao; + + @Autowired + private RuleNodeDao ruleNodeDao; + + ListeningExecutorService executor; + + @Before + public void setUp() { + tenantId1 = TenantId.fromUUID(Uuids.timeBased()); + ruleChainId1 = new RuleChainId(UUID.randomUUID()); + tenantId2 = TenantId.fromUUID(Uuids.timeBased()); + ruleChainId2 = new RuleChainId(UUID.randomUUID()); + + ruleNodeIds = createRuleNodes(tenantId1, tenantId2, ruleChainId1, ruleChainId2, COUNT); + } + + @After + public void tearDown() throws Exception { + ruleNodeDao.removeAllByIds(ruleNodeIds); + if (executor != null) { + executor.shutdownNow(); + } + } + + @Test + public void testSaveRuleName0x00_thenSomeDatabaseException() { + RuleNode ruleNode = getRuleNode(ruleChainId1, "T", "\u0000"); + assertThatThrownBy(() -> ruleNodeIds.add(ruleNodeDao.save(tenantId1, ruleNode).getUuidId())); + } + + @Test + public void testFindRuleNodesByTenantIdAndType() { + List ruleNodes1 = ruleNodeDao.findRuleNodesByTenantIdAndType(tenantId1, "A", PREFIX_FOR_RULE_NODE_NAME); + assertEquals(20, ruleNodes1.size()); + + List ruleNodes2 = ruleNodeDao.findRuleNodesByTenantIdAndType(tenantId2, "B", PREFIX_FOR_RULE_NODE_NAME); + assertEquals(20, ruleNodes2.size()); + + ruleNodes1 = ruleNodeDao.findRuleNodesByTenantIdAndType(tenantId1, "A", null); + assertEquals(20, ruleNodes1.size()); + + ruleNodes2 = ruleNodeDao.findRuleNodesByTenantIdAndType(tenantId2, "B", null); + assertEquals(20, ruleNodes2.size()); + } + + @Test + public void testFindRuleNodesByType() { + PageData ruleNodes = ruleNodeDao.findAllRuleNodesByType( "A", new PageLink(10, 0, PREFIX_FOR_RULE_NODE_NAME)); + assertEquals(20, ruleNodes.getTotalElements()); + assertEquals(2, ruleNodes.getTotalPages()); + assertEquals(10, ruleNodes.getData().size()); + + ruleNodes = ruleNodeDao.findAllRuleNodesByType( "A", new PageLink(10, 0)); + assertEquals(20, ruleNodes.getTotalElements()); + assertEquals(2, ruleNodes.getTotalPages()); + assertEquals(10, ruleNodes.getData().size()); + } + + @Test + public void testFindRuleNodesByTypeAndVersionLessThan() { + PageData ruleNodes = ruleNodeDao.findAllRuleNodesByTypeAndVersionLessThan( "A", 1, new PageLink(10, 0, PREFIX_FOR_RULE_NODE_NAME)); + assertEquals(20, ruleNodes.getTotalElements()); + assertEquals(2, ruleNodes.getTotalPages()); + assertEquals(10, ruleNodes.getData().size()); + + ruleNodes = ruleNodeDao.findAllRuleNodesByTypeAndVersionLessThan( "A", 1, new PageLink(10, 0)); + assertEquals(20, ruleNodes.getTotalElements()); + assertEquals(2, ruleNodes.getTotalPages()); + assertEquals(10, ruleNodes.getData().size()); + } + + private List createRuleNodes(TenantId tenantId1, TenantId tenantId2, RuleChainId ruleChainId1, RuleChainId ruleChainId2, int count) { + var chain1 = new RuleChain(ruleChainId1); + chain1.setTenantId(tenantId1); + chain1.setName(ruleChainId1.toString()); + ruleChainDao.save(tenantId1, chain1); + var chain2 = new RuleChain(ruleChainId2); + chain2.setTenantId(tenantId2); + chain2.setName(ruleChainId2.toString()); + ruleChainDao.save(tenantId2, chain2); + List savedRuleNodeIds = new ArrayList<>(); + for (int i = 0; i < count / 2; i++) { + savedRuleNodeIds.add(ruleNodeDao.save(tenantId1, getRuleNode(ruleChainId1, "A", Integer.toString(i))).getUuidId()); + savedRuleNodeIds.add(ruleNodeDao.save(tenantId2, getRuleNode(ruleChainId2, "B", Integer.toString(i + count / 2))).getUuidId()); + } + return savedRuleNodeIds; + } + + private RuleNode getRuleNode(RuleChainId ruleChainId, String type, String nameSuffix) { + return getRuleNode(ruleChainId, Uuids.timeBased(), type, nameSuffix); + } + + private RuleNode getRuleNode(RuleChainId ruleChainId, UUID ruleNodeId, String type, String nameSuffix) { + RuleNode ruleNode = new RuleNode(); + ruleNode.setId(new RuleNodeId(ruleNodeId)); + ruleNode.setRuleChainId(ruleChainId); + ruleNode.setName(nameSuffix); + ruleNode.setType(type); + ruleNode.setConfiguration(JacksonUtil.newObjectNode().put("searchHint", PREFIX_FOR_RULE_NODE_NAME + nameSuffix)); + ruleNode.setConfigurationVersion(0); + return ruleNode; + } +}