From 5118e1ed92674f669a072e98381a3a6707597a10 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Wed, 30 Jun 2021 20:40:38 +0300 Subject: [PATCH 1/7] events: performance improvements: UI fast response with latest events, ttl cleanup speedup (with schema update) --- .../upgrade/3.3.0/schema_update_event.sql | 90 +++++++++++++++++++ .../install/SqlDatabaseUpgradeService.java | 11 +++ .../service/ttl/EventsCleanUpService.java | 2 +- .../src/main/resources/thingsboard.yml | 2 +- .../service/ttl/EventsCleanUpServiceTest.java | 36 ++++++++ 5 files changed, 139 insertions(+), 2 deletions(-) create mode 100644 application/src/main/data/upgrade/3.3.0/schema_update_event.sql create mode 100644 application/src/test/java/org/thingsboard/server/service/ttl/EventsCleanUpServiceTest.java diff --git a/application/src/main/data/upgrade/3.3.0/schema_update_event.sql b/application/src/main/data/upgrade/3.3.0/schema_update_event.sql new file mode 100644 index 0000000000..6cbde51536 --- /dev/null +++ b/application/src/main/data/upgrade/3.3.0/schema_update_event.sql @@ -0,0 +1,90 @@ +-- +-- Copyright © 2016-2021 The Thingsboard Authors +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +-- PROCEDURE: public.cleanup_events_by_ttl(bigint, bigint, bigint) + +DROP PROCEDURE IF EXISTS public.cleanup_events_by_ttl(bigint, bigint, bigint); + +CREATE OR REPLACE PROCEDURE public.cleanup_events_by_ttl( + ttl bigint, + debug_ttl bigint, + INOUT deleted bigint) +LANGUAGE 'plpgsql' +AS $BODY$ +DECLARE + ttl_ts bigint; + debug_ttl_ts bigint; + ttl_deleted_count bigint DEFAULT 0; + debug_ttl_deleted_count bigint DEFAULT 0; +BEGIN + IF ttl > 0 THEN + ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - ttl::bigint * 1000)::bigint; + + DELETE FROM event + WHERE ts < ttl_ts + AND NOT event_type IN ('DEBUG_RULE_NODE', 'DEBUG_RULE_CHAIN', 'DEBUG_CONVERTER', 'DEBUG_INTEGRATION'); + + GET DIAGNOSTICS ttl_deleted_count = ROW_COUNT; + END IF; + + IF debug_ttl > 0 THEN + debug_ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - debug_ttl::bigint * 1000)::bigint; + + DELETE FROM event + WHERE ts < debug_ttl_ts + AND event_type IN ('DEBUG_RULE_NODE', 'DEBUG_RULE_CHAIN', 'DEBUG_CONVERTER', 'DEBUG_INTEGRATION'); + + GET DIAGNOSTICS debug_ttl_deleted_count = ROW_COUNT; + END IF; + + RAISE NOTICE 'Events removed by ttl: %', ttl_deleted_count; + RAISE NOTICE 'Debug Events removed by ttl: %', debug_ttl_deleted_count; + deleted := ttl_deleted_count + debug_ttl_deleted_count; +END +$BODY$; + + +-- Index: idx_event_ts + +DROP INDEX IF EXISTS public.idx_event_ts; + +-- Hint: add CONCURRENTLY to CREATE INDEX query in case of more then 1 million records or during live update +-- CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_event_ts +CREATE INDEX IF NOT EXISTS idx_event_ts + ON public.event USING btree + (ts DESC NULLS LAST) + WITH (FILLFACTOR=95); + +COMMENT ON INDEX public.idx_event_ts + IS 'This index helps to delete events by TTL using timestamp'; + + +-- Index: idx_event_tenant_entity_type_entity_event_type_created_time_des + +DROP INDEX IF EXISTS public.idx_event_tenant_entity_type_entity_event_type_created_time_des; + +CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_event_tenant_entity_type_entity_event_type_created_time_des +CREATE INDEX IF NOT EXISTS idx_event_tenant_entity_type_entity_event_type_created_time_des + ON public.event USING btree + (tenant_id ASC NULLS LAST, entity_type ASC NULLS LAST, entity_id ASC NULLS LAST, event_type ASC NULLS LAST, created_time DESC NULLS LAST) + WITH (FILLFACTOR=95); + +COMMENT ON INDEX public.idx_event_tenant_entity_type_entity_event_type_created_time_des + IS 'This index helps to open latest events on UI fast'; + +-- Index: idx_event_type_entity_id +-- Description: replaced with more suitable idx_event_tenant_entity_type_entity_event_type_created_time_des +DROP INDEX IF EXISTS public.idx_event_type_entity_id; \ No newline at end of file diff --git a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java index 470f39f401..cb0b83abb5 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java @@ -465,6 +465,17 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService log.error("Failed updating schema!!!", e); } break; + case "3.3.0": + try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) { + log.info("Updating schema ..."); + log.info("Updating indexes and procedure for event table..."); + schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "3.3.0", "schema_update_event.sql"); + loadSql(schemaUpdateFile, conn); + log.info("Schema updated."); + } catch (Exception e) { + log.error("Failed updating schema!!!", e); + } + break; default: throw new RuntimeException("Unable to upgrade SQL database, unsupported fromVersion: " + fromVersion); } diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/EventsCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/EventsCleanUpService.java index a51910b7ed..e2d6842241 100644 --- a/application/src/main/java/org/thingsboard/server/service/ttl/EventsCleanUpService.java +++ b/application/src/main/java/org/thingsboard/server/service/ttl/EventsCleanUpService.java @@ -45,7 +45,7 @@ public class EventsCleanUpService extends AbstractCleanUpService { this.eventService = eventService; } - @Scheduled(initialDelayString = "${sql.ttl.events.execution_interval_ms}", fixedDelayString = "${sql.ttl.events.execution_interval_ms}") + @Scheduled(initialDelayString = "#{T(org.apache.commons.lang3.RandomUtils).nextLong(0, ${sql.ttl.events.execution_interval_ms})}", fixedDelayString = "${sql.ttl.events.execution_interval_ms}") public void cleanUp() { if (ttlTaskExecutionEnabled && isSystemTenantPartitionMine()) { eventService.cleanupEvents(ttl, debugTtl); diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 79e8d9e1d3..19d512df1e 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -266,7 +266,7 @@ sql: ts_key_value_ttl: "${SQL_TTL_TS_TS_KEY_VALUE_TTL:0}" # Number of seconds events: enabled: "${SQL_TTL_EVENTS_ENABLED:true}" - execution_interval_ms: "${SQL_TTL_EVENTS_EXECUTION_INTERVAL:86400000}" # Number of milliseconds. The current value corresponds to one day + execution_interval_ms: "${SQL_TTL_EVENTS_EXECUTION_INTERVAL:2220000}" # Number of milliseconds (max random initial delay and fixed period). # 37minutes to avoid common interval spikes events_ttl: "${SQL_TTL_EVENTS_EVENTS_TTL:0}" # Number of seconds debug_events_ttl: "${SQL_TTL_EVENTS_DEBUG_EVENTS_TTL:604800}" # Number of seconds. The current value corresponds to one week edge_events: diff --git a/application/src/test/java/org/thingsboard/server/service/ttl/EventsCleanUpServiceTest.java b/application/src/test/java/org/thingsboard/server/service/ttl/EventsCleanUpServiceTest.java new file mode 100644 index 0000000000..b2eb9f39f1 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/service/ttl/EventsCleanUpServiceTest.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.ttl; + +import lombok.extern.slf4j.Slf4j; +import org.junit.Test; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.lessThanOrEqualTo; + +@Slf4j +public class EventsCleanUpServiceTest { + + @Test + public void givenInterval_whenRandomDelay_ThenDelayInInterval() { + final long executionIntervalMs = 2220000; //37min + final long randomDelay = org.apache.commons.lang3.RandomUtils.nextLong(0, executionIntervalMs); //same as @Scheduled(initialDelayString = ... + log.info("randomDelay {}", randomDelay); + assertThat(randomDelay, greaterThanOrEqualTo(0L)); + assertThat(randomDelay, lessThanOrEqualTo(executionIntervalMs)); + } +} From 6cf63eba9f1f2359307e8829eede3049b747264c Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 1 Jul 2021 11:39:27 +0300 Subject: [PATCH 2/7] events: fixed update from version (3.2.2) --- .../upgrade/{3.3.0 => 3.2.2}/schema_update_event.sql | 0 .../service/install/SqlDatabaseUpgradeService.java | 12 ++---------- 2 files changed, 2 insertions(+), 10 deletions(-) rename application/src/main/data/upgrade/{3.3.0 => 3.2.2}/schema_update_event.sql (100%) diff --git a/application/src/main/data/upgrade/3.3.0/schema_update_event.sql b/application/src/main/data/upgrade/3.2.2/schema_update_event.sql similarity index 100% rename from application/src/main/data/upgrade/3.3.0/schema_update_event.sql rename to application/src/main/data/upgrade/3.2.2/schema_update_event.sql diff --git a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java index cb0b83abb5..8d9efec940 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java @@ -460,16 +460,8 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService loadSql(schemaUpdateFile, conn); log.info("Edge TTL functions successfully loaded!"); conn.createStatement().execute("UPDATE tb_schema_settings SET schema_version = 3003000;"); - log.info("Schema updated."); - } catch (Exception e) { - log.error("Failed updating schema!!!", e); - } - break; - case "3.3.0": - try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) { - log.info("Updating schema ..."); - log.info("Updating indexes and procedure for event table..."); - schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "3.3.0", "schema_update_event.sql"); + log.info("Updating indexes and TTL procedure for event table..."); + schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "3.2.2", "schema_update_event.sql"); loadSql(schemaUpdateFile, conn); log.info("Schema updated."); } catch (Exception e) { From 1037e5d28ed67e8e4eecbeb06cbf5024ea08afe4 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 1 Jul 2021 12:26:39 +0300 Subject: [PATCH 3/7] events: test refactored as Spring Boot test to verify random delay expression and default value in yaml --- .../service/ttl/EventsCleanUpService.java | 5 +++- .../service/ttl/EventsCleanUpServiceTest.java | 24 +++++++++++++++---- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/EventsCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/EventsCleanUpService.java index e2d6842241..34cb2e1320 100644 --- a/application/src/main/java/org/thingsboard/server/service/ttl/EventsCleanUpService.java +++ b/application/src/main/java/org/thingsboard/server/service/ttl/EventsCleanUpService.java @@ -29,6 +29,9 @@ import org.thingsboard.server.service.ttl.AbstractCleanUpService; @Service public class EventsCleanUpService extends AbstractCleanUpService { + public static final String RANDOM_DELAY_INTERVAL_MS_EXPRESSION = + "#{T(org.apache.commons.lang3.RandomUtils).nextLong(0, ${sql.ttl.events.execution_interval_ms})}"; + @Value("${sql.ttl.events.events_ttl}") private long ttl; @@ -45,7 +48,7 @@ public class EventsCleanUpService extends AbstractCleanUpService { this.eventService = eventService; } - @Scheduled(initialDelayString = "#{T(org.apache.commons.lang3.RandomUtils).nextLong(0, ${sql.ttl.events.execution_interval_ms})}", fixedDelayString = "${sql.ttl.events.execution_interval_ms}") + @Scheduled(initialDelayString = RANDOM_DELAY_INTERVAL_MS_EXPRESSION, fixedDelayString = "${sql.ttl.events.execution_interval_ms}") public void cleanUp() { if (ttlTaskExecutionEnabled && isSystemTenantPartitionMine()) { eventService.cleanupEvents(ttl, debugTtl); diff --git a/application/src/test/java/org/thingsboard/server/service/ttl/EventsCleanUpServiceTest.java b/application/src/test/java/org/thingsboard/server/service/ttl/EventsCleanUpServiceTest.java index b2eb9f39f1..422d5ef358 100644 --- a/application/src/test/java/org/thingsboard/server/service/ttl/EventsCleanUpServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/ttl/EventsCleanUpServiceTest.java @@ -17,20 +17,34 @@ package org.thingsboard.server.service.ttl; import lombok.extern.slf4j.Slf4j; import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.junit4.SpringRunner; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.lessThanOrEqualTo; +import static org.thingsboard.server.service.ttl.EventsCleanUpService.RANDOM_DELAY_INTERVAL_MS_EXPRESSION; +@RunWith(SpringRunner.class) +@SpringBootTest(classes = EventsCleanUpServiceTest.class) @Slf4j public class EventsCleanUpServiceTest { + @Value(RANDOM_DELAY_INTERVAL_MS_EXPRESSION) + long randomDelayMs; + @Value("${sql.ttl.events.execution_interval_ms}") + long executionIntervalMs; + @Test public void givenInterval_whenRandomDelay_ThenDelayInInterval() { - final long executionIntervalMs = 2220000; //37min - final long randomDelay = org.apache.commons.lang3.RandomUtils.nextLong(0, executionIntervalMs); //same as @Scheduled(initialDelayString = ... - log.info("randomDelay {}", randomDelay); - assertThat(randomDelay, greaterThanOrEqualTo(0L)); - assertThat(randomDelay, lessThanOrEqualTo(executionIntervalMs)); + log.info("randomDelay {}", randomDelayMs); + log.info("executionIntervalMs {}", executionIntervalMs); + assertThat(executionIntervalMs, is(2220000L)); + assertThat(randomDelayMs, greaterThanOrEqualTo(0L)); + assertThat(randomDelayMs, lessThanOrEqualTo(executionIntervalMs)); } + } From f60137de34a3424be147c2f78ec57dbcee440fb9 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Wed, 7 Jul 2021 13:07:59 +0300 Subject: [PATCH 4/7] events: schema_update_event fixed typo with a comment. schema-entities-idx updated. Upgrade order fixed based on review comments. --- .../data/upgrade/3.2.2/schema_update_event.sql | 2 +- .../install/SqlDatabaseUpgradeService.java | 3 ++- .../main/resources/sql/schema-entities-idx.sql | 18 ++++++++++++++++-- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/application/src/main/data/upgrade/3.2.2/schema_update_event.sql b/application/src/main/data/upgrade/3.2.2/schema_update_event.sql index 6cbde51536..c8c4e428c8 100644 --- a/application/src/main/data/upgrade/3.2.2/schema_update_event.sql +++ b/application/src/main/data/upgrade/3.2.2/schema_update_event.sql @@ -76,7 +76,7 @@ COMMENT ON INDEX public.idx_event_ts DROP INDEX IF EXISTS public.idx_event_tenant_entity_type_entity_event_type_created_time_des; -CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_event_tenant_entity_type_entity_event_type_created_time_des +-- CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_event_tenant_entity_type_entity_event_type_created_time_des CREATE INDEX IF NOT EXISTS idx_event_tenant_entity_type_entity_event_type_created_time_des ON public.event USING btree (tenant_id ASC NULLS LAST, entity_type ASC NULLS LAST, entity_id ASC NULLS LAST, event_type ASC NULLS LAST, created_time DESC NULLS LAST) diff --git a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java index 8d9efec940..fcb9632020 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java @@ -459,10 +459,11 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "3.2.2", "schema_update_ttl.sql"); loadSql(schemaUpdateFile, conn); log.info("Edge TTL functions successfully loaded!"); - conn.createStatement().execute("UPDATE tb_schema_settings SET schema_version = 3003000;"); log.info("Updating indexes and TTL procedure for event table..."); schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "3.2.2", "schema_update_event.sql"); loadSql(schemaUpdateFile, conn); + log.info("Updating schema settings..."); + conn.createStatement().execute("UPDATE tb_schema_settings SET schema_version = 3003000;"); log.info("Schema updated."); } catch (Exception e) { log.error("Failed updating schema!!!", e); diff --git a/dao/src/main/resources/sql/schema-entities-idx.sql b/dao/src/main/resources/sql/schema-entities-idx.sql index d649e44418..443d99faad 100644 --- a/dao/src/main/resources/sql/schema-entities-idx.sql +++ b/dao/src/main/resources/sql/schema-entities-idx.sql @@ -22,8 +22,6 @@ CREATE INDEX IF NOT EXISTS idx_alarm_tenant_created_time ON alarm(tenant_id, cre CREATE INDEX IF NOT EXISTS idx_alarm_tenant_alarm_type_created_time ON alarm(tenant_id, type, created_time DESC); -CREATE INDEX IF NOT EXISTS idx_event_type_entity_id ON event(tenant_id, event_type, entity_type, entity_id); - CREATE INDEX IF NOT EXISTS idx_relation_to_id ON relation(relation_type_group, to_type, to_id); CREATE INDEX IF NOT EXISTS idx_relation_from_id ON relation(relation_type_group, from_type, from_id); @@ -47,3 +45,19 @@ CREATE INDEX IF NOT EXISTS idx_attribute_kv_by_key_and_last_update_ts ON attribu CREATE INDEX IF NOT EXISTS idx_audit_log_tenant_id_and_created_time ON audit_log(tenant_id, created_time); CREATE INDEX IF NOT EXISTS idx_rpc_tenant_id_device_id ON rpc(tenant_id, device_id); + +CREATE INDEX IF NOT EXISTS idx_event_ts + ON public.event USING btree + (ts DESC NULLS LAST) + WITH (FILLFACTOR=95); + +COMMENT ON INDEX public.idx_event_ts + IS 'This index helps to delete events by TTL using timestamp'; + +CREATE INDEX IF NOT EXISTS idx_event_tenant_entity_type_entity_event_type_created_time_des + ON public.event USING btree + (tenant_id ASC NULLS LAST, entity_type ASC NULLS LAST, entity_id ASC NULLS LAST, event_type ASC NULLS LAST, created_time DESC NULLS LAST) + WITH (FILLFACTOR=95); + +COMMENT ON INDEX public.idx_event_tenant_entity_type_entity_event_type_created_time_des + IS 'This index helps to open latest events on UI fast'; \ No newline at end of file From 059089367a077247230bcfd39a2a26b106b05507 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 8 Jul 2021 16:26:21 +0300 Subject: [PATCH 5/7] hsqldb version upgrade. postgresql specific syntax enabled for test suite sql.syntax_pgs=true --- dao/src/test/resources/sql-test.properties | 2 +- pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dao/src/test/resources/sql-test.properties b/dao/src/test/resources/sql-test.properties index 358b9ea3af..e81637ac3e 100644 --- a/dao/src/test/resources/sql-test.properties +++ b/dao/src/test/resources/sql-test.properties @@ -15,7 +15,7 @@ spring.jpa.database-platform=org.hibernate.dialect.HSQLDialect spring.datasource.username=sa spring.datasource.password= -spring.datasource.url=jdbc:hsqldb:file:target/tmp/testDb;sql.enforce_size=false +spring.datasource.url=jdbc:hsqldb:file:target/tmp/testDb;sql.enforce_size=false;sql.syntax_pgs=true spring.datasource.driverClassName=org.hsqldb.jdbc.JDBCDriver spring.datasource.hikari.maximumPoolSize = 50 diff --git a/pom.xml b/pom.xml index 5f88903559..a60a81e82c 100755 --- a/pom.xml +++ b/pom.xml @@ -90,7 +90,7 @@ 1.15.0 1.64 2.0.1 - 2.5.0 + 2.6.0 2.5.3 1.2.1 42.2.20 From 2853bbf6651a7d942470e47643a7a584c0bb65f5 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Fri, 9 Jul 2021 09:58:02 +0300 Subject: [PATCH 6/7] refactored DatabaseSchemaService (Abstract and Psql), psql-specific indexes decoupled with common indexes. SQL simplified and notes added. This fixed hsql test suite tests. PsqlEntityDatabaseSchemaServiceTest added --- .../upgrade/3.2.2/schema_update_event.sql | 6 +-- .../PsqlEntityDatabaseSchemaService.java | 16 +++++- .../SqlAbstractDatabaseSchemaService.java | 27 +++++----- .../PsqlEntityDatabaseSchemaServiceTest.java | 54 +++++++++++++++++++ .../sql/schema-entities-idx-psql-addon.sql | 38 +++++++++++++ .../resources/sql/schema-entities-idx.sql | 16 ------ 6 files changed, 123 insertions(+), 34 deletions(-) create mode 100644 application/src/test/java/org/thingsboard/server/service/install/PsqlEntityDatabaseSchemaServiceTest.java create mode 100644 dao/src/main/resources/sql/schema-entities-idx-psql-addon.sql diff --git a/application/src/main/data/upgrade/3.2.2/schema_update_event.sql b/application/src/main/data/upgrade/3.2.2/schema_update_event.sql index c8c4e428c8..9b673bc7b6 100644 --- a/application/src/main/data/upgrade/3.2.2/schema_update_event.sql +++ b/application/src/main/data/upgrade/3.2.2/schema_update_event.sql @@ -64,7 +64,7 @@ DROP INDEX IF EXISTS public.idx_event_ts; -- Hint: add CONCURRENTLY to CREATE INDEX query in case of more then 1 million records or during live update -- CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_event_ts CREATE INDEX IF NOT EXISTS idx_event_ts - ON public.event USING btree + ON public.event (ts DESC NULLS LAST) WITH (FILLFACTOR=95); @@ -78,8 +78,8 @@ DROP INDEX IF EXISTS public.idx_event_tenant_entity_type_entity_event_type_creat -- CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_event_tenant_entity_type_entity_event_type_created_time_des CREATE INDEX IF NOT EXISTS idx_event_tenant_entity_type_entity_event_type_created_time_des - ON public.event USING btree - (tenant_id ASC NULLS LAST, entity_type ASC NULLS LAST, entity_id ASC NULLS LAST, event_type ASC NULLS LAST, created_time DESC NULLS LAST) + ON public.event + (tenant_id ASC, entity_type ASC, entity_id ASC, event_type ASC, created_time DESC NULLS LAST) WITH (FILLFACTOR=95); COMMENT ON INDEX public.idx_event_tenant_entity_type_entity_event_type_created_time_des diff --git a/application/src/main/java/org/thingsboard/server/service/install/PsqlEntityDatabaseSchemaService.java b/application/src/main/java/org/thingsboard/server/service/install/PsqlEntityDatabaseSchemaService.java index 6bf3f4822d..e47a8ae0da 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/PsqlEntityDatabaseSchemaService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/PsqlEntityDatabaseSchemaService.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.install; +import lombok.extern.slf4j.Slf4j; import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Service; import org.thingsboard.server.dao.util.PsqlDao; @@ -22,9 +23,22 @@ import org.thingsboard.server.dao.util.PsqlDao; @Service @PsqlDao @Profile("install") +@Slf4j public class PsqlEntityDatabaseSchemaService extends SqlAbstractDatabaseSchemaService implements EntityDatabaseSchemaService { + public static final String SCHEMA_ENTITIES_SQL = "schema-entities.sql"; + public static final String SCHEMA_ENTITIES_IDX_SQL = "schema-entities-idx.sql"; + public static final String SCHEMA_ENTITIES_IDX_PSQL_ADDON_SQL = "schema-entities-idx-psql-addon.sql"; + public PsqlEntityDatabaseSchemaService() { - super("schema-entities.sql", "schema-entities-idx.sql"); + super(SCHEMA_ENTITIES_SQL, SCHEMA_ENTITIES_IDX_SQL); } + + @Override + public void createDatabaseIndexes() throws Exception { + super.createDatabaseIndexes(); + log.info("Installing SQL DataBase schema PostgreSQL specific indexes part: " + SCHEMA_ENTITIES_IDX_PSQL_ADDON_SQL); + executeQueryFromFile(SCHEMA_ENTITIES_IDX_PSQL_ADDON_SQL); + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/install/SqlAbstractDatabaseSchemaService.java b/application/src/main/java/org/thingsboard/server/service/install/SqlAbstractDatabaseSchemaService.java index 1e652880a0..ec56cc39b4 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/SqlAbstractDatabaseSchemaService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/SqlAbstractDatabaseSchemaService.java @@ -19,7 +19,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; -import java.nio.charset.Charset; +import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -59,14 +59,8 @@ public abstract class SqlAbstractDatabaseSchemaService implements DatabaseSchema @Override public void createDatabaseSchema(boolean createIndexes) throws Exception { - log.info("Installing SQL DataBase schema part: " + schemaSql); - - Path schemaFile = Paths.get(installScripts.getDataDir(), SQL_DIR, schemaSql); - try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) { - String sql = new String(Files.readAllBytes(schemaFile), Charset.forName("UTF-8")); - conn.createStatement().execute(sql); //NOSONAR, ignoring because method used to load initial thingsboard database schema - } + executeQueryFromFile(schemaSql); if (createIndexes) { this.createDatabaseIndexes(); @@ -77,11 +71,15 @@ public abstract class SqlAbstractDatabaseSchemaService implements DatabaseSchema public void createDatabaseIndexes() throws Exception { if (schemaIdxSql != null) { log.info("Installing SQL DataBase schema indexes part: " + schemaIdxSql); - Path schemaIdxFile = Paths.get(installScripts.getDataDir(), SQL_DIR, schemaIdxSql); - try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) { - String sql = new String(Files.readAllBytes(schemaIdxFile), Charset.forName("UTF-8")); - conn.createStatement().execute(sql); //NOSONAR, ignoring because method used to load initial thingsboard database schema - } + executeQueryFromFile(schemaIdxSql); + } + } + + void executeQueryFromFile(String schemaIdxSql) throws SQLException, IOException { + Path schemaIdxFile = Paths.get(installScripts.getDataDir(), SQL_DIR, schemaIdxSql); + String sql = Files.readString(schemaIdxFile); + try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) { + conn.createStatement().execute(sql); //NOSONAR, ignoring because method used to load initial thingsboard database schema } } @@ -91,7 +89,8 @@ public abstract class SqlAbstractDatabaseSchemaService implements DatabaseSchema log.info("Successfully executed query: {}", query); Thread.sleep(5000); } catch (InterruptedException | SQLException e) { - log.info("Failed to execute query: {} due to: {}", query, e.getMessage()); + log.error("Failed to execute query: {} due to: {}", query, e.getMessage()); + throw new RuntimeException("Failed to execute query: " + query, e); } } diff --git a/application/src/test/java/org/thingsboard/server/service/install/PsqlEntityDatabaseSchemaServiceTest.java b/application/src/test/java/org/thingsboard/server/service/install/PsqlEntityDatabaseSchemaServiceTest.java new file mode 100644 index 0000000000..eb08390ec7 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/service/install/PsqlEntityDatabaseSchemaServiceTest.java @@ -0,0 +1,54 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.install; + +import org.junit.Test; + +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.BDDMockito.willDoNothing; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +public class PsqlEntityDatabaseSchemaServiceTest { + + @Test + public void givenPsqlDbSchemaService_whenCreateDatabaseSchema_thenVerifyPsqlIndexSpecificCall() throws Exception { + PsqlEntityDatabaseSchemaService service = spy(new PsqlEntityDatabaseSchemaService()); + willDoNothing().given(service).executeQueryFromFile(anyString()); + + service.createDatabaseSchema(); + + verify(service, times(1)).createDatabaseIndexes(); + verify(service, times(1)).executeQueryFromFile(PsqlEntityDatabaseSchemaService.SCHEMA_ENTITIES_SQL); + verify(service, times(1)).executeQueryFromFile(PsqlEntityDatabaseSchemaService.SCHEMA_ENTITIES_IDX_SQL); + verify(service, times(1)).executeQueryFromFile(PsqlEntityDatabaseSchemaService.SCHEMA_ENTITIES_IDX_PSQL_ADDON_SQL); + verify(service, times(3)).executeQueryFromFile(anyString()); + } + + @Test + public void givenPsqlDbSchemaService_whenCreateDatabaseIndexes_thenVerifyPsqlIndexSpecificCall() throws Exception { + PsqlEntityDatabaseSchemaService service = spy(new PsqlEntityDatabaseSchemaService()); + willDoNothing().given(service).executeQueryFromFile(anyString()); + + service.createDatabaseIndexes(); + + verify(service, times(1)).executeQueryFromFile(PsqlEntityDatabaseSchemaService.SCHEMA_ENTITIES_IDX_SQL); + verify(service, times(1)).executeQueryFromFile(PsqlEntityDatabaseSchemaService.SCHEMA_ENTITIES_IDX_PSQL_ADDON_SQL); + verify(service, times(2)).executeQueryFromFile(anyString()); + } + +} diff --git a/dao/src/main/resources/sql/schema-entities-idx-psql-addon.sql b/dao/src/main/resources/sql/schema-entities-idx-psql-addon.sql new file mode 100644 index 0000000000..9884d48a1b --- /dev/null +++ b/dao/src/main/resources/sql/schema-entities-idx-psql-addon.sql @@ -0,0 +1,38 @@ +-- +-- Copyright © 2016-2021 The Thingsboard Authors +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +-- This file describes PostgreSQL-specific indexes that not supported by hsql +-- It is not a stand-alone file! Run schema-entities-idx.sql before! +-- Note: Hibernate DESC order translates to native SQL "ORDER BY .. DESC NULLS LAST" +-- While creating index PostgreSQL transforms short notation (ts DESC) to the full (DESC NULLS FIRST) +-- That difference between NULLS LAST and NULLS FIRST prevents to hit index while querying latest by ts +-- That why we need to define DESC index explicitly as (ts DESC NULLS LAST) + +CREATE INDEX IF NOT EXISTS idx_event_ts + ON public.event + (ts DESC NULLS LAST) + WITH (FILLFACTOR=95); + +COMMENT ON INDEX public.idx_event_ts + IS 'This index helps to delete events by TTL using timestamp'; + +CREATE INDEX IF NOT EXISTS idx_event_tenant_entity_type_entity_event_type_created_time_des + ON public.event + (tenant_id ASC, entity_type ASC, entity_id ASC, event_type ASC, created_time DESC NULLS LAST) + WITH (FILLFACTOR=95); + +COMMENT ON INDEX public.idx_event_tenant_entity_type_entity_event_type_created_time_des + IS 'This index helps to open latest events on UI fast'; \ No newline at end of file diff --git a/dao/src/main/resources/sql/schema-entities-idx.sql b/dao/src/main/resources/sql/schema-entities-idx.sql index 443d99faad..78b1eda905 100644 --- a/dao/src/main/resources/sql/schema-entities-idx.sql +++ b/dao/src/main/resources/sql/schema-entities-idx.sql @@ -45,19 +45,3 @@ CREATE INDEX IF NOT EXISTS idx_attribute_kv_by_key_and_last_update_ts ON attribu CREATE INDEX IF NOT EXISTS idx_audit_log_tenant_id_and_created_time ON audit_log(tenant_id, created_time); CREATE INDEX IF NOT EXISTS idx_rpc_tenant_id_device_id ON rpc(tenant_id, device_id); - -CREATE INDEX IF NOT EXISTS idx_event_ts - ON public.event USING btree - (ts DESC NULLS LAST) - WITH (FILLFACTOR=95); - -COMMENT ON INDEX public.idx_event_ts - IS 'This index helps to delete events by TTL using timestamp'; - -CREATE INDEX IF NOT EXISTS idx_event_tenant_entity_type_entity_event_type_created_time_des - ON public.event USING btree - (tenant_id ASC NULLS LAST, entity_type ASC NULLS LAST, entity_id ASC NULLS LAST, event_type ASC NULLS LAST, created_time DESC NULLS LAST) - WITH (FILLFACTOR=95); - -COMMENT ON INDEX public.idx_event_tenant_entity_type_entity_event_type_created_time_des - IS 'This index helps to open latest events on UI fast'; \ No newline at end of file From 0754cfe9c9e7a3fc9d6d084743edf5a5eaf34ddc Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Fri, 9 Jul 2021 10:24:35 +0300 Subject: [PATCH 7/7] fixed merge for schema-entities-idx.sql --- .../sql/schema-entities-idx-psql-addon.sql | 2 +- .../main/resources/sql/schema-entities-idx.sql | 16 ---------------- 2 files changed, 1 insertion(+), 17 deletions(-) diff --git a/dao/src/main/resources/sql/schema-entities-idx-psql-addon.sql b/dao/src/main/resources/sql/schema-entities-idx-psql-addon.sql index 9884d48a1b..5ea65a3380 100644 --- a/dao/src/main/resources/sql/schema-entities-idx-psql-addon.sql +++ b/dao/src/main/resources/sql/schema-entities-idx-psql-addon.sql @@ -35,4 +35,4 @@ CREATE INDEX IF NOT EXISTS idx_event_tenant_entity_type_entity_event_type_create WITH (FILLFACTOR=95); COMMENT ON INDEX public.idx_event_tenant_entity_type_entity_event_type_created_time_des - IS 'This index helps to open latest events on UI fast'; \ No newline at end of file + IS 'This index helps to open latest events on UI fast'; diff --git a/dao/src/main/resources/sql/schema-entities-idx.sql b/dao/src/main/resources/sql/schema-entities-idx.sql index 443d99faad..78b1eda905 100644 --- a/dao/src/main/resources/sql/schema-entities-idx.sql +++ b/dao/src/main/resources/sql/schema-entities-idx.sql @@ -45,19 +45,3 @@ CREATE INDEX IF NOT EXISTS idx_attribute_kv_by_key_and_last_update_ts ON attribu CREATE INDEX IF NOT EXISTS idx_audit_log_tenant_id_and_created_time ON audit_log(tenant_id, created_time); CREATE INDEX IF NOT EXISTS idx_rpc_tenant_id_device_id ON rpc(tenant_id, device_id); - -CREATE INDEX IF NOT EXISTS idx_event_ts - ON public.event USING btree - (ts DESC NULLS LAST) - WITH (FILLFACTOR=95); - -COMMENT ON INDEX public.idx_event_ts - IS 'This index helps to delete events by TTL using timestamp'; - -CREATE INDEX IF NOT EXISTS idx_event_tenant_entity_type_entity_event_type_created_time_des - ON public.event USING btree - (tenant_id ASC NULLS LAST, entity_type ASC NULLS LAST, entity_id ASC NULLS LAST, event_type ASC NULLS LAST, created_time DESC NULLS LAST) - WITH (FILLFACTOR=95); - -COMMENT ON INDEX public.idx_event_tenant_entity_type_entity_event_type_created_time_des - IS 'This index helps to open latest events on UI fast'; \ No newline at end of file