dendi-wang/thingsboard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Dedicated datasource for events

Browse Source
This commit is contained in:
ViacheslavKlimov 2024-07-25 13:53:14 +03:00
parent 8e7d70c5b3
commit 248c268d6a
11 changed files with 230 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
application/src/main/resources/thingsboard.yml
View File

@ -762,6 +762,22 @@ spring:
# This property increases the number of connections in the pool as demand increases. At the same time, the property ensures that the pool doesn't grow to the point of exhausting a system's resources, which ultimately affects an application's performance and availability
maximumPoolSize: "${SPRING_DATASOURCE_MAXIMUM_POOL_SIZE:16}"
registerMbeans: "${SPRING_DATASOURCE_HIKARI_REGISTER_MBEANS:false}" # true - enable MBean to diagnose pools state via JMX
dedicated:
enabled: "${SPRING_DEDICATED_DATASOURCE_ENABLED:true}"
# Database driver for Spring JPA - org.postgresql.Driver
driverClassName: "${SPRING_DEDICATED_DATASOURCE_DRIVER_CLASS_NAME:org.postgresql.Driver}"
# Database connection URL
url: "${SPRING_DEDICATED_DATASOURCE_URL:jdbc:postgresql://localhost:5432/thingsboard_ce_events}"
# Database user name
username: "${SPRING_DEDICATED_DATASOURCE_USERNAME:postgres}"
# Database user password
password: "${SPRING_DEDICATED_DATASOURCE_PASSWORD:postgres}"
hikari:
# This property controls the amount of time that a connection can be out of the pool before a message is logged indicating a possible connection leak. A value of 0 means leak detection is disabled
leakDetectionThreshold: "${SPRING_DEDICATED_DATASOURCE_HIKARI_LEAK_DETECTION_THRESHOLD:0}"
# This property increases the number of connections in the pool as demand increases. At the same time, the property ensures that the pool doesn't grow to the point of exhausting a system's resources, which ultimately affects an application's performance and availability
maximumPoolSize: "${SPRING_DEDICATED_DATASOURCE_MAXIMUM_POOL_SIZE:16}"
registerMbeans: "${SPRING_DEDICATED_DATASOURCE_HIKARI_REGISTER_MBEANS:false}" # true - enable MBean to diagnose pools state via JMX
# Audit log parameters
audit-log:

111
dao/src/main/java/org/thingsboard/server/dao/DedicatedJpaDaoConfig.java Normal file
View File

@ -0,0 +1,111 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.dao;
import com.zaxxer.hikari.HikariDataSource;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.jdbc.DataSourceProperties;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.boot.orm.jpa.EntityManagerFactoryBuilder;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.data.jpa.repository.config.EnableJpaRepositories;
import org.springframework.data.repository.config.BootstrapMode;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.orm.jpa.JpaTransactionManager;
import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean;
import org.springframework.transaction.support.TransactionTemplate;
import org.thingsboard.server.dao.model.sql.ErrorEventEntity;
import org.thingsboard.server.dao.model.sql.LifecycleEventEntity;
import org.thingsboard.server.dao.model.sql.RuleChainDebugEventEntity;
import org.thingsboard.server.dao.model.sql.RuleNodeDebugEventEntity;
import org.thingsboard.server.dao.model.sql.StatisticsEventEntity;
import javax.sql.DataSource;
import java.util.Objects;
@Configuration
@EnableJpaRepositories(value = "org.thingsboard.server.dao.sql.event", bootstrapMode = BootstrapMode.LAZY,
entityManagerFactoryRef = "dedicatedEntityManagerFactory", transactionManagerRef = "dedicatedTransactionManager")
public class DedicatedJpaDaoConfig {
@Value("${spring.datasource.dedicated.enabled:false}")
private boolean dedicatedDataSourceEnabled;
@Bean
@ConfigurationProperties("spring.datasource.dedicated")
public DataSourceProperties dedicatedDataSourceProperties() {
if (dedicatedDataSourceEnabled) {
return new DataSourceProperties();
} else {
return null;
}
}
@ConfigurationProperties(prefix = "spring.datasource.dedicated.hikari")
@Bean
public DataSource dedicatedDataSource(@Qualifier("dedicatedDataSourceProperties") DataSourceProperties dedicatedDataSourceProperties) {
if (dedicatedDataSourceEnabled) {
return dedicatedDataSourceProperties.initializeDataSourceBuilder().type(HikariDataSource.class).build();
} else {
return null;
}
}
@Bean
public LocalContainerEntityManagerFactoryBean dedicatedEntityManagerFactory(@Qualifier("dedicatedDataSource") DataSource dedicatedDataSource,
@Qualifier("dataSource") DataSource defaultDataSource,
EntityManagerFactoryBuilder builder) {
if (dedicatedDataSourceEnabled) {
return builder
.dataSource(dedicatedDataSource)
.packages(LifecycleEventEntity.class, StatisticsEventEntity.class, ErrorEventEntity.class, RuleNodeDebugEventEntity.class, RuleChainDebugEventEntity.class)
.persistenceUnit("dedicated")
.build();
} else {
return null;
}
}
@Bean
public JpaTransactionManager dedicatedTransactionManager(@Qualifier("dedicatedEntityManagerFactory") LocalContainerEntityManagerFactoryBean dedicatedEntityManagerFactory) {
if (dedicatedDataSourceEnabled) {
return new JpaTransactionManager(Objects.requireNonNull(dedicatedEntityManagerFactory.getObject()));
} else {
return null;
}
}
@Bean
public TransactionTemplate dedicatedTransactionTemplate(@Qualifier("dedicatedTransactionManager") JpaTransactionManager dedicatedTransactionManager) {
if (dedicatedDataSourceEnabled) {
return new TransactionTemplate(dedicatedTransactionManager);
} else {
return null;
}
}
@Bean
public JdbcTemplate dedicatedJdbcTemplate(@Qualifier("dedicatedDataSource") DataSource dedicatedDataSource) {
if (dedicatedDataSourceEnabled) {
return new JdbcTemplate(dedicatedDataSource);
} else {
return null;
}
}
}

93
dao/src/main/java/org/thingsboard/server/dao/JpaDaoConfig.java
View File

@ -15,23 +15,98 @@
*/
package org.thingsboard.server.dao;
import org.springframework.boot.autoconfigure.domain.EntityScan;
import com.zaxxer.hikari.HikariDataSource;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.boot.autoconfigure.jdbc.DataSourceProperties;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.boot.orm.jpa.EntityManagerFactoryBuilder;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.FilterType;
import org.springframework.context.annotation.Primary;
import org.springframework.data.jpa.repository.config.EnableJpaRepositories;
import org.springframework.data.repository.config.BootstrapMode;
import org.springframework.transaction.annotation.EnableTransactionManagement;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate;
import org.springframework.orm.jpa.JpaTransactionManager;
import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean;
import org.springframework.transaction.support.TransactionTemplate;
import org.thingsboard.server.dao.sql.event.EventRepository;
import org.thingsboard.server.dao.util.TbAutoConfiguration;
/**
* @author Valerii Sosliuk
*/
import javax.sql.DataSource;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
@Configuration
@TbAutoConfiguration
@ComponentScan({"org.thingsboard.server.dao.sql", "org.thingsboard.server.dao.attributes", "org.thingsboard.server.dao.cache", "org.thingsboard.server.cache"})
@EnableJpaRepositories(value = "org.thingsboard.server.dao.sql", bootstrapMode = BootstrapMode.LAZY)
@EntityScan("org.thingsboard.server.dao.model.sql")
@EnableTransactionManagement
@ComponentScan({"org.thingsboard.server.dao.sql", "org.thingsboard.server.dao.attributes", "org.thingsboard.server.dao.sqlts.dictionary", "org.thingsboard.server.dao.cache", "org.thingsboard.server.cache"})
@EnableJpaRepositories(value = {"org.thingsboard.server.dao.sql", "org.thingsboard.server.dao.sqlts.dictionary"},
excludeFilters = @ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, classes = {
EventRepository.class
}), bootstrapMode = BootstrapMode.LAZY)
public class JpaDaoConfig {
@Bean
@ConfigurationProperties("spring.datasource")
public DataSourceProperties dataSourceProperties() {
return new DataSourceProperties();
}
@Primary
@ConfigurationProperties(prefix = "spring.datasource.hikari")
@Bean
public DataSource dataSource(@Qualifier("dataSourceProperties") DataSourceProperties dataSourceProperties) {
return dataSourceProperties.initializeDataSourceBuilder().type(HikariDataSource.class).build();
}
@Primary
@Bean
public LocalContainerEntityManagerFactoryBean entityManagerFactory(@Qualifier("dataSource") DataSource dataSource,
EntityManagerFactoryBuilder builder,
@Autowired(required = false) SqlTsLatestDaoConfig tsLatestDaoConfig,
@Autowired(required = false) SqlTsDaoConfig tsDaoConfig) {
List<String> packages = new ArrayList<>();
packages.add("org.thingsboard.server.dao.model.sql");
packages.add("org.thingsboard.server.dao.model.sqlts.dictionary");
if (tsLatestDaoConfig != null) {
packages.add("org.thingsboard.server.dao.model.sqlts.latest");
}
if (tsDaoConfig != null) {
packages.add("org.thingsboard.server.dao.model.sqlts.ts");
}
return builder
.dataSource(dataSource)
.packages(packages.toArray(String[]::new))
.persistenceUnit("default")
.build();
}
@Primary
@Bean
public JpaTransactionManager transactionManager(@Qualifier("entityManagerFactory") LocalContainerEntityManagerFactoryBean entityManagerFactory) {
return new JpaTransactionManager(Objects.requireNonNull(entityManagerFactory.getObject()));
}
@Primary
@Bean
public TransactionTemplate transactionTemplate(@Qualifier("transactionManager") JpaTransactionManager transactionManager) {
return new TransactionTemplate(transactionManager);
}
@Primary
@Bean
public JdbcTemplate jdbcTemplate(@Qualifier("dataSource") DataSource dataSource) {
return new JdbcTemplate(dataSource);
}
@Primary
@Bean
public NamedParameterJdbcTemplate namedParameterJdbcTemplate(@Qualifier("dataSource") DataSource dataSource) {
return new NamedParameterJdbcTemplate(dataSource);
}
}

34
dao/src/main/java/org/thingsboard/server/dao/SqlTimeseriesDaoConfig.java
View File

@ -1,34 +0,0 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.dao;
import org.springframework.boot.autoconfigure.domain.EntityScan;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.Configuration;
import org.springframework.data.jpa.repository.config.EnableJpaRepositories;
import org.springframework.data.repository.config.BootstrapMode;
import org.springframework.transaction.annotation.EnableTransactionManagement;
import org.thingsboard.server.dao.util.TbAutoConfiguration;
@Configuration
@TbAutoConfiguration
@ComponentScan({"org.thingsboard.server.dao.sqlts.dictionary"})
@EnableJpaRepositories(value = {"org.thingsboard.server.dao.sqlts.dictionary"}, bootstrapMode = BootstrapMode.LAZY)
@EntityScan({"org.thingsboard.server.dao.model.sqlts.dictionary"})
@EnableTransactionManagement
public class SqlTimeseriesDaoConfig {
}

2
dao/src/main/java/org/thingsboard/server/dao/SqlTsDaoConfig.java
View File

@ -15,7 +15,6 @@
*/
package org.thingsboard.server.dao;
import org.springframework.boot.autoconfigure.domain.EntityScan;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.Configuration;
import org.springframework.data.jpa.repository.config.EnableJpaRepositories;
@ -28,7 +27,6 @@ import org.thingsboard.server.dao.util.TbAutoConfiguration;
@TbAutoConfiguration
@ComponentScan({"org.thingsboard.server.dao.sqlts.sql", "org.thingsboard.server.dao.sqlts.insert.sql"})
@EnableJpaRepositories(value = {"org.thingsboard.server.dao.sqlts.ts", "org.thingsboard.server.dao.sqlts.insert.sql"}, bootstrapMode = BootstrapMode.LAZY)
@EntityScan({"org.thingsboard.server.dao.model.sqlts.ts"})
@EnableTransactionManagement
@SqlTsDao
public class SqlTsDaoConfig {

2
dao/src/main/java/org/thingsboard/server/dao/SqlTsLatestDaoConfig.java
View File

@ -15,7 +15,6 @@
*/
package org.thingsboard.server.dao;
import org.springframework.boot.autoconfigure.domain.EntityScan;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.Configuration;
import org.springframework.data.jpa.repository.config.EnableJpaRepositories;
@ -28,7 +27,6 @@ import org.thingsboard.server.dao.util.TbAutoConfiguration;
@TbAutoConfiguration
@ComponentScan({"org.thingsboard.server.dao.sqlts.sql"})
@EnableJpaRepositories(value = {"org.thingsboard.server.dao.sqlts.insert.latest.sql", "org.thingsboard.server.dao.sqlts.latest"}, bootstrapMode = BootstrapMode.LAZY)
@EntityScan({"org.thingsboard.server.dao.model.sqlts.latest"})
@EnableTransactionManagement
@SqlTsLatestDao
public class SqlTsLatestDaoConfig {

5
dao/src/main/java/org/thingsboard/server/dao/sql/event/EventInsertRepository.java
View File

@ -15,7 +15,9 @@
*/
package org.thingsboard.server.dao.sql.event;
import lombok.Getter;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.jdbc.core.BatchPreparedStatementSetter;
import org.springframework.jdbc.core.JdbcTemplate;
@ -55,10 +57,13 @@ public class EventInsertRepository {
private final Map<EventType, String> insertStmtMap = new ConcurrentHashMap<>();
@Getter
@Autowired
@Qualifier("dedicatedJdbcTemplate")
protected JdbcTemplate jdbcTemplate;
@Autowired
@Qualifier("dedicatedTransactionTemplate")
private TransactionTemplate transactionTemplate;
@Value("${sql.remove_null_chars:true}")

2
dao/src/main/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDao.java
View File

@ -157,7 +157,7 @@ public class JpaBaseEventDao implements EventDao {
}
}
partitioningRepository.createPartitionIfNotExists(event.getType().getTable(), event.getCreatedTime(),
partitionConfiguration.getPartitionSizeInMs(event.getType()));
partitionConfiguration.getPartitionSizeInMs(event.getType()), eventInsertRepository.getJdbcTemplate());
return queue.add(event);
}

12
dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/sql/SqlPartitioningRepository.java
View File

@ -49,11 +49,21 @@ public class SqlPartitioningRepository {
@Transactional(propagation = Propagation.NOT_SUPPORTED)
public void save(SqlPartition partition) {
save(partition, jdbcTemplate);
}
@Transactional(propagation = Propagation.NOT_SUPPORTED)
public void save(SqlPartition partition, JdbcTemplate jdbcTemplate) {
jdbcTemplate.execute(partition.getQuery());
}
@Transactional(propagation = Propagation.NOT_SUPPORTED) // executing non-transactionally, so that parent transaction is not aborted on partition save error
public void createPartitionIfNotExists(String table, long entityTs, long partitionDurationMs) {
createPartitionIfNotExists(table, entityTs, partitionDurationMs, jdbcTemplate);
}
@Transactional(propagation = Propagation.NOT_SUPPORTED) // executing non-transactionally, so that parent transaction is not aborted on partition save error
public void createPartitionIfNotExists(String table, long entityTs, long partitionDurationMs, JdbcTemplate jdbcTemplate) {
long partitionStartTs = calculatePartitionStartTime(entityTs, partitionDurationMs);
Map<Long, SqlPartition> partitions = tablesPartitions.computeIfAbsent(table, t -> new ConcurrentHashMap<>());
if (!partitions.containsKey(partitionStartTs)) {
@ -62,7 +72,7 @@ public class SqlPartitioningRepository {
try {
if (partitions.containsKey(partitionStartTs)) return;
log.info("Saving partition {}-{} for table {}", partition.getStart(), partition.getEnd(), table);
save(partition);
save(partition, jdbcTemplate);
log.trace("Adding partition to map: {}", partition);
partitions.put(partition.getStart(), partition);
} catch (Exception e) {

2
dao/src/test/java/org/thingsboard/server/dao/AbstractDaoServiceTest.java
View File

@ -28,7 +28,7 @@ import org.thingsboard.server.common.stats.StatsFactory;
import org.thingsboard.server.dao.service.DaoSqlTest;
@RunWith(SpringRunner.class)
@ContextConfiguration(classes = {JpaDaoConfig.class, SqlTsDaoConfig.class, SqlTsLatestDaoConfig.class, SqlTimeseriesDaoConfig.class})
@ContextConfiguration(classes = {JpaDaoConfig.class, SqlTsDaoConfig.class, SqlTsLatestDaoConfig.class, DedicatedJpaDaoConfig.class})
@DaoSqlTest
@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS)
@TestExecutionListeners({

2
dao/src/test/java/org/thingsboard/server/dao/AbstractJpaDaoTest.java
View File

@ -30,7 +30,7 @@ import org.thingsboard.server.dao.service.DaoSqlTest;
* Created by Valerii Sosliuk on 4/22/2017.
*/
@RunWith(SpringRunner.class)
@ContextConfiguration(classes = {JpaDaoConfig.class, SqlTsDaoConfig.class, SqlTsLatestDaoConfig.class, SqlTimeseriesDaoConfig.class})
@ContextConfiguration(classes = {JpaDaoConfig.class, SqlTsDaoConfig.class, SqlTsLatestDaoConfig.class, DedicatedJpaDaoConfig.class})
@DaoSqlTest
@TestExecutionListeners({
DependencyInjectionTestExecutionListener.class,