Separate dao implementations for dedicated datasource

This commit is contained in:
ViacheslavKlimov 2024-08-06 13:43:29 +03:00
parent 868b480ff9
commit f53f5e4fbb
14 changed files with 259 additions and 170 deletions

View File

@ -17,6 +17,7 @@ package org.thingsboard.server.controller;
import com.datastax.oss.driver.api.core.uuid.Uuids; import com.datastax.oss.driver.api.core.uuid.Uuids;
import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.core.type.TypeReference;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.junit.After; import org.junit.After;
import org.junit.Assert; import org.junit.Assert;
@ -38,7 +39,7 @@ import org.thingsboard.server.common.data.security.Authority;
import org.thingsboard.server.dao.audit.AuditLogDao; import org.thingsboard.server.dao.audit.AuditLogDao;
import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.model.ModelConstants;
import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.dao.service.DaoSqlTest;
import org.thingsboard.server.dao.sqlts.insert.sql.DedicatedSqlPartitioningRepository; import org.thingsboard.server.dao.sqlts.insert.sql.SqlPartitioningRepository;
import org.thingsboard.server.service.ttl.AuditLogsCleanUpService; import org.thingsboard.server.service.ttl.AuditLogsCleanUpService;
import java.text.ParseException; import java.text.ParseException;
@ -64,8 +65,9 @@ public class AuditLogControllerTest extends AbstractControllerTest {
@Autowired @Autowired
private AuditLogDao auditLogDao; private AuditLogDao auditLogDao;
@Getter
@SpyBean @SpyBean
private DedicatedSqlPartitioningRepository partitioningRepository; private SqlPartitioningRepository partitioningRepository;
@SpyBean @SpyBean
private AuditLogsCleanUpService auditLogsCleanUpService; private AuditLogsCleanUpService auditLogsCleanUpService;
@ -183,12 +185,12 @@ public class AuditLogControllerTest extends AbstractControllerTest {
@Test @Test
public void whenSavingNewAuditLog_thenCheckAndCreatePartitionIfNotExists() throws ParseException { public void whenSavingNewAuditLog_thenCheckAndCreatePartitionIfNotExists() throws ParseException {
long entityTs = ISO_8601_EXTENDED_DATETIME_TIME_ZONE_FORMAT.parse("2024-01-01T01:43:11Z").getTime(); long entityTs = ISO_8601_EXTENDED_DATETIME_TIME_ZONE_FORMAT.parse("2024-01-01T01:43:11Z").getTime();
reset(partitioningRepository); reset(getPartitioningRepository());
AuditLog auditLog = createAuditLog(ActionType.LOGIN, tenantAdminUserId, entityTs); AuditLog auditLog = createAuditLog(ActionType.LOGIN, tenantAdminUserId, entityTs);
verify(partitioningRepository).createPartitionIfNotExists(eq("audit_log"), eq(auditLog.getCreatedTime()), eq(partitionDurationInMs)); verify(getPartitioningRepository()).createPartitionIfNotExists(eq("audit_log"), eq(auditLog.getCreatedTime()), eq(partitionDurationInMs));
List<Long> partitions = partitioningRepository.fetchPartitions("audit_log"); List<Long> partitions = getPartitioningRepository().fetchPartitions("audit_log");
assertThat(partitions).contains(partitioningRepository.calculatePartitionStartTime(auditLog.getCreatedTime(), partitionDurationInMs)); assertThat(partitions).contains(getPartitioningRepository().calculatePartitionStartTime(auditLog.getCreatedTime(), partitionDurationInMs));
} }
@Test @Test
@ -197,15 +199,15 @@ public class AuditLogControllerTest extends AbstractControllerTest {
final long oldAuditLogTs = ISO_8601_EXTENDED_DATETIME_TIME_ZONE_FORMAT.parse("2020-10-01T00:00:00Z").getTime(); final long oldAuditLogTs = ISO_8601_EXTENDED_DATETIME_TIME_ZONE_FORMAT.parse("2020-10-01T00:00:00Z").getTime();
final long currentTimeMillis = oldAuditLogTs + TimeUnit.SECONDS.toMillis(auditLogsTtlInSec) * 2; final long currentTimeMillis = oldAuditLogTs + TimeUnit.SECONDS.toMillis(auditLogsTtlInSec) * 2;
final long partitionStartTs = partitioningRepository.calculatePartitionStartTime(oldAuditLogTs, partitionDurationInMs); final long partitionStartTs = getPartitioningRepository().calculatePartitionStartTime(oldAuditLogTs, partitionDurationInMs);
partitioningRepository.createPartitionIfNotExists("audit_log", oldAuditLogTs, partitionDurationInMs); getPartitioningRepository().createPartitionIfNotExists("audit_log", oldAuditLogTs, partitionDurationInMs);
List<Long> partitions = partitioningRepository.fetchPartitions("audit_log"); List<Long> partitions = getPartitioningRepository().fetchPartitions("audit_log");
assertThat(partitions).contains(partitionStartTs); assertThat(partitions).contains(partitionStartTs);
willReturn(currentTimeMillis).given(auditLogsCleanUpService).getCurrentTimeMillis(); willReturn(currentTimeMillis).given(auditLogsCleanUpService).getCurrentTimeMillis();
auditLogsCleanUpService.cleanUp(); auditLogsCleanUpService.cleanUp();
partitions = partitioningRepository.fetchPartitions("audit_log"); partitions = getPartitioningRepository().fetchPartitions("audit_log");
assertThat(partitions).as("partitions cleared").doesNotContain(partitionStartTs); assertThat(partitions).as("partitions cleared").doesNotContain(partitionStartTs);
assertThat(partitions).as("only newer partitions left").allSatisfy(partitionsStart -> { assertThat(partitions).as("only newer partitions left").allSatisfy(partitionsStart -> {
long partitionEndTs = partitionsStart + partitionDurationInMs; long partitionEndTs = partitionsStart + partitionDurationInMs;
@ -218,17 +220,17 @@ public class AuditLogControllerTest extends AbstractControllerTest {
// creating partition bigger than sql.audit_logs.partition_size // creating partition bigger than sql.audit_logs.partition_size
long entityTs = ISO_8601_EXTENDED_DATETIME_TIME_ZONE_FORMAT.parse("2022-04-29T07:43:11Z").getTime(); long entityTs = ISO_8601_EXTENDED_DATETIME_TIME_ZONE_FORMAT.parse("2022-04-29T07:43:11Z").getTime();
//the partition 7 days is overlapping default partition size 1 day, use in the far past to not affect other tests //the partition 7 days is overlapping default partition size 1 day, use in the far past to not affect other tests
partitioningRepository.createPartitionIfNotExists("audit_log", entityTs, TimeUnit.DAYS.toMillis(7)); getPartitioningRepository().createPartitionIfNotExists("audit_log", entityTs, TimeUnit.DAYS.toMillis(7));
List<Long> partitions = partitioningRepository.fetchPartitions("audit_log"); List<Long> partitions = getPartitioningRepository().fetchPartitions("audit_log");
log.warn("entityTs [{}], fetched partitions {}", entityTs, partitions); log.warn("entityTs [{}], fetched partitions {}", entityTs, partitions);
assertThat(partitions).contains(ISO_8601_EXTENDED_DATETIME_TIME_ZONE_FORMAT.parse("2022-04-28T00:00:00Z").getTime()); assertThat(partitions).contains(ISO_8601_EXTENDED_DATETIME_TIME_ZONE_FORMAT.parse("2022-04-28T00:00:00Z").getTime());
partitioningRepository.cleanupPartitionsCache("audit_log", entityTs, 0); getPartitioningRepository().cleanupPartitionsCache("audit_log", entityTs, 0);
assertDoesNotThrow(() -> { assertDoesNotThrow(() -> {
// expecting partition overlap error on partition save // expecting partition overlap error on partition save
createAuditLog(ActionType.LOGIN, tenantAdminUserId, entityTs); createAuditLog(ActionType.LOGIN, tenantAdminUserId, entityTs);
}); });
assertThat(partitioningRepository.fetchPartitions("audit_log")) assertThat(getPartitioningRepository().fetchPartitions("audit_log"))
.contains(ISO_8601_EXTENDED_DATETIME_TIME_ZONE_FORMAT.parse("2022-04-28T00:00:00Z").getTime()); .contains(ISO_8601_EXTENDED_DATETIME_TIME_ZONE_FORMAT.parse("2022-04-28T00:00:00Z").getTime());
} }

View File

@ -15,8 +15,11 @@
*/ */
package org.thingsboard.server.controller; package org.thingsboard.server.controller;
import lombok.Getter;
import org.springframework.boot.test.mock.mockito.SpyBean;
import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.TestPropertySource;
import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.dao.service.DaoSqlTest;
import org.thingsboard.server.dao.sqlts.insert.sql.DedicatedSqlPartitioningRepository;
@DaoSqlTest @DaoSqlTest
@TestPropertySource(properties = { @TestPropertySource(properties = {
@ -25,4 +28,9 @@ import org.thingsboard.server.dao.service.DaoSqlTest;
"spring.datasource.dedicated.driverClassName=${spring.datasource.driverClassName}", "spring.datasource.dedicated.driverClassName=${spring.datasource.driverClassName}",
}) })
public class AuditLogControllerTest_DedicatedDataSource extends AuditLogControllerTest { public class AuditLogControllerTest_DedicatedDataSource extends AuditLogControllerTest {
@Getter
@SpyBean
private DedicatedSqlPartitioningRepository partitioningRepository;
} }

View File

@ -13,10 +13,14 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.thingsboard.server.dao.sql.event; package org.thingsboard.server.dao.config;
public interface EventCleanupRepository { import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
void cleanupEvents(long eventExpTime, boolean debug); import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
@Retention(RetentionPolicy.RUNTIME)
@ConditionalOnProperty(value = "spring.datasource.dedicated.enabled", havingValue = "true")
public @interface DedicatedDataSource {
} }

View File

@ -17,7 +17,6 @@ package org.thingsboard.server.dao.config;
import com.zaxxer.hikari.HikariDataSource; import com.zaxxer.hikari.HikariDataSource;
import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.boot.autoconfigure.jdbc.DataSourceProperties; import org.springframework.boot.autoconfigure.jdbc.DataSourceProperties;
import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.boot.orm.jpa.EntityManagerFactoryBuilder; import org.springframework.boot.orm.jpa.EntityManagerFactoryBuilder;
@ -46,7 +45,7 @@ import java.util.Objects;
* - add the package of this JpaRepository to @EnableJpaRepositories in DedicatedJpaDaoConfig * - add the package of this JpaRepository to @EnableJpaRepositories in DedicatedJpaDaoConfig
* - add the entity class to packages list in dedicatedEntityManagerFactory in DedicatedJpaDaoConfig * - add the entity class to packages list in dedicatedEntityManagerFactory in DedicatedJpaDaoConfig
* */ * */
@ConditionalOnProperty(value = "spring.datasource.dedicated.enabled", havingValue = "true") @DedicatedDataSource
@Configuration @Configuration
@EnableJpaRepositories(value = {"org.thingsboard.server.dao.sql.event", "org.thingsboard.server.dao.sql.audit"}, @EnableJpaRepositories(value = {"org.thingsboard.server.dao.sql.event", "org.thingsboard.server.dao.sql.audit"},
bootstrapMode = BootstrapMode.LAZY, bootstrapMode = BootstrapMode.LAZY,

View File

@ -0,0 +1,26 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.dao.config;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
@Retention(RetentionPolicy.RUNTIME)
@ConditionalOnProperty(value = "spring.datasource.dedicated.enabled", havingValue = "false", matchIfMissing = true)
public @interface DefaultDataSource {
}

View File

@ -15,38 +15,13 @@
*/ */
package org.thingsboard.server.dao.config; package org.thingsboard.server.dao.config;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Configuration;
import org.springframework.data.jpa.repository.config.EnableJpaRepositories; import org.springframework.data.jpa.repository.config.EnableJpaRepositories;
import org.springframework.data.repository.config.BootstrapMode; import org.springframework.data.repository.config.BootstrapMode;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.orm.jpa.JpaTransactionManager;
import org.springframework.transaction.support.TransactionTemplate;
import static org.thingsboard.server.dao.config.DedicatedJpaDaoConfig.DEDICATED_JDBC_TEMPLATE; @DefaultDataSource
import static org.thingsboard.server.dao.config.DedicatedJpaDaoConfig.DEDICATED_TRANSACTION_MANAGER;
import static org.thingsboard.server.dao.config.DedicatedJpaDaoConfig.DEDICATED_TRANSACTION_TEMPLATE;
@ConditionalOnProperty(value = "spring.datasource.dedicated.enabled", havingValue = "false", matchIfMissing = true)
@Configuration @Configuration
@EnableJpaRepositories(value = {"org.thingsboard.server.dao.sql.event", "org.thingsboard.server.dao.sql.audit"}, bootstrapMode = BootstrapMode.LAZY) @EnableJpaRepositories(value = {"org.thingsboard.server.dao.sql.event", "org.thingsboard.server.dao.sql.audit"}, bootstrapMode = BootstrapMode.LAZY)
public class DefaultDedicatedJpaDaoConfig { public class DefaultDedicatedJpaDaoConfig {
@Bean(DEDICATED_JDBC_TEMPLATE)
public JdbcTemplate dedicatedJdbcTemplate(@Qualifier("jdbcTemplate") JdbcTemplate defaultJdbcTemplate) {
return defaultJdbcTemplate;
}
@Bean(DEDICATED_TRANSACTION_MANAGER)
public JpaTransactionManager dedicatedTransactionManager(@Qualifier("transactionManager") JpaTransactionManager defaultTransactionManager) {
return defaultTransactionManager;
}
@Bean(DEDICATED_TRANSACTION_TEMPLATE)
public TransactionTemplate dedicatedTransactionTemplate(@Qualifier("transactionTemplate") TransactionTemplate defaultTransactionTemplate) {
return defaultTransactionTemplate;
}
} }

View File

@ -0,0 +1,87 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.dao.sql.audit;
import jakarta.persistence.EntityManager;
import jakarta.persistence.PersistenceContext;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Component;
import org.springframework.transaction.annotation.Transactional;
import org.thingsboard.server.common.data.audit.AuditLog;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.dao.config.DedicatedDataSource;
import org.thingsboard.server.dao.sqlts.insert.sql.DedicatedSqlPartitioningRepository;
import org.thingsboard.server.dao.util.SqlDao;
import java.util.Collection;
import java.util.UUID;
import static org.thingsboard.server.dao.config.DedicatedJpaDaoConfig.DEDICATED_JDBC_TEMPLATE;
import static org.thingsboard.server.dao.config.DedicatedJpaDaoConfig.DEDICATED_PERSISTENCE_UNIT;
import static org.thingsboard.server.dao.config.DedicatedJpaDaoConfig.DEDICATED_TRANSACTION_MANAGER;
@DedicatedDataSource
@Component
@SqlDao
public class DedicatedJpaAuditLogDao extends JpaAuditLogDao {
@Autowired
@Qualifier(DEDICATED_JDBC_TEMPLATE)
private JdbcTemplate jdbcTemplate;
@PersistenceContext(unitName = DEDICATED_PERSISTENCE_UNIT)
private EntityManager entityManager;
public DedicatedJpaAuditLogDao(AuditLogRepository auditLogRepository, DedicatedSqlPartitioningRepository partitioningRepository) {
super(auditLogRepository, partitioningRepository);
}
@Transactional(transactionManager = DEDICATED_TRANSACTION_MANAGER)
@Override
public AuditLog save(TenantId tenantId, AuditLog domain) {
return super.save(tenantId, domain);
}
@Transactional(transactionManager = DEDICATED_TRANSACTION_MANAGER)
@Override
public AuditLog saveAndFlush(TenantId tenantId, AuditLog domain) {
return super.saveAndFlush(tenantId, domain);
}
@Transactional(transactionManager = DEDICATED_TRANSACTION_MANAGER)
@Override
public boolean removeById(TenantId tenantId, UUID id) {
return super.removeById(tenantId, id);
}
@Transactional(transactionManager = DEDICATED_TRANSACTION_MANAGER)
@Override
public void removeAllByIds(Collection<UUID> ids) {
super.removeAllByIds(ids);
}
@Override
protected EntityManager getEntityManager() {
return entityManager;
}
@Override
protected JdbcTemplate getJdbcTemplate() {
return jdbcTemplate;
}
}

View File

@ -15,42 +15,33 @@
*/ */
package org.thingsboard.server.dao.sql.audit; package org.thingsboard.server.dao.sql.audit;
import jakarta.persistence.EntityManager;
import jakarta.persistence.PersistenceContext;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Value;
import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import org.springframework.transaction.annotation.Transactional;
import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.audit.ActionType;
import org.thingsboard.server.common.data.audit.AuditLog; import org.thingsboard.server.common.data.audit.AuditLog;
import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.CustomerId;
import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.id.UserId;
import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.TimePageLink; import org.thingsboard.server.common.data.page.TimePageLink;
import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.DaoUtil;
import org.thingsboard.server.dao.audit.AuditLogDao; import org.thingsboard.server.dao.audit.AuditLogDao;
import org.thingsboard.server.dao.config.DefaultDataSource;
import org.thingsboard.server.dao.model.sql.AuditLogEntity; import org.thingsboard.server.dao.model.sql.AuditLogEntity;
import org.thingsboard.server.dao.sql.JpaPartitionedAbstractDao; import org.thingsboard.server.dao.sql.JpaPartitionedAbstractDao;
import org.thingsboard.server.dao.sqlts.insert.sql.DedicatedSqlPartitioningRepository; import org.thingsboard.server.dao.sqlts.insert.sql.SqlPartitioningRepository;
import org.thingsboard.server.dao.util.SqlDao; import org.thingsboard.server.dao.util.SqlDao;
import java.util.Collection;
import java.util.List; import java.util.List;
import java.util.UUID; import java.util.UUID;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import static org.thingsboard.server.dao.config.DedicatedJpaDaoConfig.DEDICATED_JDBC_TEMPLATE;
import static org.thingsboard.server.dao.config.DedicatedJpaDaoConfig.DEDICATED_PERSISTENCE_UNIT;
import static org.thingsboard.server.dao.config.DedicatedJpaDaoConfig.DEDICATED_TRANSACTION_MANAGER;
import static org.thingsboard.server.dao.model.ModelConstants.AUDIT_LOG_TABLE_NAME; import static org.thingsboard.server.dao.model.ModelConstants.AUDIT_LOG_TABLE_NAME;
@DefaultDataSource
@Component @Component
@SqlDao @SqlDao
@RequiredArgsConstructor @RequiredArgsConstructor
@ -58,40 +49,11 @@ import static org.thingsboard.server.dao.model.ModelConstants.AUDIT_LOG_TABLE_NA
public class JpaAuditLogDao extends JpaPartitionedAbstractDao<AuditLogEntity, AuditLog> implements AuditLogDao { public class JpaAuditLogDao extends JpaPartitionedAbstractDao<AuditLogEntity, AuditLog> implements AuditLogDao {
private final AuditLogRepository auditLogRepository; private final AuditLogRepository auditLogRepository;
private final DedicatedSqlPartitioningRepository partitioningRepository; private final SqlPartitioningRepository partitioningRepository;
@Autowired
@Qualifier(DEDICATED_JDBC_TEMPLATE)
private JdbcTemplate jdbcTemplate;
@PersistenceContext(unitName = DEDICATED_PERSISTENCE_UNIT)
private EntityManager entityManager;
@Value("${sql.audit_logs.partition_size:168}") @Value("${sql.audit_logs.partition_size:168}")
private int partitionSizeInHours; private int partitionSizeInHours;
@Transactional(transactionManager = DEDICATED_TRANSACTION_MANAGER)
@Override
public AuditLog save(TenantId tenantId, AuditLog domain) {
return super.save(tenantId, domain);
}
@Transactional(transactionManager = DEDICATED_TRANSACTION_MANAGER)
@Override
public AuditLog saveAndFlush(TenantId tenantId, AuditLog domain) {
return super.saveAndFlush(tenantId, domain);
}
@Transactional(transactionManager = DEDICATED_TRANSACTION_MANAGER)
@Override
public boolean removeById(TenantId tenantId, UUID id) {
return super.removeById(tenantId, id);
}
@Transactional(transactionManager = DEDICATED_TRANSACTION_MANAGER)
@Override
public void removeAllByIds(Collection<UUID> ids) {
super.removeAllByIds(ids);
}
@Override @Override
public PageData<AuditLog> findAuditLogsByTenantIdAndEntityId(UUID tenantId, EntityId entityId, List<ActionType> actionTypes, TimePageLink pageLink) { public PageData<AuditLog> findAuditLogsByTenantIdAndEntityId(UUID tenantId, EntityId entityId, List<ActionType> actionTypes, TimePageLink pageLink) {
return DaoUtil.toPageData( return DaoUtil.toPageData(
@ -157,19 +119,6 @@ public class JpaAuditLogDao extends JpaPartitionedAbstractDao<AuditLogEntity, Au
partitioningRepository.createPartitionIfNotExists(AUDIT_LOG_TABLE_NAME, entity.getCreatedTime(), TimeUnit.HOURS.toMillis(partitionSizeInHours)); partitioningRepository.createPartitionIfNotExists(AUDIT_LOG_TABLE_NAME, entity.getCreatedTime(), TimeUnit.HOURS.toMillis(partitionSizeInHours));
} }
@Override
protected EntityManager getEntityManager() {
if (entityManager != null) {
return entityManager;
}
return super.getEntityManager();
}
@Override
protected JdbcTemplate getJdbcTemplate() {
return jdbcTemplate;
}
@Override @Override
protected Class<AuditLogEntity> getEntityClass() { protected Class<AuditLogEntity> getEntityClass() {
return AuditLogEntity.class; return AuditLogEntity.class;

View File

@ -0,0 +1,36 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.dao.sql.event;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Repository;
import org.springframework.transaction.support.TransactionTemplate;
import org.thingsboard.server.dao.config.DedicatedDataSource;
import static org.thingsboard.server.dao.config.DedicatedJpaDaoConfig.DEDICATED_JDBC_TEMPLATE;
import static org.thingsboard.server.dao.config.DedicatedJpaDaoConfig.DEDICATED_TRANSACTION_TEMPLATE;
@DedicatedDataSource
@Repository
public class DedicatedEventInsertRepository extends EventInsertRepository {
public DedicatedEventInsertRepository(@Qualifier(DEDICATED_JDBC_TEMPLATE) JdbcTemplate jdbcTemplate,
@Qualifier(DEDICATED_TRANSACTION_TEMPLATE) TransactionTemplate transactionTemplate) {
super(jdbcTemplate, transactionTemplate);
}
}

View File

@ -0,0 +1,45 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.dao.sql.event;
import org.springframework.stereotype.Component;
import org.thingsboard.server.common.stats.StatsFactory;
import org.thingsboard.server.dao.config.DedicatedDataSource;
import org.thingsboard.server.dao.sql.ScheduledLogExecutorComponent;
import org.thingsboard.server.dao.sqlts.insert.sql.DedicatedSqlPartitioningRepository;
import org.thingsboard.server.dao.util.SqlDao;
@DedicatedDataSource
@Component
@SqlDao
public class DedicatedJpaEventDao extends JpaBaseEventDao {
public DedicatedJpaEventDao(EventPartitionConfiguration partitionConfiguration,
DedicatedSqlPartitioningRepository partitioningRepository,
LifecycleEventRepository lcEventRepository,
StatisticsEventRepository statsEventRepository,
ErrorEventRepository errorEventRepository,
DedicatedEventInsertRepository eventInsertRepository,
RuleNodeDebugEventRepository ruleNodeDebugEventRepository,
RuleChainDebugEventRepository ruleChainDebugEventRepository,
ScheduledLogExecutorComponent logExecutor,
StatsFactory statsFactory) {
super(partitionConfiguration, partitioningRepository, lcEventRepository, statsEventRepository,
errorEventRepository, eventInsertRepository, ruleNodeDebugEventRepository,
ruleChainDebugEventRepository, logExecutor, statsFactory);
}
}

View File

@ -16,9 +16,7 @@
package org.thingsboard.server.dao.sql.event; package org.thingsboard.server.dao.sql.event;
import jakarta.annotation.PostConstruct; import jakarta.annotation.PostConstruct;
import lombok.Getter; import lombok.RequiredArgsConstructor;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Value;
import org.springframework.jdbc.core.BatchPreparedStatementSetter; import org.springframework.jdbc.core.BatchPreparedStatementSetter;
import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.core.JdbcTemplate;
@ -34,6 +32,7 @@ import org.thingsboard.server.common.data.event.LifecycleEvent;
import org.thingsboard.server.common.data.event.RuleChainDebugEvent; import org.thingsboard.server.common.data.event.RuleChainDebugEvent;
import org.thingsboard.server.common.data.event.RuleNodeDebugEvent; import org.thingsboard.server.common.data.event.RuleNodeDebugEvent;
import org.thingsboard.server.common.data.event.StatisticsEvent; import org.thingsboard.server.common.data.event.StatisticsEvent;
import org.thingsboard.server.dao.config.DefaultDataSource;
import org.thingsboard.server.dao.util.SqlDao; import org.thingsboard.server.dao.util.SqlDao;
import java.sql.PreparedStatement; import java.sql.PreparedStatement;
@ -46,12 +45,11 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import static org.thingsboard.server.dao.config.DedicatedJpaDaoConfig.DEDICATED_JDBC_TEMPLATE; @DefaultDataSource
import static org.thingsboard.server.dao.config.DedicatedJpaDaoConfig.DEDICATED_TRANSACTION_TEMPLATE;
@Repository @Repository
@Transactional @Transactional
@SqlDao @SqlDao
@RequiredArgsConstructor
public class EventInsertRepository { public class EventInsertRepository {
private static final ThreadLocal<Pattern> PATTERN_THREAD_LOCAL = ThreadLocal.withInitial(() -> Pattern.compile(String.valueOf(Character.MIN_VALUE))); private static final ThreadLocal<Pattern> PATTERN_THREAD_LOCAL = ThreadLocal.withInitial(() -> Pattern.compile(String.valueOf(Character.MIN_VALUE)));
@ -60,14 +58,8 @@ public class EventInsertRepository {
private final Map<EventType, String> insertStmtMap = new ConcurrentHashMap<>(); private final Map<EventType, String> insertStmtMap = new ConcurrentHashMap<>();
@Getter private final JdbcTemplate jdbcTemplate;
@Autowired private final TransactionTemplate transactionTemplate;
@Qualifier(DEDICATED_JDBC_TEMPLATE)
protected JdbcTemplate jdbcTemplate;
@Autowired
@Qualifier(DEDICATED_TRANSACTION_TEMPLATE)
private TransactionTemplate transactionTemplate;
@Value("${sql.remove_null_chars:true}") @Value("${sql.remove_null_chars:true}")
private boolean removeNullChars; private boolean removeNullChars;
@ -244,4 +236,5 @@ public class EventInsertRepository {
} }
return strValue; return strValue;
} }
} }

View File

@ -38,12 +38,13 @@ import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.TimePageLink; import org.thingsboard.server.common.data.page.TimePageLink;
import org.thingsboard.server.common.stats.StatsFactory; import org.thingsboard.server.common.stats.StatsFactory;
import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.DaoUtil;
import org.thingsboard.server.dao.config.DefaultDataSource;
import org.thingsboard.server.dao.event.EventDao; import org.thingsboard.server.dao.event.EventDao;
import org.thingsboard.server.dao.model.sql.EventEntity; import org.thingsboard.server.dao.model.sql.EventEntity;
import org.thingsboard.server.dao.sql.ScheduledLogExecutorComponent; import org.thingsboard.server.dao.sql.ScheduledLogExecutorComponent;
import org.thingsboard.server.dao.sql.TbSqlBlockingQueueParams; import org.thingsboard.server.dao.sql.TbSqlBlockingQueueParams;
import org.thingsboard.server.dao.sql.TbSqlBlockingQueueWrapper; import org.thingsboard.server.dao.sql.TbSqlBlockingQueueWrapper;
import org.thingsboard.server.dao.sqlts.insert.sql.DedicatedSqlPartitioningRepository; import org.thingsboard.server.dao.sqlts.insert.sql.SqlPartitioningRepository;
import org.thingsboard.server.dao.util.SqlDao; import org.thingsboard.server.dao.util.SqlDao;
import java.util.Comparator; import java.util.Comparator;
@ -54,22 +55,19 @@ import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import java.util.function.Function; import java.util.function.Function;
/** @DefaultDataSource
* Created by Valerii Sosliuk on 5/3/2017.
*/
@Slf4j
@Component @Component
@SqlDao @SqlDao
@RequiredArgsConstructor @RequiredArgsConstructor
@Slf4j
public class JpaBaseEventDao implements EventDao { public class JpaBaseEventDao implements EventDao {
private final EventPartitionConfiguration partitionConfiguration; private final EventPartitionConfiguration partitionConfiguration;
private final DedicatedSqlPartitioningRepository partitioningRepository; private final SqlPartitioningRepository partitioningRepository;
private final LifecycleEventRepository lcEventRepository; private final LifecycleEventRepository lcEventRepository;
private final StatisticsEventRepository statsEventRepository; private final StatisticsEventRepository statsEventRepository;
private final ErrorEventRepository errorEventRepository; private final ErrorEventRepository errorEventRepository;
private final EventInsertRepository eventInsertRepository; private final EventInsertRepository eventInsertRepository;
private final EventCleanupRepository eventCleanupRepository;
private final RuleNodeDebugEventRepository ruleNodeDebugEventRepository; private final RuleNodeDebugEventRepository ruleNodeDebugEventRepository;
private final RuleChainDebugEventRepository ruleChainDebugEventRepository; private final RuleChainDebugEventRepository ruleChainDebugEventRepository;
private final ScheduledLogExecutorComponent logExecutor; private final ScheduledLogExecutorComponent logExecutor;
@ -377,7 +375,7 @@ public class JpaBaseEventDao implements EventDao {
if (regularEventExpTs > 0) { if (regularEventExpTs > 0) {
log.info("Going to cleanup regular events with exp time: {}", regularEventExpTs); log.info("Going to cleanup regular events with exp time: {}", regularEventExpTs);
if (cleanupDb) { if (cleanupDb) {
eventCleanupRepository.cleanupEvents(regularEventExpTs, false); cleanupEvents(regularEventExpTs, false);
} else { } else {
cleanupPartitionsCache(regularEventExpTs, false); cleanupPartitionsCache(regularEventExpTs, false);
} }
@ -385,13 +383,25 @@ public class JpaBaseEventDao implements EventDao {
if (debugEventExpTs > 0) { if (debugEventExpTs > 0) {
log.info("Going to cleanup debug events with exp time: {}", debugEventExpTs); log.info("Going to cleanup debug events with exp time: {}", debugEventExpTs);
if (cleanupDb) { if (cleanupDb) {
eventCleanupRepository.cleanupEvents(debugEventExpTs, true); cleanupEvents(debugEventExpTs, true);
} else { } else {
cleanupPartitionsCache(debugEventExpTs, true); cleanupPartitionsCache(debugEventExpTs, true);
} }
} }
} }
private void cleanupEvents(long eventExpTime, boolean debug) {
for (EventType eventType : EventType.values()) {
if (eventType.isDebug() == debug) {
cleanupPartitions(eventType, eventExpTime);
}
}
}
private void cleanupPartitions(EventType eventType, long eventExpTime) {
partitioningRepository.dropPartitionsBefore(eventType.getTable(), eventExpTime, partitionConfiguration.getPartitionSizeInMs(eventType));
}
private void cleanupPartitionsCache(long expTime, boolean isDebug) { private void cleanupPartitionsCache(long expTime, boolean isDebug) {
for (EventType eventType : EventType.values()) { for (EventType eventType : EventType.values()) {
if (eventType.isDebug() == isDebug) { if (eventType.isDebug() == isDebug) {

View File

@ -1,47 +0,0 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.dao.sql.event;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Repository;
import org.thingsboard.server.common.data.event.EventType;
import org.thingsboard.server.dao.sql.JpaAbstractDaoListeningExecutorService;
import org.thingsboard.server.dao.sqlts.insert.sql.DedicatedSqlPartitioningRepository;
@Slf4j
@Repository
@RequiredArgsConstructor
public class SqlEventCleanupRepository extends JpaAbstractDaoListeningExecutorService implements EventCleanupRepository {
private final EventPartitionConfiguration partitionConfiguration;
private final DedicatedSqlPartitioningRepository partitioningRepository;
@Override
public void cleanupEvents(long eventExpTime, boolean debug) {
for (EventType eventType : EventType.values()) {
if (eventType.isDebug() == debug) {
cleanupEvents(eventType, eventExpTime);
}
}
}
private void cleanupEvents(EventType eventType, long eventExpTime) {
partitioningRepository.dropPartitionsBefore(eventType.getTable(), eventExpTime, partitionConfiguration.getPartitionSizeInMs(eventType));
}
}

View File

@ -21,11 +21,13 @@ import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Repository; import org.springframework.stereotype.Repository;
import org.springframework.transaction.annotation.Propagation; import org.springframework.transaction.annotation.Propagation;
import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.annotation.Transactional;
import org.thingsboard.server.dao.config.DedicatedDataSource;
import org.thingsboard.server.dao.timeseries.SqlPartition; import org.thingsboard.server.dao.timeseries.SqlPartition;
import static org.thingsboard.server.dao.config.DedicatedJpaDaoConfig.DEDICATED_JDBC_TEMPLATE; import static org.thingsboard.server.dao.config.DedicatedJpaDaoConfig.DEDICATED_JDBC_TEMPLATE;
import static org.thingsboard.server.dao.config.DedicatedJpaDaoConfig.DEDICATED_TRANSACTION_MANAGER; import static org.thingsboard.server.dao.config.DedicatedJpaDaoConfig.DEDICATED_TRANSACTION_MANAGER;
@DedicatedDataSource
@Repository @Repository
public class DedicatedSqlPartitioningRepository extends SqlPartitioningRepository { public class DedicatedSqlPartitioningRepository extends SqlPartitioningRepository {