dendi-wang/thingsboard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #10669 from smatvienko-tb/fix/partitioning-tests

Browse Source 
Audit Log partitioning test improved by using fixed timestamp to avoid flaky behaviour
This commit is contained in:
Andrew Shvayka 2024-04-30 11:37:51 +03:00 committed by GitHub
commit 2f097e8bbf
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 56 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
application/src/main/java/org/thingsboard/server/service/ttl/AuditLogsCleanUpService.java
View File

@ -50,7 +50,8 @@ public class AuditLogsCleanUpService extends AbstractCleanUpService {
@Scheduled(initialDelayString = "#{T(org.apache.commons.lang3.RandomUtils).nextLong(0, ${sql.ttl.audit_logs.checking_interval_ms})}", @Scheduled(initialDelayString = "#{T(org.apache.commons.lang3.RandomUtils).nextLong(0, ${sql.ttl.audit_logs.checking_interval_ms})}",
fixedDelayString = "${sql.ttl.audit_logs.checking_interval_ms}") fixedDelayString = "${sql.ttl.audit_logs.checking_interval_ms}")
public void cleanUp() { public void cleanUp() {
long auditLogsExpTime = System.currentTimeMillis() - TimeUnit.SECONDS.toMillis(ttlInSec); long auditLogsExpTime = getCurrentTimeMillis() - TimeUnit.SECONDS.toMillis(ttlInSec);
log.debug("cleanup {}", auditLogsExpTime);
if (isSystemTenantPartitionMine()) { if (isSystemTenantPartitionMine()) {
auditLogDao.cleanUpAuditLogs(auditLogsExpTime); auditLogDao.cleanUpAuditLogs(auditLogsExpTime);
} else { } else {
@ -58,4 +59,8 @@ public class AuditLogsCleanUpService extends AbstractCleanUpService {
} }
} }
public long getCurrentTimeMillis() {
return System.currentTimeMillis();
}
} }

80
application/src/test/java/org/thingsboard/server/controller/AuditLogControllerTest.java
View File

@ -15,7 +15,9 @@
*/ */
package org.thingsboard.server.controller; package org.thingsboard.server.controller;
import com.datastax.oss.driver.api.core.uuid.Uuids;
import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.core.type.TypeReference;
import lombok.extern.slf4j.Slf4j;
import org.junit.After; import org.junit.After;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Before; import org.junit.Before;
@ -28,6 +30,7 @@ import org.thingsboard.server.common.data.Tenant;
import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.User;
import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.audit.ActionType;
import org.thingsboard.server.common.data.audit.AuditLog; import org.thingsboard.server.common.data.audit.AuditLog;
import org.thingsboard.server.common.data.id.AuditLogId;
import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.TimePageLink; import org.thingsboard.server.common.data.page.TimePageLink;
@ -38,18 +41,21 @@ import org.thingsboard.server.dao.service.DaoSqlTest;
import org.thingsboard.server.dao.sqlts.insert.sql.SqlPartitioningRepository; import org.thingsboard.server.dao.sqlts.insert.sql.SqlPartitioningRepository;
import org.thingsboard.server.service.ttl.AuditLogsCleanUpService; import org.thingsboard.server.service.ttl.AuditLogsCleanUpService;
import java.time.LocalDate; import java.text.ParseException;
import java.time.ZoneOffset;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.UUID;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import static org.apache.commons.lang3.time.DateFormatUtils.ISO_8601_EXTENDED_DATETIME_TIME_ZONE_FORMAT;
import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.mockito.ArgumentMatchers.eq; import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.BDDMockito.willReturn;
import static org.mockito.Mockito.reset; import static org.mockito.Mockito.reset;
import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verify;
@Slf4j
@DaoSqlTest @DaoSqlTest
public class AuditLogControllerTest extends AbstractControllerTest { public class AuditLogControllerTest extends AbstractControllerTest {
@ -60,7 +66,7 @@ public class AuditLogControllerTest extends AbstractControllerTest {
private AuditLogDao auditLogDao; private AuditLogDao auditLogDao;
@SpyBean @SpyBean
private SqlPartitioningRepository partitioningRepository; private SqlPartitioningRepository partitioningRepository;
@Autowired @SpyBean
private AuditLogsCleanUpService auditLogsCleanUpService; private AuditLogsCleanUpService auditLogsCleanUpService;
@Value("#{${sql.audit_logs.partition_size} * 60 * 60 * 1000}") @Value("#{${sql.audit_logs.partition_size} * 60 * 60 * 1000}")
@ -96,7 +102,7 @@ public class AuditLogControllerTest extends AbstractControllerTest {
@Test @Test
public void testAuditLogs() throws Exception { public void testAuditLogs() throws Exception {
for (int i = 0; i < 178; i++) { for (int i = 0; i < 11; i++) {
Device device = new Device(); Device device = new Device();
device.setName("Device" + i); device.setName("Device" + i);
device.setType("default"); device.setType("default");
@ -104,7 +110,7 @@ public class AuditLogControllerTest extends AbstractControllerTest {
} }
List<AuditLog> loadedAuditLogs = new ArrayList<>(); List<AuditLog> loadedAuditLogs = new ArrayList<>();
TimePageLink pageLink = new TimePageLink(23); TimePageLink pageLink = new TimePageLink(5);
PageData<AuditLog> pageData; PageData<AuditLog> pageData;
do { do {
pageData = doGetTypedWithTimePageLink("/api/audit/logs?", pageData = doGetTypedWithTimePageLink("/api/audit/logs?",
@ -116,10 +122,10 @@ public class AuditLogControllerTest extends AbstractControllerTest {
} }
} while (pageData.hasNext()); } while (pageData.hasNext());
Assert.assertEquals(178, loadedAuditLogs.size()); Assert.assertEquals(11, loadedAuditLogs.size());
loadedAuditLogs = new ArrayList<>(); loadedAuditLogs = new ArrayList<>();
pageLink = new TimePageLink(23); pageLink = new TimePageLink(5);
do { do {
pageData = doGetTypedWithTimePageLink("/api/audit/logs/customer/" + ModelConstants.NULL_UUID + "?", pageData = doGetTypedWithTimePageLink("/api/audit/logs/customer/" + ModelConstants.NULL_UUID + "?",
new TypeReference<PageData<AuditLog>>() { new TypeReference<PageData<AuditLog>>() {
@ -130,10 +136,10 @@ public class AuditLogControllerTest extends AbstractControllerTest {
} }
} while (pageData.hasNext()); } while (pageData.hasNext());
Assert.assertEquals(178, loadedAuditLogs.size()); Assert.assertEquals(11, loadedAuditLogs.size());
loadedAuditLogs = new ArrayList<>(); loadedAuditLogs = new ArrayList<>();
pageLink = new TimePageLink(23); pageLink = new TimePageLink(5);
do { do {
pageData = doGetTypedWithTimePageLink("/api/audit/logs/user/" + tenantAdmin.getId().getId().toString() + "?", pageData = doGetTypedWithTimePageLink("/api/audit/logs/user/" + tenantAdmin.getId().getId().toString() + "?",
new TypeReference<PageData<AuditLog>>() { new TypeReference<PageData<AuditLog>>() {
@ -144,7 +150,7 @@ public class AuditLogControllerTest extends AbstractControllerTest {
} }
} while (pageData.hasNext()); } while (pageData.hasNext());
Assert.assertEquals(178, loadedAuditLogs.size()); Assert.assertEquals(11, loadedAuditLogs.size());
} }
@Test @Test
@ -153,13 +159,13 @@ public class AuditLogControllerTest extends AbstractControllerTest {
device.setName("Device name"); device.setName("Device name");
device.setType("default"); device.setType("default");
Device savedDevice = doPost("/api/device", device, Device.class); Device savedDevice = doPost("/api/device", device, Device.class);
for (int i = 0; i < 178; i++) { for (int i = 0; i < 11; i++) {
savedDevice.setName("Device name" + i); savedDevice.setName("Device name" + i);
doPost("/api/device", savedDevice, Device.class); doPost("/api/device", savedDevice, Device.class);
} }
List<AuditLog> loadedAuditLogs = new ArrayList<>(); List<AuditLog> loadedAuditLogs = new ArrayList<>();
TimePageLink pageLink = new TimePageLink(23); TimePageLink pageLink = new TimePageLink(5);
PageData<AuditLog> pageData; PageData<AuditLog> pageData;
do { do {
pageData = doGetTypedWithTimePageLink("/api/audit/logs/entity/DEVICE/" + savedDevice.getId().getId() + "?", pageData = doGetTypedWithTimePageLink("/api/audit/logs/entity/DEVICE/" + savedDevice.getId().getId() + "?",
@ -171,54 +177,61 @@ public class AuditLogControllerTest extends AbstractControllerTest {
} }
} while (pageData.hasNext()); } while (pageData.hasNext());
Assert.assertEquals(179, loadedAuditLogs.size()); Assert.assertEquals(11 + 1, loadedAuditLogs.size());
} }
@Test @Test
public void whenSavingNewAuditLog_thenCheckAndCreatePartitionIfNotExists() { public void whenSavingNewAuditLog_thenCheckAndCreatePartitionIfNotExists() throws ParseException {
long entityTs = ISO_8601_EXTENDED_DATETIME_TIME_ZONE_FORMAT.parse("2024-01-01T01:43:11Z").getTime();
reset(partitioningRepository); reset(partitioningRepository);
AuditLog auditLog = createAuditLog(ActionType.LOGIN, tenantAdminUserId); AuditLog auditLog = createAuditLog(ActionType.LOGIN, tenantAdminUserId, entityTs);
verify(partitioningRepository).createPartitionIfNotExists(eq("audit_log"), eq(auditLog.getCreatedTime()), eq(partitionDurationInMs)); verify(partitioningRepository).createPartitionIfNotExists(eq("audit_log"), eq(auditLog.getCreatedTime()), eq(partitionDurationInMs));
List<Long> partitions = partitioningRepository.fetchPartitions("audit_log"); List<Long> partitions = partitioningRepository.fetchPartitions("audit_log");
assertThat(partitions).singleElement().satisfies(partitionStartTs -> { assertThat(partitions).contains(partitioningRepository.calculatePartitionStartTime(auditLog.getCreatedTime(), partitionDurationInMs));
assertThat(partitionStartTs).isEqualTo(partitioningRepository.calculatePartitionStartTime(auditLog.getCreatedTime(), partitionDurationInMs));
});
} }
@Test @Test
public void whenCleaningUpAuditLogsByTtl_thenDropOldPartitions() { public void whenCleaningUpAuditLogsByTtl_thenDropOldPartitions() throws ParseException {
long oldAuditLogTs = LocalDate.of(2020, 10, 1).atStartOfDay().toInstant(ZoneOffset.UTC).toEpochMilli();
long partitionStartTs = partitioningRepository.calculatePartitionStartTime(oldAuditLogTs, partitionDurationInMs); final long oldAuditLogTs = ISO_8601_EXTENDED_DATETIME_TIME_ZONE_FORMAT.parse("2020-10-01T00:00:00Z").getTime();
final long currentTimeMillis = oldAuditLogTs + TimeUnit.SECONDS.toMillis(auditLogsTtlInSec) * 2;
final long partitionStartTs = partitioningRepository.calculatePartitionStartTime(oldAuditLogTs, partitionDurationInMs);
partitioningRepository.createPartitionIfNotExists("audit_log", oldAuditLogTs, partitionDurationInMs); partitioningRepository.createPartitionIfNotExists("audit_log", oldAuditLogTs, partitionDurationInMs);
List<Long> partitions = partitioningRepository.fetchPartitions("audit_log"); List<Long> partitions = partitioningRepository.fetchPartitions("audit_log");
assertThat(partitions).contains(partitionStartTs); assertThat(partitions).contains(partitionStartTs);
willReturn(currentTimeMillis).given(auditLogsCleanUpService).getCurrentTimeMillis();
auditLogsCleanUpService.cleanUp(); auditLogsCleanUpService.cleanUp();
partitions = partitioningRepository.fetchPartitions("audit_log"); partitions = partitioningRepository.fetchPartitions("audit_log");
assertThat(partitions).doesNotContain(partitionStartTs); assertThat(partitions).as("partitions cleared").doesNotContain(partitionStartTs);
assertThat(partitions).allSatisfy(partitionsStart -> { assertThat(partitions).as("only newer partitions left").allSatisfy(partitionsStart -> {
long partitionEndTs = partitionsStart + partitionDurationInMs; long partitionEndTs = partitionsStart + partitionDurationInMs;
assertThat(partitionEndTs).isGreaterThan(System.currentTimeMillis() - TimeUnit.SECONDS.toMillis(auditLogsTtlInSec)); assertThat(partitionEndTs).isGreaterThan(currentTimeMillis - TimeUnit.SECONDS.toMillis(auditLogsTtlInSec));
}); });
} }
@Test @Test
public void whenSavingAuditLogAndPartitionSaveErrorOccurred_thenSaveAuditLogAnyway() throws Exception { public void whenSavingAuditLogAndPartitionSaveErrorOccurred_thenSaveAuditLogAnyway() throws Exception {
// creating partition bigger than sql.audit_logs.partition_size // creating partition bigger than sql.audit_logs.partition_size
partitioningRepository.createPartitionIfNotExists("audit_log", System.currentTimeMillis(), TimeUnit.DAYS.toMillis(7)); long entityTs = ISO_8601_EXTENDED_DATETIME_TIME_ZONE_FORMAT.parse("2024-04-29T07:43:11Z").getTime();
partitioningRepository.createPartitionIfNotExists("audit_log", entityTs, TimeUnit.DAYS.toMillis(7));
List<Long> partitions = partitioningRepository.fetchPartitions("audit_log"); List<Long> partitions = partitioningRepository.fetchPartitions("audit_log");
assertThat(partitions).size().isOne(); log.warn("entityTs [{}], fetched partitions {}", entityTs, partitions);
partitioningRepository.cleanupPartitionsCache("audit_log", System.currentTimeMillis(), 0); assertThat(partitions).contains(ISO_8601_EXTENDED_DATETIME_TIME_ZONE_FORMAT.parse("2024-04-29T00:00:00Z").getTime());
partitioningRepository.cleanupPartitionsCache("audit_log", entityTs, 0);
assertDoesNotThrow(() -> { assertDoesNotThrow(() -> {
// expecting partition overlap error on partition save // expecting partition overlap error on partition save
createAuditLog(ActionType.LOGIN, tenantAdminUserId); createAuditLog(ActionType.LOGIN, tenantAdminUserId, entityTs);
}); });
assertThat(partitioningRepository.fetchPartitions("audit_log")).isEqualTo(partitions); assertThat(partitioningRepository.fetchPartitions("audit_log"))
.contains(ISO_8601_EXTENDED_DATETIME_TIME_ZONE_FORMAT.parse("2024-04-29T00:00:00Z").getTime());;
} }
private AuditLog createAuditLog(ActionType actionType, EntityId entityId) { private AuditLog createAuditLog(ActionType actionType, EntityId entityId, long entityTs) {
AuditLog auditLog = new AuditLog(); AuditLog auditLog = new AuditLog();
auditLog.setTenantId(tenantId); auditLog.setTenantId(tenantId);
auditLog.setCustomerId(null); auditLog.setCustomerId(null);
@ -226,6 +239,13 @@ public class AuditLogControllerTest extends AbstractControllerTest {
auditLog.setEntityId(entityId); auditLog.setEntityId(entityId);
auditLog.setUserName(tenantAdmin.getEmail()); auditLog.setUserName(tenantAdmin.getEmail());
auditLog.setActionType(actionType); auditLog.setActionType(actionType);
if (entityTs > 0) {
UUID uuid = Uuids.startOf(entityTs);
auditLog.setId(new AuditLogId(uuid));
auditLog.setCreatedTime(Uuids.unixTimestamp(uuid));
}
return auditLogDao.save(tenantId, auditLog); return auditLogDao.save(tenantId, auditLog);
} }