dendi-wang/thingsboard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Performance Improvement

Browse Source
This commit is contained in:
Andrii Shvaika 2020-01-30 15:14:11 +02:00
parent 41cbdd154e
commit 52e1bef1d2
1 changed files with 37 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

62
dao/src/main/java/org/thingsboard/server/dao/sqlts/psql/JpaPsqlTimeseriesDao.java
View File

@ -48,10 +48,7 @@ import java.time.LocalDateTime;
import java.time.ZoneOffset; import java.time.ZoneOffset;
import java.time.ZonedDateTime; import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter; import java.time.format.DateTimeFormatter;
import java.util.ArrayList; import java.util.*;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ConcurrentMap;
@ -67,7 +64,7 @@ import static org.thingsboard.server.dao.timeseries.SqlTsPartitionDate.EPOCH_STA
public class JpaPsqlTimeseriesDao extends AbstractSimpleSqlTimeseriesDao<TsKvEntity> implements TimeseriesDao { public class JpaPsqlTimeseriesDao extends AbstractSimpleSqlTimeseriesDao<TsKvEntity> implements TimeseriesDao {
private final ConcurrentMap<String, Integer> tsKvDictionaryMap = new ConcurrentHashMap<>(); private final ConcurrentMap<String, Integer> tsKvDictionaryMap = new ConcurrentHashMap<>();
private final Set<PsqlPartition> partitions = ConcurrentHashMap.newKeySet(); private final Map<Long, PsqlPartition> partitions = new ConcurrentHashMap<>();
private static final ReentrantLock tsCreationLock = new ReentrantLock(); private static final ReentrantLock tsCreationLock = new ReentrantLock();
private static final ReentrantLock partitionCreationLock = new ReentrantLock(); private static final ReentrantLock partitionCreationLock = new ReentrantLock();
@ -82,6 +79,7 @@ public class JpaPsqlTimeseriesDao extends AbstractSimpleSqlTimeseriesDao<TsKvEnt
private PsqlPartitioningRepository partitioningRepository; private PsqlPartitioningRepository partitioningRepository;
private SqlTsPartitionDate tsFormat; private SqlTsPartitionDate tsFormat;
private PsqlPartition indefinitePartition;
@Value("${sql.ts_key_value_partitioning}") @Value("${sql.ts_key_value_partitioning}")
private String partitioning; private String partitioning;
@ -92,6 +90,10 @@ public class JpaPsqlTimeseriesDao extends AbstractSimpleSqlTimeseriesDao<TsKvEnt
Optional<SqlTsPartitionDate> partition = SqlTsPartitionDate.parse(partitioning); Optional<SqlTsPartitionDate> partition = SqlTsPartitionDate.parse(partitioning);
if (partition.isPresent()) { if (partition.isPresent()) {
tsFormat = partition.get(); tsFormat = partition.get();
if (tsFormat.equals(SqlTsPartitionDate.INDEFINITE)) {
indefinitePartition = new PsqlPartition(toMills(EPOCH_START), Long.MAX_VALUE, tsFormat.getPattern());
savePartition(indefinitePartition);
}
} else { } else {
log.warn("Incorrect configuration of partitioning {}", partitioning); log.warn("Incorrect configuration of partitioning {}", partitioning);
throw new RuntimeException("Failed to parse partitioning property: " + partitioning + "!"); throw new RuntimeException("Failed to parse partitioning property: " + partitioning + "!");
@ -116,23 +118,22 @@ public class JpaPsqlTimeseriesDao extends AbstractSimpleSqlTimeseriesDao<TsKvEnt
entity.setLongValue(tsKvEntry.getLongValue().orElse(null)); entity.setLongValue(tsKvEntry.getLongValue().orElse(null));
entity.setBooleanValue(tsKvEntry.getBooleanValue().orElse(null)); entity.setBooleanValue(tsKvEntry.getBooleanValue().orElse(null));
PsqlPartition psqlPartition = toPartition(tsKvEntry.getTs()); PsqlPartition psqlPartition = toPartition(tsKvEntry.getTs());
savePartition(psqlPartition);
log.trace("Saving entity: {}", entity); log.trace("Saving entity: {}", entity);
return tsQueue.add(new EntityContainer(entity, psqlPartition.getPartitionDate())); return tsQueue.add(new EntityContainer(entity, psqlPartition.getPartitionDate()));
} }
@Override @Override
public ListenableFuture<Void> remove(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { public ListenableFuture<Void> remove(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) {
return service.submit(() -> { return service.submit(() -> {
String strKey = query.getKey(); String strKey = query.getKey();
Integer keyId = getOrSaveKeyId(strKey); Integer keyId = getOrSaveKeyId(strKey);
tsKvRepository.delete( tsKvRepository.delete(
entityId.getId(), entityId.getId(),
keyId, keyId,
query.getStartTs(), query.getStartTs(),
query.getEndTs()); query.getEndTs());
return null; return null;
}); });
} }
@Override @Override
@ -286,13 +287,13 @@ public class JpaPsqlTimeseriesDao extends AbstractSimpleSqlTimeseriesDao<TsKvEnt
} }
private void savePartition(PsqlPartition psqlPartition) { private void savePartition(PsqlPartition psqlPartition) {
if (!partitions.contains(psqlPartition)) { if (!partitions.containsKey(psqlPartition.getStart())) {
partitionCreationLock.lock(); partitionCreationLock.lock();
try { try {
log.trace("Saving partition: {}", psqlPartition); log.trace("Saving partition: {}", psqlPartition);
partitioningRepository.save(psqlPartition); partitioningRepository.save(psqlPartition);
log.trace("Adding partition to Set: {}", psqlPartition); log.trace("Adding partition to Set: {}", psqlPartition);
partitions.add(psqlPartition); partitions.put(psqlPartition.getStart(), psqlPartition);
} finally { } finally {
partitionCreationLock.unlock(); partitionCreationLock.unlock();
} }
@ -300,17 +301,28 @@ public class JpaPsqlTimeseriesDao extends AbstractSimpleSqlTimeseriesDao<TsKvEnt
} }
private PsqlPartition toPartition(long ts) { private PsqlPartition toPartition(long ts) {
if (tsFormat.getTruncateUnit().equals(SqlTsPartitionDate.INDEFINITE.getTruncateUnit())) { if (tsFormat.equals(SqlTsPartitionDate.INDEFINITE)) {
return new PsqlPartition(toMills(EPOCH_START), Long.MAX_VALUE, tsFormat.getPattern()); return indefinitePartition;
} else { } else {
LocalDateTime time = LocalDateTime.ofInstant(Instant.ofEpochMilli(ts), ZoneOffset.UTC); LocalDateTime time = LocalDateTime.ofInstant(Instant.ofEpochMilli(ts), ZoneOffset.UTC);
LocalDateTime localDateTimeStart = tsFormat.trancateTo(time); LocalDateTime localDateTimeStart = tsFormat.trancateTo(time);
LocalDateTime localDateTimeEnd = tsFormat.plusTo(localDateTimeStart); long partitionStartTs = toMills(localDateTimeStart);
ZonedDateTime zonedDateTime = localDateTimeStart.atZone(ZoneOffset.UTC); PsqlPartition partition = partitions.get(partitionStartTs);
String partitionDate = zonedDateTime.format(DateTimeFormatter.ofPattern(tsFormat.getPattern())); if (partition != null) {
return new PsqlPartition(toMills(localDateTimeStart), toMills(localDateTimeEnd), partitionDate); return partition;
} else {
LocalDateTime localDateTimeEnd = tsFormat.plusTo(localDateTimeStart);
long partitionEndTs = toMills(localDateTimeEnd);
ZonedDateTime zonedDateTime = localDateTimeStart.atZone(ZoneOffset.UTC);
String partitionDate = zonedDateTime.format(DateTimeFormatter.ofPattern(tsFormat.getPattern()));
partition = new PsqlPartition(partitionStartTs, partitionEndTs, partitionDate);
savePartition(partition);
return partition;
}
} }
} }
private long toMills(LocalDateTime time) { return time.toInstant(ZoneOffset.UTC).toEpochMilli(); } private static long toMills(LocalDateTime time) {
return time.toInstant(ZoneOffset.UTC).toEpochMilli();
}
} }