dendi-wang/thingsboard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

InMemoryStorage performance improved. Many test cases added since it is essential piece of code.

Browse Source
This commit is contained in:
Sergey Matvienko 2022-04-15 16:40:32 +03:00
parent b9b4d06376
commit f8a6751182
2 changed files with 70 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
common/queue/src/main/java/org/thingsboard/server/queue/memory/DefaultInMemoryStorage.java
View File

@ -67,26 +67,22 @@ public final class DefaultInMemoryStorage implements InMemoryStorage {
return storage.computeIfAbsent(topic, (t) -> new LinkedBlockingQueue<>()).add(msg); return storage.computeIfAbsent(topic, (t) -> new LinkedBlockingQueue<>()).add(msg);
} }
@SuppressWarnings("unchecked")
@Override @Override
public <T extends TbQueueMsg> List<T> get(String topic) throws InterruptedException { public <T extends TbQueueMsg> List<T> get(String topic) throws InterruptedException {
if (storage.containsKey(topic)) { final BlockingQueue<TbQueueMsg> queue = storage.get(topic);
List<T> entities; if (queue != null) {
@SuppressWarnings("unchecked") final TbQueueMsg firstMsg = queue.poll();
T first = (T) storage.get(topic).poll(); if (firstMsg != null) {
if (first != null) { final int queueSize = queue.size();
entities = new ArrayList<>(); if (queueSize > 0) {
entities.add(first); final List<TbQueueMsg> entities = new ArrayList<>(Math.min(queueSize, 999) + 1);
List<TbQueueMsg> otherList = new ArrayList<>(); entities.add(firstMsg);
storage.get(topic).drainTo(otherList, 999); queue.drainTo(entities, 999);
for (TbQueueMsg other : otherList) { return (List<T>) entities;
@SuppressWarnings("unchecked")
T entity = (T) other;
entities.add(entity);
} }
} else { return Collections.singletonList((T) firstMsg);
entities = Collections.emptyList();
} }
return entities;
} }
return Collections.emptyList(); return Collections.emptyList();
} }

66
common/queue/src/test/java/org/thingsboard/server/queue/memory/DefaultInMemoryStorageTest.java
View File

@ -30,6 +30,9 @@ import static org.mockito.Mockito.mock;
@Slf4j @Slf4j
public class DefaultInMemoryStorageTest { public class DefaultInMemoryStorageTest {
static final int MAX_POLL_SIZE = 1000;
final Gson gson = new Gson();
final String topic = "tb_core_notification.tb-node-0";
InMemoryStorage storage = new DefaultInMemoryStorage(); InMemoryStorage storage = new DefaultInMemoryStorage();
@ -47,19 +50,66 @@ public class DefaultInMemoryStorageTest {
} }
@Test @Test
public void givenQueue_whenPoll_thenReturnList() throws InterruptedException { public void givenQueueWithMoreThenBatchSize_whenPoll_thenReturnFullListAndSecondList() throws InterruptedException {
Gson gson = new Gson(); List<TbQueueMsg> msgs = new ArrayList<>(MAX_POLL_SIZE + 1);
String topic = "tb_core_notification.tb-node-0"; for (int i = 0; i < MAX_POLL_SIZE + 1; i++) {
List<TbQueueMsg> msgs = new ArrayList<>(1001);
for (int i = 0; i < 1001; i++) {
DefaultTbQueueMsg msg = gson.fromJson("{\"key\": \"" + UUID.randomUUID() + "\"}", DefaultTbQueueMsg.class); DefaultTbQueueMsg msg = gson.fromJson("{\"key\": \"" + UUID.randomUUID() + "\"}", DefaultTbQueueMsg.class);
msgs.add(msg); msgs.add(msg);
storage.put(topic, msg); storage.put(topic, msg);
} }
assertThat(storage.getLagTotal()).as("total lag is 1001").isEqualTo(1001); assertThat(storage.getLagTotal()).as("total lag is 1001").isEqualTo(MAX_POLL_SIZE + 1);
assertThat(storage.get(topic)).as("poll exactly 1000 msgs").isEqualTo(msgs.subList(0, 1000)); assertThat(storage.get(topic)).as("poll exactly 1000 msgs").isEqualTo(msgs.subList(0, MAX_POLL_SIZE));
assertThat(storage.get(topic)).as("poll last 1 message").isEqualTo(msgs.subList(1000, 1001)); assertThat(storage.get(topic)).as("poll last 1 message").isEqualTo(msgs.subList(MAX_POLL_SIZE, MAX_POLL_SIZE + 1));
assertThat(storage.getLagTotal()).as("total lag is zero").isEqualTo(0); assertThat(storage.getLagTotal()).as("total lag is zero").isEqualTo(0);
} }
private void testPollOnce(final int msgCount) throws InterruptedException {
List<TbQueueMsg> msgs = new ArrayList<>(msgCount);
for (int i = 0; i < msgCount; i++) {
DefaultTbQueueMsg msg = gson.fromJson("{\"key\": \"" + UUID.randomUUID() + "\"}", DefaultTbQueueMsg.class);
msgs.add(msg);
storage.put(topic, msg);
}
assertThat(storage.getLagTotal()).as("total lag before poll").isEqualTo(msgCount);
assertThat(storage.get(topic)).as("polled exactly msgs").isEqualTo(msgs.subList(0, msgCount));
assertThat(storage.getLagTotal()).as("final lag is zero").isEqualTo(0);
}
@Test
public void givenQueueWithExactBatchSize_whenPoll_thenReturnExactBatchSizeList() throws InterruptedException {
testPollOnce(MAX_POLL_SIZE);
}
@Test
public void givenQueueWithExactBatchSizeMinusOne_whenPoll_thenReturnCorrectSizeList() throws InterruptedException {
testPollOnce(MAX_POLL_SIZE - 1);
}
@Test
public void givenQueueWithExactBatchSizeMinusTen_whenPoll_thenReturnCorrectSizeList() throws InterruptedException {
testPollOnce(MAX_POLL_SIZE - 10);
}
@Test
public void givenQueueEmpty_whenPoll_thenReturnEmptyList() throws InterruptedException {
testPollOnce(0);
}
@Test
public void givenQueueWithSingleMessage_whenPoll_thenReturnSingletonList() throws InterruptedException {
testPollOnce(1);
}
@Test
public void givenQueueWithTwoMessages_whenPoll_thenReturnCorrectSizeList() throws InterruptedException {
testPollOnce(2);
}
@Test
public void givenQueueWithTenMessages_whenPoll_thenReturnCorrectSizeList() throws InterruptedException {
testPollOnce(10);
}
} }