dendi-wang/thingsboard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactored edge event process. Code review fixes

Browse Source
This commit is contained in:
Volodymyr Babak 2020-12-10 18:51:26 +02:00
parent 57adec670b
commit 3a46cbe5d8
6 changed files with 158 additions and 97 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

90
application/src/main/java/org/thingsboard/server/service/edge/DefaultEdgeNotificationService.java
View File

@ -441,27 +441,41 @@ public class DefaultEdgeNotificationService implements EdgeNotificationService {
private void processAlarm(TenantId tenantId, TransportProtos.EdgeNotificationMsgProto edgeNotificationMsg) { private void processAlarm(TenantId tenantId, TransportProtos.EdgeNotificationMsgProto edgeNotificationMsg) {
AlarmId alarmId = new AlarmId(new UUID(edgeNotificationMsg.getEntityIdMSB(), edgeNotificationMsg.getEntityIdLSB())); AlarmId alarmId = new AlarmId(new UUID(edgeNotificationMsg.getEntityIdMSB(), edgeNotificationMsg.getEntityIdLSB()));
ListenableFuture<Alarm> alarmFuture = alarmService.findAlarmByIdAsync(tenantId, alarmId); ListenableFuture<Alarm> alarmFuture = alarmService.findAlarmByIdAsync(tenantId, alarmId);
Futures.transform(alarmFuture, alarm -> { Futures.addCallback(alarmFuture, new FutureCallback<Alarm>() {
if (alarm != null) { @Override
EdgeEventType type = getEdgeQueueTypeByEntityType(alarm.getOriginator().getEntityType()); public void onSuccess(@Nullable Alarm alarm) {
if (type != null) { if (alarm != null) {
ListenableFuture<List<EdgeId>> relatedEdgeIdsByEntityIdFuture = edgeService.findRelatedEdgeIdsByEntityId(tenantId, alarm.getOriginator()); EdgeEventType type = getEdgeQueueTypeByEntityType(alarm.getOriginator().getEntityType());
Futures.transform(relatedEdgeIdsByEntityIdFuture, relatedEdgeIdsByEntityId -> { if (type != null) {
if (relatedEdgeIdsByEntityId != null) { ListenableFuture<List<EdgeId>> relatedEdgeIdsByEntityIdFuture = edgeService.findRelatedEdgeIdsByEntityId(tenantId, alarm.getOriginator());
for (EdgeId edgeId : relatedEdgeIdsByEntityId) { Futures.addCallback(relatedEdgeIdsByEntityIdFuture, new FutureCallback<List<EdgeId>>() {
saveEdgeEvent(tenantId, @Override
edgeId, public void onSuccess(@Nullable List<EdgeId> relatedEdgeIdsByEntityId) {
EdgeEventType.ALARM, if (relatedEdgeIdsByEntityId != null) {
EdgeEventActionType.valueOf(edgeNotificationMsg.getAction()), for (EdgeId edgeId : relatedEdgeIdsByEntityId) {
alarmId, saveEdgeEvent(tenantId,
null); edgeId,
EdgeEventType.ALARM,
EdgeEventActionType.valueOf(edgeNotificationMsg.getAction()),
alarmId,
null);
}
}
} }
}
return null; @Override
}, dbCallbackExecutorService); public void onFailure(Throwable t) {
log.warn("[{}] can't find related edge ids by entity id [{}]", tenantId.getId(), alarm.getOriginator(), t);
}
}, dbCallbackExecutorService);
}
} }
} }
return null;
@Override
public void onFailure(Throwable t) {
log.warn("[{}] can't find alarm by id [{}]", tenantId.getId(), alarmId.getId(), t);
}
}, dbCallbackExecutorService); }, dbCallbackExecutorService);
} }
@ -473,26 +487,34 @@ public class DefaultEdgeNotificationService implements EdgeNotificationService {
futures.add(edgeService.findRelatedEdgeIdsByEntityId(tenantId, relation.getTo())); futures.add(edgeService.findRelatedEdgeIdsByEntityId(tenantId, relation.getTo()));
futures.add(edgeService.findRelatedEdgeIdsByEntityId(tenantId, relation.getFrom())); futures.add(edgeService.findRelatedEdgeIdsByEntityId(tenantId, relation.getFrom()));
ListenableFuture<List<List<EdgeId>>> combinedFuture = Futures.allAsList(futures); ListenableFuture<List<List<EdgeId>>> combinedFuture = Futures.allAsList(futures);
Futures.transform(combinedFuture, listOfListsEdgeIds -> { Futures.addCallback(combinedFuture, new FutureCallback<List<List<EdgeId>>>() {
Set<EdgeId> uniqueEdgeIds = new HashSet<>(); @Override
if (listOfListsEdgeIds != null && !listOfListsEdgeIds.isEmpty()) { public void onSuccess(@Nullable List<List<EdgeId>> listOfListsEdgeIds) {
for (List<EdgeId> listOfListsEdgeId : listOfListsEdgeIds) { Set<EdgeId> uniqueEdgeIds = new HashSet<>();
if (listOfListsEdgeId != null) { if (listOfListsEdgeIds != null && !listOfListsEdgeIds.isEmpty()) {
uniqueEdgeIds.addAll(listOfListsEdgeId); for (List<EdgeId> listOfListsEdgeId : listOfListsEdgeIds) {
if (listOfListsEdgeId != null) {
uniqueEdgeIds.addAll(listOfListsEdgeId);
}
}
}
if (!uniqueEdgeIds.isEmpty()) {
for (EdgeId edgeId : uniqueEdgeIds) {
saveEdgeEvent(tenantId,
edgeId,
EdgeEventType.RELATION,
EdgeEventActionType.valueOf(edgeNotificationMsg.getAction()),
null,
mapper.valueToTree(relation));
} }
} }
} }
if (!uniqueEdgeIds.isEmpty()) {
for (EdgeId edgeId : uniqueEdgeIds) { @Override
saveEdgeEvent(tenantId, public void onFailure(Throwable t) {
edgeId, log.warn("[{}] can't find related edge ids by relation to id [{}] and relation from id [{}]" ,
EdgeEventType.RELATION, tenantId.getId(), relation.getTo().getId(), relation.getFrom().getId(), t);
EdgeEventActionType.valueOf(edgeNotificationMsg.getAction()),
null,
mapper.valueToTree(relation));
}
} }
return null;
}, dbCallbackExecutorService); }, dbCallbackExecutorService);
} }
} }

83
application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcService.java
View File

@ -26,6 +26,7 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.thingsboard.common.util.ThingsBoardThreadFactory;
import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.DataConstants;
import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.edge.Edge;
import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.EdgeId;
@ -48,10 +49,12 @@ import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.util.Collections; import java.util.Collections;
import java.util.Map; import java.util.Map;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors; import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
@Service @Service
@ -62,6 +65,7 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i
private final ConcurrentMap<EdgeId, EdgeGrpcSession> sessions = new ConcurrentHashMap<>(); private final ConcurrentMap<EdgeId, EdgeGrpcSession> sessions = new ConcurrentHashMap<>();
private final ConcurrentMap<EdgeId, Boolean> sessionNewEvents = new ConcurrentHashMap<>(); private final ConcurrentMap<EdgeId, Boolean> sessionNewEvents = new ConcurrentHashMap<>();
private final ConcurrentMap<EdgeId, ScheduledFuture<?>> sessionEdgeEventChecks = new ConcurrentHashMap<>();
private static final ObjectMapper mapper = new ObjectMapper(); private static final ObjectMapper mapper = new ObjectMapper();
@Value("${edges.rpc.port}") @Value("${edges.rpc.port}")
@ -77,6 +81,9 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i
@Value("${edges.rpc.client_max_keep_alive_time_sec}") @Value("${edges.rpc.client_max_keep_alive_time_sec}")
private int clientMaxKeepAliveTimeSec; private int clientMaxKeepAliveTimeSec;
@Value("${edges.scheduler_pool_size}")
private int schedulerPoolSize;
@Autowired @Autowired
private EdgeContextComponent ctx; private EdgeContextComponent ctx;
@ -85,7 +92,7 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i
private Server server; private Server server;
private ExecutorService executor; private ScheduledExecutorService scheduler;
@PostConstruct @PostConstruct
public void init() { public void init() {
@ -112,8 +119,7 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i
throw new RuntimeException("Failed to start Edge RPC server!"); throw new RuntimeException("Failed to start Edge RPC server!");
} }
log.info("Edge RPC service initialized!"); log.info("Edge RPC service initialized!");
executor = Executors.newSingleThreadExecutor(); this.scheduler = Executors.newScheduledThreadPool(schedulerPoolSize, ThingsBoardThreadFactory.forName("edge-scheduler"));
processHandleMessages();
} }
@PreDestroy @PreDestroy
@ -121,8 +127,16 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i
if (server != null) { if (server != null) {
server.shutdownNow(); server.shutdownNow();
} }
if (executor != null) { for (Map.Entry<EdgeId, ScheduledFuture<?>> entry : sessionEdgeEventChecks.entrySet()) {
executor.shutdownNow(); EdgeId edgeId = entry.getKey();
ScheduledFuture<?> sessionEdgeEventCheck = entry.getValue();
if (sessionEdgeEventCheck != null && !sessionEdgeEventCheck.isCancelled() && !sessionEdgeEventCheck.isDone()) {
sessionEdgeEventCheck.cancel(true);
sessionEdgeEventChecks.remove(edgeId);
}
}
if (scheduler != null) {
scheduler.shutdownNow();
} }
} }
@ -150,6 +164,7 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i
session.close(); session.close();
sessions.remove(edgeId); sessions.remove(edgeId);
sessionNewEvents.remove(edgeId); sessionNewEvents.remove(edgeId);
cancelScheduleEdgeEventsCheck(edgeId);
} }
} }
@ -168,6 +183,7 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i
sessionNewEvents.put(edgeId, false); sessionNewEvents.put(edgeId, false);
save(edgeId, DefaultDeviceStateService.ACTIVITY_STATE, true); save(edgeId, DefaultDeviceStateService.ACTIVITY_STATE, true);
save(edgeId, DefaultDeviceStateService.LAST_CONNECT_TIME, System.currentTimeMillis()); save(edgeId, DefaultDeviceStateService.LAST_CONNECT_TIME, System.currentTimeMillis());
scheduleEdgeEventsCheck(edgeGrpcSession);
} }
public EdgeGrpcSession getEdgeGrpcSessionById(EdgeId edgeId) { public EdgeGrpcSession getEdgeGrpcSessionById(EdgeId edgeId) {
@ -179,38 +195,38 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i
} }
} }
private void processHandleMessages() { private void scheduleEdgeEventsCheck(EdgeGrpcSession session) {
executor.submit(() -> { EdgeId edgeId = session.getEdge().getId();
while (!Thread.interrupted()) { UUID tenantId = session.getEdge().getTenantId().getId();
if (sessions.containsKey(edgeId)) {
ScheduledFuture<?> schedule = scheduler.schedule(() -> {
try { try {
if (sessions.size() > 0) { if (sessionNewEvents.get(edgeId)) {
for (Map.Entry<EdgeId, EdgeGrpcSession> entry : sessions.entrySet()) { log.trace("[{}] Set session new events flag to false", edgeId.getId());
EdgeId edgeId = entry.getKey(); sessionNewEvents.put(edgeId, false);
EdgeGrpcSession session = entry.getValue(); session.processEdgeEvents();
if (sessionNewEvents.get(edgeId)) {
log.trace("[{}] set session new events flag to false", edgeId.getId());
sessionNewEvents.put(edgeId, false);
// TODO: voba - at the moment all edge events are processed in a single thread. Maybe this should be updated?
session.processHandleMessages();
}
}
} else {
log.trace("No sessions available");
}
log.trace("Sleep for the next run");
try {
Thread.sleep(ctx.getEdgeEventStorageSettings().getNoRecordsSleepInterval());
} catch (InterruptedException ignore) {
} }
} catch (Exception e) { } catch (Exception e) {
log.warn("Failed to process messages handling!", e); log.warn("[{}] Failed to process edge events for edge [{}]!", tenantId, session.getEdge().getId().getId(), e);
try {
Thread.sleep(1000);
} catch (InterruptedException ignore) {
}
} }
scheduleEdgeEventsCheck(session);
}, ctx.getEdgeEventStorageSettings().getNoRecordsSleepInterval(), TimeUnit.MILLISECONDS);
sessionEdgeEventChecks.put(edgeId, schedule);
log.trace("[{}] Check edge event was scheduler for edge [{}]", tenantId, edgeId.getId());
} else {
log.debug("[{}] Session was removed and edge event check schedule must not be started [{}]",
tenantId, edgeId.getId());
}
}
private void cancelScheduleEdgeEventsCheck(EdgeId edgeId) {
if (sessionEdgeEventChecks.containsKey(edgeId)) {
ScheduledFuture<?> sessionEdgeEventCheck = sessionEdgeEventChecks.get(edgeId);
if (sessionEdgeEventCheck != null && !sessionEdgeEventCheck.isCancelled() && !sessionEdgeEventCheck.isDone()) {
sessionEdgeEventCheck.cancel(true);
sessionEdgeEventChecks.remove(edgeId);
} }
}); }
} }
private void onEdgeDisconnect(EdgeId edgeId) { private void onEdgeDisconnect(EdgeId edgeId) {
@ -219,6 +235,7 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i
sessionNewEvents.remove(edgeId); sessionNewEvents.remove(edgeId);
save(edgeId, DefaultDeviceStateService.ACTIVITY_STATE, false); save(edgeId, DefaultDeviceStateService.ACTIVITY_STATE, false);
save(edgeId, DefaultDeviceStateService.LAST_DISCONNECT_TIME, System.currentTimeMillis()); save(edgeId, DefaultDeviceStateService.LAST_DISCONNECT_TIME, System.currentTimeMillis());
cancelScheduleEdgeEventsCheck(edgeId);
} }
private void save(EdgeId edgeId, String key, long value) { private void save(EdgeId edgeId, String key, long value) {

2
application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java
View File

@ -258,7 +258,7 @@ public final class EdgeGrpcSession implements Closeable {
} }
} }
void processHandleMessages() throws ExecutionException, InterruptedException { void processEdgeEvents() throws ExecutionException, InterruptedException {
log.trace("[{}] processHandleMessages started", this.sessionId); log.trace("[{}] processHandleMessages started", this.sessionId);
if (isConnected()) { if (isConnected()) {
Long queueStartTs = getQueueStartTs().get(); Long queueStartTs = getQueueStartTs().get();

61
application/src/main/java/org/thingsboard/server/service/edge/rpc/init/DefaultSyncEdgeService.java
View File

@ -545,34 +545,47 @@ public class DefaultSyncEdgeService implements SyncEdgeService {
futures.add(findRelationByQuery(edge, entityId, EntitySearchDirection.FROM)); futures.add(findRelationByQuery(edge, entityId, EntitySearchDirection.FROM));
futures.add(findRelationByQuery(edge, entityId, EntitySearchDirection.TO)); futures.add(findRelationByQuery(edge, entityId, EntitySearchDirection.TO));
ListenableFuture<List<List<EntityRelation>>> relationsListFuture = Futures.allAsList(futures); ListenableFuture<List<List<EntityRelation>>> relationsListFuture = Futures.allAsList(futures);
return Futures.transform(relationsListFuture, relationsList -> { SettableFuture<Void> futureToSet = SettableFuture.create();
try { Futures.addCallback(relationsListFuture, new FutureCallback<List<List<EntityRelation>>>() {
if (relationsList != null && !relationsList.isEmpty()) { @Override
for (List<EntityRelation> entityRelations : relationsList) { public void onSuccess(@Nullable List<List<EntityRelation>> relationsList) {
log.trace("[{}] [{}] [{}] relation(s) are going to be pushed to edge.", edge.getId(), entityId, entityRelations.size()); try {
for (EntityRelation relation : entityRelations) { if (relationsList != null && !relationsList.isEmpty()) {
try { for (List<EntityRelation> entityRelations : relationsList) {
if (!relation.getFrom().getEntityType().equals(EntityType.EDGE) && log.trace("[{}] [{}] [{}] relation(s) are going to be pushed to edge.", edge.getId(), entityId, entityRelations.size());
!relation.getTo().getEntityType().equals(EntityType.EDGE)) { for (EntityRelation relation : entityRelations) {
saveEdgeEvent(edge.getTenantId(), try {
edge.getId(), if (!relation.getFrom().getEntityType().equals(EntityType.EDGE) &&
EdgeEventType.RELATION, !relation.getTo().getEntityType().equals(EntityType.EDGE)) {
EdgeEventActionType.ADDED, saveEdgeEvent(edge.getTenantId(),
null, edge.getId(),
mapper.valueToTree(relation)); EdgeEventType.RELATION,
EdgeEventActionType.ADDED,
null,
mapper.valueToTree(relation));
}
} catch (Exception e) {
log.error("Exception during loading relation [{}] to edge on sync!", relation, e);
futureToSet.setException(e);
return;
}
}
} }
} catch (Exception e) {
log.error("Exception during loading relation [{}] to edge on sync!", relation, e);
} }
futureToSet.set(null);
} catch (Exception e) {
log.error("Exception during loading relation(s) to edge on sync!", e);
futureToSet.setException(e);
} }
} }
}
} catch (Exception e) { @Override
log.error("Exception during loading relation(s) to edge on sync!", e); public void onFailure(Throwable t) {
throw new RuntimeException("Exception during loading relation(s) to edge on sync!", e); log.error("[{}] Can't find relation by query. Entity id [{}]", edge.getTenantId(), entityId, t);
} futureToSet.setException(t);
return null; }
}, dbCallbackExecutorService); }, dbCallbackExecutorService);
return futureToSet;
} }
private ListenableFuture<List<EntityRelation>> findRelationByQuery(Edge edge, EntityId entityId, EntitySearchDirection direction) { private ListenableFuture<List<EntityRelation>> findRelationByQuery(Edge edge, EntityId entityId, EntitySearchDirection direction) {

1
application/src/main/resources/thingsboard.yml
View File

@ -602,6 +602,7 @@ edges:
max_read_records_count: "${EDGES_RPC_STORAGE_MAX_READ_RECORDS_COUNT:50}" max_read_records_count: "${EDGES_RPC_STORAGE_MAX_READ_RECORDS_COUNT:50}"
no_read_records_sleep: "${EDGES_RPC_NO_READ_RECORDS_SLEEP:1000}" no_read_records_sleep: "${EDGES_RPC_NO_READ_RECORDS_SLEEP:1000}"
sleep_between_batches: "${EDGES_RPC_SLEEP_BETWEEN_BATCHES:1000}" sleep_between_batches: "${EDGES_RPC_SLEEP_BETWEEN_BATCHES:1000}"
scheduler_pool_size: "${EDGES_SCHEDULER_POOL_SIZE:4}"
edge_events_ttl: "${EDGES_EDGE_EVENTS_TTL:0}" edge_events_ttl: "${EDGES_EDGE_EVENTS_TTL:0}"
state: state:
persistToTelemetry: "${EDGES_PERSIST_STATE_TO_TELEMETRY:false}" persistToTelemetry: "${EDGES_PERSIST_STATE_TO_TELEMETRY:false}"

18
dao/src/main/java/org/thingsboard/server/dao/edge/EdgeServiceImpl.java
View File

@ -16,6 +16,7 @@
package org.thingsboard.server.dao.edge; package org.thingsboard.server.dao.edge;
import com.google.common.base.Function; import com.google.common.base.Function;
import com.google.common.util.concurrent.FutureCallback;
import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.MoreExecutors; import com.google.common.util.concurrent.MoreExecutors;
@ -324,13 +325,20 @@ public class EdgeServiceImpl extends AbstractEntityService implements EdgeServic
public void assignDefaultRuleChainsToEdge(TenantId tenantId, EdgeId edgeId) { public void assignDefaultRuleChainsToEdge(TenantId tenantId, EdgeId edgeId) {
log.trace("Executing assignDefaultRuleChainsToEdge, tenantId [{}], edgeId [{}]", tenantId, edgeId); log.trace("Executing assignDefaultRuleChainsToEdge, tenantId [{}], edgeId [{}]", tenantId, edgeId);
ListenableFuture<List<RuleChain>> future = ruleChainService.findDefaultEdgeRuleChainsByTenantId(tenantId); ListenableFuture<List<RuleChain>> future = ruleChainService.findDefaultEdgeRuleChainsByTenantId(tenantId);
Futures.transform(future, ruleChains -> { Futures.addCallback(future, new FutureCallback<List<RuleChain>>() {
if (ruleChains != null && !ruleChains.isEmpty()) { @Override
for (RuleChain ruleChain : ruleChains) { public void onSuccess(List<RuleChain> ruleChains) {
ruleChainService.assignRuleChainToEdge(tenantId, ruleChain.getId(), edgeId); if (ruleChains != null && !ruleChains.isEmpty()) {
for (RuleChain ruleChain : ruleChains) {
ruleChainService.assignRuleChainToEdge(tenantId, ruleChain.getId(), edgeId);
}
} }
} }
return null;
@Override
public void onFailure(Throwable t) {
log.warn("[{}] can't find default edge rule chains [{}]", tenantId.getId(), edgeId.getId(), t);
}
}, MoreExecutors.directExecutor()); }, MoreExecutors.directExecutor());
} }