Save time series strategies: initial BE implementation
This commit is contained in:
		
						commit
						6ca45f1962
					
				@ -348,7 +348,8 @@ public class DefaultTbEntityViewService extends AbstractTbEntityService implemen
 | 
				
			|||||||
                        .tenantId(entityView.getTenantId())
 | 
					                        .tenantId(entityView.getTenantId())
 | 
				
			||||||
                        .entityId(entityId)
 | 
					                        .entityId(entityId)
 | 
				
			||||||
                        .entries(latestValues)
 | 
					                        .entries(latestValues)
 | 
				
			||||||
                        .onlyLatest(true)
 | 
					                        .saveTimeseries(false)
 | 
				
			||||||
 | 
					                        .saveLatest(true)
 | 
				
			||||||
                        .callback(new FutureCallback<Void>() {
 | 
					                        .callback(new FutureCallback<Void>() {
 | 
				
			||||||
                            @Override
 | 
					                            @Override
 | 
				
			||||||
                            public void onSuccess(@Nullable Void tmp) {
 | 
					                            public void onSuccess(@Nullable Void tmp) {
 | 
				
			||||||
 | 
				
			|||||||
@ -118,10 +118,10 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer
 | 
				
			|||||||
        EntityId entityId = request.getEntityId();
 | 
					        EntityId entityId = request.getEntityId();
 | 
				
			||||||
        checkInternalEntity(entityId);
 | 
					        checkInternalEntity(entityId);
 | 
				
			||||||
        boolean sysTenant = TenantId.SYS_TENANT_ID.equals(tenantId) || tenantId == null;
 | 
					        boolean sysTenant = TenantId.SYS_TENANT_ID.equals(tenantId) || tenantId == null;
 | 
				
			||||||
        if (sysTenant || request.isOnlyLatest() || apiUsageStateService.getApiUsageState(tenantId).isDbStorageEnabled()) {
 | 
					        if (sysTenant || !request.isSaveTimeseries() || apiUsageStateService.getApiUsageState(tenantId).isDbStorageEnabled()) {
 | 
				
			||||||
            KvUtils.validate(request.getEntries(), valueNoXssValidation);
 | 
					            KvUtils.validate(request.getEntries(), valueNoXssValidation);
 | 
				
			||||||
            ListenableFuture<Integer> future = saveTimeseriesInternal(request);
 | 
					            ListenableFuture<Integer> future = saveTimeseriesInternal(request);
 | 
				
			||||||
            if (!request.isOnlyLatest()) {
 | 
					            if (request.isSaveTimeseries()) {
 | 
				
			||||||
                FutureCallback<Integer> callback = getApiUsageCallback(tenantId, request.getCustomerId(), sysTenant, request.getCallback());
 | 
					                FutureCallback<Integer> callback = getApiUsageCallback(tenantId, request.getCustomerId(), sysTenant, request.getCallback());
 | 
				
			||||||
                Futures.addCallback(future, callback, tsCallBackExecutor);
 | 
					                Futures.addCallback(future, callback, tsCallBackExecutor);
 | 
				
			||||||
            }
 | 
					            }
 | 
				
			||||||
@ -135,17 +135,21 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer
 | 
				
			|||||||
        TenantId tenantId = request.getTenantId();
 | 
					        TenantId tenantId = request.getTenantId();
 | 
				
			||||||
        EntityId entityId = request.getEntityId();
 | 
					        EntityId entityId = request.getEntityId();
 | 
				
			||||||
        ListenableFuture<Integer> saveFuture;
 | 
					        ListenableFuture<Integer> saveFuture;
 | 
				
			||||||
        if (request.isOnlyLatest()) {
 | 
					        if (request.isSaveTimeseries() && request.isSaveLatest()) {
 | 
				
			||||||
            saveFuture = Futures.transform(tsService.saveLatest(tenantId, entityId, request.getEntries()), result -> 0, MoreExecutors.directExecutor());
 | 
					 | 
				
			||||||
        } else if (request.isSaveLatest()) {
 | 
					 | 
				
			||||||
            saveFuture = tsService.save(tenantId, entityId, request.getEntries(), request.getTtl());
 | 
					            saveFuture = tsService.save(tenantId, entityId, request.getEntries(), request.getTtl());
 | 
				
			||||||
        } else {
 | 
					        } else if (request.isSaveLatest()) {
 | 
				
			||||||
 | 
					            saveFuture = Futures.transform(tsService.saveLatest(tenantId, entityId, request.getEntries()), result -> 0, MoreExecutors.directExecutor());
 | 
				
			||||||
 | 
					        } else if (request.isSaveTimeseries()) {
 | 
				
			||||||
            saveFuture = tsService.saveWithoutLatest(tenantId, entityId, request.getEntries(), request.getTtl());
 | 
					            saveFuture = tsService.saveWithoutLatest(tenantId, entityId, request.getEntries(), request.getTtl());
 | 
				
			||||||
 | 
					        } else {
 | 
				
			||||||
 | 
					            saveFuture = Futures.immediateFuture(0);
 | 
				
			||||||
        }
 | 
					        }
 | 
				
			||||||
 | 
					
 | 
				
			||||||
        addMainCallback(saveFuture, request.getCallback());
 | 
					        addMainCallback(saveFuture, request.getCallback());
 | 
				
			||||||
 | 
					        if (request.isSendWsUpdate()) {
 | 
				
			||||||
            addWsCallback(saveFuture, success -> onTimeSeriesUpdate(tenantId, entityId, request.getEntries()));
 | 
					            addWsCallback(saveFuture, success -> onTimeSeriesUpdate(tenantId, entityId, request.getEntries()));
 | 
				
			||||||
        if (request.isSaveLatest() && !request.isOnlyLatest()) {
 | 
					        }
 | 
				
			||||||
 | 
					        if (request.isSaveTimeseries() && request.isSaveLatest()) {
 | 
				
			||||||
            addEntityViewCallback(tenantId, entityId, request.getEntries());
 | 
					            addEntityViewCallback(tenantId, entityId, request.getEntries());
 | 
				
			||||||
        }
 | 
					        }
 | 
				
			||||||
        return saveFuture;
 | 
					        return saveFuture;
 | 
				
			||||||
@ -232,7 +236,8 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer
 | 
				
			|||||||
                                                .tenantId(tenantId)
 | 
					                                                .tenantId(tenantId)
 | 
				
			||||||
                                                .entityId(entityView.getId())
 | 
					                                                .entityId(entityView.getId())
 | 
				
			||||||
                                                .entries(entityViewLatest)
 | 
					                                                .entries(entityViewLatest)
 | 
				
			||||||
                                                .onlyLatest(true)
 | 
					                                                .saveTimeseries(false)
 | 
				
			||||||
 | 
					                                                .saveLatest(true)
 | 
				
			||||||
                                                .callback(new FutureCallback<>() {
 | 
					                                                .callback(new FutureCallback<>() {
 | 
				
			||||||
                                                    @Override
 | 
					                                                    @Override
 | 
				
			||||||
                                                    public void onSuccess(@Nullable Void tmp) {}
 | 
					                                                    public void onSuccess(@Nullable Void tmp) {}
 | 
				
			||||||
 | 
				
			|||||||
@ -38,8 +38,9 @@ public class TimeseriesSaveRequest {
 | 
				
			|||||||
    private final EntityId entityId;
 | 
					    private final EntityId entityId;
 | 
				
			||||||
    private final List<TsKvEntry> entries;
 | 
					    private final List<TsKvEntry> entries;
 | 
				
			||||||
    private final long ttl;
 | 
					    private final long ttl;
 | 
				
			||||||
 | 
					    private final boolean saveTimeseries;
 | 
				
			||||||
    private final boolean saveLatest;
 | 
					    private final boolean saveLatest;
 | 
				
			||||||
    private final boolean onlyLatest;
 | 
					    private final boolean sendWsUpdate;
 | 
				
			||||||
    private final FutureCallback<Void> callback;
 | 
					    private final FutureCallback<Void> callback;
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    public static Builder builder() {
 | 
					    public static Builder builder() {
 | 
				
			||||||
@ -53,9 +54,10 @@ public class TimeseriesSaveRequest {
 | 
				
			|||||||
        private EntityId entityId;
 | 
					        private EntityId entityId;
 | 
				
			||||||
        private List<TsKvEntry> entries;
 | 
					        private List<TsKvEntry> entries;
 | 
				
			||||||
        private long ttl;
 | 
					        private long ttl;
 | 
				
			||||||
        private FutureCallback<Void> callback;
 | 
					        private boolean saveTimeseries = true;
 | 
				
			||||||
        private boolean saveLatest = true;
 | 
					        private boolean saveLatest = true;
 | 
				
			||||||
        private boolean onlyLatest;
 | 
					        private boolean sendWsUpdate = true;
 | 
				
			||||||
 | 
					        private FutureCallback<Void> callback;
 | 
				
			||||||
 | 
					
 | 
				
			||||||
        Builder() {}
 | 
					        Builder() {}
 | 
				
			||||||
 | 
					
 | 
				
			||||||
@ -92,14 +94,18 @@ public class TimeseriesSaveRequest {
 | 
				
			|||||||
            return this;
 | 
					            return this;
 | 
				
			||||||
        }
 | 
					        }
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					        public Builder saveTimeseries(boolean saveTimeseries) {
 | 
				
			||||||
 | 
					            this.saveTimeseries = saveTimeseries;
 | 
				
			||||||
 | 
					            return this;
 | 
				
			||||||
 | 
					        }
 | 
				
			||||||
 | 
					
 | 
				
			||||||
        public Builder saveLatest(boolean saveLatest) {
 | 
					        public Builder saveLatest(boolean saveLatest) {
 | 
				
			||||||
            this.saveLatest = saveLatest;
 | 
					            this.saveLatest = saveLatest;
 | 
				
			||||||
            return this;
 | 
					            return this;
 | 
				
			||||||
        }
 | 
					        }
 | 
				
			||||||
 | 
					
 | 
				
			||||||
        public Builder onlyLatest(boolean onlyLatest) {
 | 
					        public Builder sendWsUpdate(boolean sendWsUpdate) {
 | 
				
			||||||
            this.onlyLatest = onlyLatest;
 | 
					            this.sendWsUpdate = sendWsUpdate;
 | 
				
			||||||
            this.saveLatest = true;
 | 
					 | 
				
			||||||
            return this;
 | 
					            return this;
 | 
				
			||||||
        }
 | 
					        }
 | 
				
			||||||
 | 
					
 | 
				
			||||||
@ -123,7 +129,7 @@ public class TimeseriesSaveRequest {
 | 
				
			|||||||
        }
 | 
					        }
 | 
				
			||||||
 | 
					
 | 
				
			||||||
        public TimeseriesSaveRequest build() {
 | 
					        public TimeseriesSaveRequest build() {
 | 
				
			||||||
            return new TimeseriesSaveRequest(tenantId, customerId, entityId, entries, ttl, saveLatest, onlyLatest, callback);
 | 
					            return new TimeseriesSaveRequest(tenantId, customerId, entityId, entries, ttl, saveTimeseries, saveLatest, sendWsUpdate, callback);
 | 
				
			||||||
        }
 | 
					        }
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    }
 | 
					    }
 | 
				
			||||||
 | 
				
			|||||||
@ -37,8 +37,14 @@ import org.thingsboard.server.common.msg.TbMsg;
 | 
				
			|||||||
import java.util.ArrayList;
 | 
					import java.util.ArrayList;
 | 
				
			||||||
import java.util.List;
 | 
					import java.util.List;
 | 
				
			||||||
import java.util.Map;
 | 
					import java.util.Map;
 | 
				
			||||||
 | 
					import java.util.UUID;
 | 
				
			||||||
import java.util.concurrent.TimeUnit;
 | 
					import java.util.concurrent.TimeUnit;
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					import static org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNodeConfiguration.PersistenceSettings;
 | 
				
			||||||
 | 
					import static org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNodeConfiguration.PersistenceSettings.Advanced;
 | 
				
			||||||
 | 
					import static org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNodeConfiguration.PersistenceSettings.Deduplicate;
 | 
				
			||||||
 | 
					import static org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNodeConfiguration.PersistenceSettings.OnEveryMessage;
 | 
				
			||||||
 | 
					import static org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNodeConfiguration.PersistenceSettings.WebSocketsOnly;
 | 
				
			||||||
import static org.thingsboard.server.common.data.msg.TbMsgType.POST_TELEMETRY_REQUEST;
 | 
					import static org.thingsboard.server.common.data.msg.TbMsgType.POST_TELEMETRY_REQUEST;
 | 
				
			||||||
 | 
					
 | 
				
			||||||
@Slf4j
 | 
					@Slf4j
 | 
				
			||||||
@ -68,12 +74,15 @@ public class TbMsgTimeseriesNode implements TbNode {
 | 
				
			|||||||
    private TbContext ctx;
 | 
					    private TbContext ctx;
 | 
				
			||||||
    private long tenantProfileDefaultStorageTtl;
 | 
					    private long tenantProfileDefaultStorageTtl;
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					    private PersistenceSettings persistenceSettings;
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    @Override
 | 
					    @Override
 | 
				
			||||||
    public void init(TbContext ctx, TbNodeConfiguration configuration) throws TbNodeException {
 | 
					    public void init(TbContext ctx, TbNodeConfiguration configuration) throws TbNodeException {
 | 
				
			||||||
        this.config = TbNodeUtils.convert(configuration, TbMsgTimeseriesNodeConfiguration.class);
 | 
					        this.config = TbNodeUtils.convert(configuration, TbMsgTimeseriesNodeConfiguration.class);
 | 
				
			||||||
        this.ctx = ctx;
 | 
					        this.ctx = ctx;
 | 
				
			||||||
        ctx.addTenantProfileListener(this::onTenantProfileUpdate);
 | 
					        ctx.addTenantProfileListener(this::onTenantProfileUpdate);
 | 
				
			||||||
        onTenantProfileUpdate(ctx.getTenantProfile());
 | 
					        onTenantProfileUpdate(ctx.getTenantProfile());
 | 
				
			||||||
 | 
					        persistenceSettings = config.getPersistenceSettings();
 | 
				
			||||||
    }
 | 
					    }
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    void onTenantProfileUpdate(TenantProfile tenantProfile) {
 | 
					    void onTenantProfileUpdate(TenantProfile tenantProfile) {
 | 
				
			||||||
@ -88,6 +97,18 @@ public class TbMsgTimeseriesNode implements TbNode {
 | 
				
			|||||||
            return;
 | 
					            return;
 | 
				
			||||||
        }
 | 
					        }
 | 
				
			||||||
        long ts = computeTs(msg, config.isUseServerTs());
 | 
					        long ts = computeTs(msg, config.isUseServerTs());
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					        PersistenceDecision persistenceDecision = makePersistenceDecision(ts, msg.getOriginator().getId());
 | 
				
			||||||
 | 
					        boolean saveTimeseries = persistenceDecision.saveTimeseries();
 | 
				
			||||||
 | 
					        boolean saveLatest = persistenceDecision.saveLatest();
 | 
				
			||||||
 | 
					        boolean sendWsUpdate = persistenceDecision.sendWsUpdate();
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					        // short-circuit
 | 
				
			||||||
 | 
					        if (!saveTimeseries && !saveLatest && !sendWsUpdate) {
 | 
				
			||||||
 | 
					            ctx.tellSuccess(msg);
 | 
				
			||||||
 | 
					            return;
 | 
				
			||||||
 | 
					        }
 | 
				
			||||||
 | 
					
 | 
				
			||||||
        String src = msg.getData();
 | 
					        String src = msg.getData();
 | 
				
			||||||
        Map<Long, List<KvEntry>> tsKvMap = JsonConverter.convertToTelemetry(JsonParser.parseString(src), ts);
 | 
					        Map<Long, List<KvEntry>> tsKvMap = JsonConverter.convertToTelemetry(JsonParser.parseString(src), ts);
 | 
				
			||||||
        if (tsKvMap.isEmpty()) {
 | 
					        if (tsKvMap.isEmpty()) {
 | 
				
			||||||
@ -111,7 +132,9 @@ public class TbMsgTimeseriesNode implements TbNode {
 | 
				
			|||||||
                .entityId(msg.getOriginator())
 | 
					                .entityId(msg.getOriginator())
 | 
				
			||||||
                .entries(tsKvEntryList)
 | 
					                .entries(tsKvEntryList)
 | 
				
			||||||
                .ttl(ttl)
 | 
					                .ttl(ttl)
 | 
				
			||||||
                .saveLatest(!config.isSkipLatestPersistence())
 | 
					                .saveTimeseries(saveTimeseries)
 | 
				
			||||||
 | 
					                .saveLatest(saveLatest)
 | 
				
			||||||
 | 
					                .sendWsUpdate(sendWsUpdate)
 | 
				
			||||||
                .callback(new TelemetryNodeCallback(ctx, msg))
 | 
					                .callback(new TelemetryNodeCallback(ctx, msg))
 | 
				
			||||||
                .build());
 | 
					                .build());
 | 
				
			||||||
    }
 | 
					    }
 | 
				
			||||||
@ -120,6 +143,37 @@ public class TbMsgTimeseriesNode implements TbNode {
 | 
				
			|||||||
        return ignoreMetadataTs ? System.currentTimeMillis() : msg.getMetaDataTs();
 | 
					        return ignoreMetadataTs ? System.currentTimeMillis() : msg.getMetaDataTs();
 | 
				
			||||||
    }
 | 
					    }
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					    private record PersistenceDecision(boolean saveTimeseries, boolean saveLatest, boolean sendWsUpdate) {}
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					    private PersistenceDecision makePersistenceDecision(long ts, UUID originatorUuid) {
 | 
				
			||||||
 | 
					        boolean saveTimeseries;
 | 
				
			||||||
 | 
					        boolean saveLatest;
 | 
				
			||||||
 | 
					        boolean sendWsUpdate;
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					        if (persistenceSettings instanceof OnEveryMessage) {
 | 
				
			||||||
 | 
					            saveTimeseries = true;
 | 
				
			||||||
 | 
					            saveLatest = true;
 | 
				
			||||||
 | 
					            sendWsUpdate = true;
 | 
				
			||||||
 | 
					        } else if (persistenceSettings instanceof WebSocketsOnly) {
 | 
				
			||||||
 | 
					            saveTimeseries = false;
 | 
				
			||||||
 | 
					            saveLatest = false;
 | 
				
			||||||
 | 
					            sendWsUpdate = true;
 | 
				
			||||||
 | 
					        } else if (persistenceSettings instanceof Deduplicate deduplicate) {
 | 
				
			||||||
 | 
					            boolean isFirstMsgInInterval = deduplicate.getDeduplicateStrategy().shouldPersist(ts, originatorUuid);
 | 
				
			||||||
 | 
					            saveTimeseries = isFirstMsgInInterval;
 | 
				
			||||||
 | 
					            saveLatest = isFirstMsgInInterval;
 | 
				
			||||||
 | 
					            sendWsUpdate = isFirstMsgInInterval;
 | 
				
			||||||
 | 
					        } else if (persistenceSettings instanceof Advanced advanced) {
 | 
				
			||||||
 | 
					            saveTimeseries = advanced.timeseries().shouldPersist(ts, originatorUuid);
 | 
				
			||||||
 | 
					            saveLatest = advanced.latest().shouldPersist(ts, originatorUuid);
 | 
				
			||||||
 | 
					            sendWsUpdate = advanced.webSockets().shouldPersist(ts, originatorUuid);
 | 
				
			||||||
 | 
					        } else { // should not happen
 | 
				
			||||||
 | 
					            throw new IllegalArgumentException("Unknown persistence settings type: " + persistenceSettings.getClass().getSimpleName());
 | 
				
			||||||
 | 
					        }
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					        return new PersistenceDecision(saveTimeseries, saveLatest, sendWsUpdate);
 | 
				
			||||||
 | 
					    }
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    @Override
 | 
					    @Override
 | 
				
			||||||
    public void destroy() {
 | 
					    public void destroy() {
 | 
				
			||||||
        ctx.removeListeners();
 | 
					        ctx.removeListeners();
 | 
				
			||||||
 | 
				
			|||||||
@ -15,22 +15,77 @@
 | 
				
			|||||||
 */
 | 
					 */
 | 
				
			||||||
package org.thingsboard.rule.engine.telemetry;
 | 
					package org.thingsboard.rule.engine.telemetry;
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					import com.fasterxml.jackson.annotation.JsonCreator;
 | 
				
			||||||
 | 
					import com.fasterxml.jackson.annotation.JsonProperty;
 | 
				
			||||||
 | 
					import com.fasterxml.jackson.annotation.JsonSubTypes;
 | 
				
			||||||
 | 
					import com.fasterxml.jackson.annotation.JsonTypeInfo;
 | 
				
			||||||
import lombok.Data;
 | 
					import lombok.Data;
 | 
				
			||||||
 | 
					import lombok.Getter;
 | 
				
			||||||
import org.thingsboard.rule.engine.api.NodeConfiguration;
 | 
					import org.thingsboard.rule.engine.api.NodeConfiguration;
 | 
				
			||||||
 | 
					import org.thingsboard.rule.engine.telemetry.strategy.PersistenceStrategy;
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					import java.util.Objects;
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					import static org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNodeConfiguration.PersistenceSettings.Advanced;
 | 
				
			||||||
 | 
					import static org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNodeConfiguration.PersistenceSettings.Deduplicate;
 | 
				
			||||||
 | 
					import static org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNodeConfiguration.PersistenceSettings.OnEveryMessage;
 | 
				
			||||||
 | 
					import static org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNodeConfiguration.PersistenceSettings.WebSocketsOnly;
 | 
				
			||||||
 | 
					
 | 
				
			||||||
@Data
 | 
					@Data
 | 
				
			||||||
public class TbMsgTimeseriesNodeConfiguration implements NodeConfiguration<TbMsgTimeseriesNodeConfiguration> {
 | 
					public class TbMsgTimeseriesNodeConfiguration implements NodeConfiguration<TbMsgTimeseriesNodeConfiguration> {
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    private long defaultTTL;
 | 
					    private long defaultTTL;
 | 
				
			||||||
    private boolean skipLatestPersistence;
 | 
					 | 
				
			||||||
    private boolean useServerTs;
 | 
					    private boolean useServerTs;
 | 
				
			||||||
 | 
					    private PersistenceSettings persistenceSettings;
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    @Override
 | 
					    @Override
 | 
				
			||||||
    public TbMsgTimeseriesNodeConfiguration defaultConfiguration() {
 | 
					    public TbMsgTimeseriesNodeConfiguration defaultConfiguration() {
 | 
				
			||||||
        TbMsgTimeseriesNodeConfiguration configuration = new TbMsgTimeseriesNodeConfiguration();
 | 
					        TbMsgTimeseriesNodeConfiguration configuration = new TbMsgTimeseriesNodeConfiguration();
 | 
				
			||||||
        configuration.setDefaultTTL(0L);
 | 
					        configuration.setDefaultTTL(0L);
 | 
				
			||||||
        configuration.setSkipLatestPersistence(false);
 | 
					 | 
				
			||||||
        configuration.setUseServerTs(false);
 | 
					        configuration.setUseServerTs(false);
 | 
				
			||||||
 | 
					        configuration.setPersistenceSettings(new OnEveryMessage());
 | 
				
			||||||
        return configuration;
 | 
					        return configuration;
 | 
				
			||||||
    }
 | 
					    }
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					    @JsonTypeInfo(
 | 
				
			||||||
 | 
					            use = JsonTypeInfo.Id.NAME,
 | 
				
			||||||
 | 
					            include = JsonTypeInfo.As.PROPERTY,
 | 
				
			||||||
 | 
					            property = "type"
 | 
				
			||||||
 | 
					    )
 | 
				
			||||||
 | 
					    @JsonSubTypes({
 | 
				
			||||||
 | 
					            @JsonSubTypes.Type(value = OnEveryMessage.class, name = "ON_EVERY_MESSAGE"),
 | 
				
			||||||
 | 
					            @JsonSubTypes.Type(value = WebSocketsOnly.class, name = "WEBSOCKETS_ONLY"),
 | 
				
			||||||
 | 
					            @JsonSubTypes.Type(value = Deduplicate.class, name = "DEDUPLICATE"),
 | 
				
			||||||
 | 
					            @JsonSubTypes.Type(value = Advanced.class, name = "ADVANCED")
 | 
				
			||||||
 | 
					    })
 | 
				
			||||||
 | 
					    sealed interface PersistenceSettings permits OnEveryMessage, Deduplicate, WebSocketsOnly, Advanced {
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					        record OnEveryMessage() implements PersistenceSettings {}
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					        record WebSocketsOnly() implements PersistenceSettings {}
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					        @Getter
 | 
				
			||||||
 | 
					        final class Deduplicate implements PersistenceSettings {
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					            public final PersistenceStrategy deduplicateStrategy;
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					            @JsonCreator
 | 
				
			||||||
 | 
					            private Deduplicate(@JsonProperty("deduplicationIntervalSecs") int deduplicationIntervalSecs) {
 | 
				
			||||||
 | 
					                this.deduplicateStrategy = PersistenceStrategy.deduplicate(deduplicationIntervalSecs);
 | 
				
			||||||
 | 
					            }
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					        }
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					        record Advanced(PersistenceStrategy timeseries, PersistenceStrategy latest, PersistenceStrategy webSockets) implements PersistenceSettings {
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					            public Advanced {
 | 
				
			||||||
 | 
					                Objects.requireNonNull(timeseries);
 | 
				
			||||||
 | 
					                Objects.requireNonNull(latest);
 | 
				
			||||||
 | 
					                Objects.requireNonNull(webSockets);
 | 
				
			||||||
 | 
					            }
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					        }
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					    }
 | 
				
			||||||
 | 
					
 | 
				
			||||||
}
 | 
					}
 | 
				
			||||||
 | 
				
			|||||||
@ -0,0 +1,54 @@
 | 
				
			|||||||
 | 
					/**
 | 
				
			||||||
 | 
					 * Copyright © 2016-2024 The Thingsboard Authors
 | 
				
			||||||
 | 
					 *
 | 
				
			||||||
 | 
					 * Licensed under the Apache License, Version 2.0 (the "License");
 | 
				
			||||||
 | 
					 * you may not use this file except in compliance with the License.
 | 
				
			||||||
 | 
					 * You may obtain a copy of the License at
 | 
				
			||||||
 | 
					 *
 | 
				
			||||||
 | 
					 *     http://www.apache.org/licenses/LICENSE-2.0
 | 
				
			||||||
 | 
					 *
 | 
				
			||||||
 | 
					 * Unless required by applicable law or agreed to in writing, software
 | 
				
			||||||
 | 
					 * distributed under the License is distributed on an "AS IS" BASIS,
 | 
				
			||||||
 | 
					 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 | 
				
			||||||
 | 
					 * See the License for the specific language governing permissions and
 | 
				
			||||||
 | 
					 * limitations under the License.
 | 
				
			||||||
 | 
					 */
 | 
				
			||||||
 | 
					package org.thingsboard.rule.engine.telemetry.strategy;
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					import com.fasterxml.jackson.annotation.JsonCreator;
 | 
				
			||||||
 | 
					import com.fasterxml.jackson.annotation.JsonProperty;
 | 
				
			||||||
 | 
					import com.github.benmanes.caffeine.cache.Caffeine;
 | 
				
			||||||
 | 
					import com.github.benmanes.caffeine.cache.LoadingCache;
 | 
				
			||||||
 | 
					import com.google.common.collect.Sets;
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					import java.time.Duration;
 | 
				
			||||||
 | 
					import java.util.Set;
 | 
				
			||||||
 | 
					import java.util.UUID;
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					final class DeduplicatePersistenceStrategy implements PersistenceStrategy {
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					    private static final int MIN_DEDUPLICATION_INTERVAL_SECS = 1;
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					    private final long deduplicationIntervalMillis;
 | 
				
			||||||
 | 
					    private final LoadingCache<Long, Set<UUID>> deduplicationCache;
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					    @JsonCreator
 | 
				
			||||||
 | 
					    public DeduplicatePersistenceStrategy(@JsonProperty("deduplicationIntervalSecs") int deduplicationIntervalSecs) {
 | 
				
			||||||
 | 
					        if (deduplicationIntervalSecs < MIN_DEDUPLICATION_INTERVAL_SECS) {
 | 
				
			||||||
 | 
					            throw new IllegalArgumentException("Deduplication interval must be at least " + MIN_DEDUPLICATION_INTERVAL_SECS + " second(s), was " + deduplicationIntervalSecs + " second(s)");
 | 
				
			||||||
 | 
					        }
 | 
				
			||||||
 | 
					        deduplicationIntervalMillis = Duration.ofSeconds(deduplicationIntervalSecs).toMillis();
 | 
				
			||||||
 | 
					        deduplicationCache = Caffeine.newBuilder()
 | 
				
			||||||
 | 
					                .softValues()
 | 
				
			||||||
 | 
					                .expireAfterAccess(Duration.ofSeconds(deduplicationIntervalSecs * 10L))
 | 
				
			||||||
 | 
					                .maximumSize(20L)
 | 
				
			||||||
 | 
					                .build(__ -> Sets.newConcurrentHashSet());
 | 
				
			||||||
 | 
					    }
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					    @Override
 | 
				
			||||||
 | 
					    public boolean shouldPersist(long ts, UUID originatorUuid) {
 | 
				
			||||||
 | 
					        long intervalNumber = ts / deduplicationIntervalMillis;
 | 
				
			||||||
 | 
					        return deduplicationCache.get(intervalNumber).add(originatorUuid);
 | 
				
			||||||
 | 
					    }
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					}
 | 
				
			||||||
@ -0,0 +1,38 @@
 | 
				
			|||||||
 | 
					/**
 | 
				
			||||||
 | 
					 * Copyright © 2016-2024 The Thingsboard Authors
 | 
				
			||||||
 | 
					 *
 | 
				
			||||||
 | 
					 * Licensed under the Apache License, Version 2.0 (the "License");
 | 
				
			||||||
 | 
					 * you may not use this file except in compliance with the License.
 | 
				
			||||||
 | 
					 * You may obtain a copy of the License at
 | 
				
			||||||
 | 
					 *
 | 
				
			||||||
 | 
					 *     http://www.apache.org/licenses/LICENSE-2.0
 | 
				
			||||||
 | 
					 *
 | 
				
			||||||
 | 
					 * Unless required by applicable law or agreed to in writing, software
 | 
				
			||||||
 | 
					 * distributed under the License is distributed on an "AS IS" BASIS,
 | 
				
			||||||
 | 
					 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 | 
				
			||||||
 | 
					 * See the License for the specific language governing permissions and
 | 
				
			||||||
 | 
					 * limitations under the License.
 | 
				
			||||||
 | 
					 */
 | 
				
			||||||
 | 
					package org.thingsboard.rule.engine.telemetry.strategy;
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					import com.fasterxml.jackson.annotation.JsonCreator;
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					import java.util.UUID;
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					final class OnEveryMessagePersistenceStrategy implements PersistenceStrategy {
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					    private static final OnEveryMessagePersistenceStrategy INSTANCE = new OnEveryMessagePersistenceStrategy();
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					    private OnEveryMessagePersistenceStrategy() {}
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					    @JsonCreator
 | 
				
			||||||
 | 
					    public static OnEveryMessagePersistenceStrategy getInstance() {
 | 
				
			||||||
 | 
					        return INSTANCE;
 | 
				
			||||||
 | 
					    }
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					    @Override
 | 
				
			||||||
 | 
					    public boolean shouldPersist(long ts, UUID originatorUuid) {
 | 
				
			||||||
 | 
					        return true;
 | 
				
			||||||
 | 
					    }
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					}
 | 
				
			||||||
@ -0,0 +1,49 @@
 | 
				
			|||||||
 | 
					/**
 | 
				
			||||||
 | 
					 * Copyright © 2016-2024 The Thingsboard Authors
 | 
				
			||||||
 | 
					 *
 | 
				
			||||||
 | 
					 * Licensed under the Apache License, Version 2.0 (the "License");
 | 
				
			||||||
 | 
					 * you may not use this file except in compliance with the License.
 | 
				
			||||||
 | 
					 * You may obtain a copy of the License at
 | 
				
			||||||
 | 
					 *
 | 
				
			||||||
 | 
					 *     http://www.apache.org/licenses/LICENSE-2.0
 | 
				
			||||||
 | 
					 *
 | 
				
			||||||
 | 
					 * Unless required by applicable law or agreed to in writing, software
 | 
				
			||||||
 | 
					 * distributed under the License is distributed on an "AS IS" BASIS,
 | 
				
			||||||
 | 
					 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 | 
				
			||||||
 | 
					 * See the License for the specific language governing permissions and
 | 
				
			||||||
 | 
					 * limitations under the License.
 | 
				
			||||||
 | 
					 */
 | 
				
			||||||
 | 
					package org.thingsboard.rule.engine.telemetry.strategy;
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					import com.fasterxml.jackson.annotation.JsonSubTypes;
 | 
				
			||||||
 | 
					import com.fasterxml.jackson.annotation.JsonTypeInfo;
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					import java.util.UUID;
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					@JsonTypeInfo(
 | 
				
			||||||
 | 
					        use = JsonTypeInfo.Id.NAME,
 | 
				
			||||||
 | 
					        include = JsonTypeInfo.As.PROPERTY,
 | 
				
			||||||
 | 
					        property = "type"
 | 
				
			||||||
 | 
					)
 | 
				
			||||||
 | 
					@JsonSubTypes({
 | 
				
			||||||
 | 
					        @JsonSubTypes.Type(value = OnEveryMessagePersistenceStrategy.class, name = "ON_EVERY_MESSAGE"),
 | 
				
			||||||
 | 
					        @JsonSubTypes.Type(value = DeduplicatePersistenceStrategy.class, name = "DEDUPLICATE"),
 | 
				
			||||||
 | 
					        @JsonSubTypes.Type(value = SkipPersistenceStrategy.class, name = "SKIP")
 | 
				
			||||||
 | 
					})
 | 
				
			||||||
 | 
					public sealed interface PersistenceStrategy permits OnEveryMessagePersistenceStrategy, DeduplicatePersistenceStrategy, SkipPersistenceStrategy {
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					    static PersistenceStrategy onEveryMessage() {
 | 
				
			||||||
 | 
					        return OnEveryMessagePersistenceStrategy.getInstance();
 | 
				
			||||||
 | 
					    }
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					    static PersistenceStrategy deduplicate(int deduplicationIntervalSecs) {
 | 
				
			||||||
 | 
					        return new DeduplicatePersistenceStrategy(deduplicationIntervalSecs);
 | 
				
			||||||
 | 
					    }
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					    static PersistenceStrategy skip() {
 | 
				
			||||||
 | 
					        return SkipPersistenceStrategy.getInstance();
 | 
				
			||||||
 | 
					    }
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					    boolean shouldPersist(long ts, UUID originatorUuid);
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					}
 | 
				
			||||||
@ -0,0 +1,38 @@
 | 
				
			|||||||
 | 
					/**
 | 
				
			||||||
 | 
					 * Copyright © 2016-2024 The Thingsboard Authors
 | 
				
			||||||
 | 
					 *
 | 
				
			||||||
 | 
					 * Licensed under the Apache License, Version 2.0 (the "License");
 | 
				
			||||||
 | 
					 * you may not use this file except in compliance with the License.
 | 
				
			||||||
 | 
					 * You may obtain a copy of the License at
 | 
				
			||||||
 | 
					 *
 | 
				
			||||||
 | 
					 *     http://www.apache.org/licenses/LICENSE-2.0
 | 
				
			||||||
 | 
					 *
 | 
				
			||||||
 | 
					 * Unless required by applicable law or agreed to in writing, software
 | 
				
			||||||
 | 
					 * distributed under the License is distributed on an "AS IS" BASIS,
 | 
				
			||||||
 | 
					 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 | 
				
			||||||
 | 
					 * See the License for the specific language governing permissions and
 | 
				
			||||||
 | 
					 * limitations under the License.
 | 
				
			||||||
 | 
					 */
 | 
				
			||||||
 | 
					package org.thingsboard.rule.engine.telemetry.strategy;
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					import com.fasterxml.jackson.annotation.JsonCreator;
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					import java.util.UUID;
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					final class SkipPersistenceStrategy implements PersistenceStrategy {
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					    private static final SkipPersistenceStrategy INSTANCE = new SkipPersistenceStrategy();
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					    private SkipPersistenceStrategy() {}
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					    @JsonCreator
 | 
				
			||||||
 | 
					    public static SkipPersistenceStrategy getInstance() {
 | 
				
			||||||
 | 
					        return INSTANCE;
 | 
				
			||||||
 | 
					    }
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					    @Override
 | 
				
			||||||
 | 
					    public boolean shouldPersist(long ts, UUID originatorUuid) {
 | 
				
			||||||
 | 
					        return false;
 | 
				
			||||||
 | 
					    }
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					}
 | 
				
			||||||
@ -33,6 +33,7 @@ import org.thingsboard.rule.engine.api.TbContext;
 | 
				
			|||||||
import org.thingsboard.rule.engine.api.TbNodeConfiguration;
 | 
					import org.thingsboard.rule.engine.api.TbNodeConfiguration;
 | 
				
			||||||
import org.thingsboard.rule.engine.api.TbNodeException;
 | 
					import org.thingsboard.rule.engine.api.TbNodeException;
 | 
				
			||||||
import org.thingsboard.rule.engine.api.TimeseriesSaveRequest;
 | 
					import org.thingsboard.rule.engine.api.TimeseriesSaveRequest;
 | 
				
			||||||
 | 
					import org.thingsboard.rule.engine.telemetry.strategy.PersistenceStrategy;
 | 
				
			||||||
import org.thingsboard.server.common.adaptor.JsonConverter;
 | 
					import org.thingsboard.server.common.adaptor.JsonConverter;
 | 
				
			||||||
import org.thingsboard.server.common.data.TenantProfile;
 | 
					import org.thingsboard.server.common.data.TenantProfile;
 | 
				
			||||||
import org.thingsboard.server.common.data.id.DeviceId;
 | 
					import org.thingsboard.server.common.data.id.DeviceId;
 | 
				
			||||||
@ -88,7 +89,7 @@ public class TbMsgTimeseriesNodeTest {
 | 
				
			|||||||
    @Test
 | 
					    @Test
 | 
				
			||||||
    public void verifyDefaultConfig() {
 | 
					    public void verifyDefaultConfig() {
 | 
				
			||||||
        assertThat(config.getDefaultTTL()).isEqualTo(0L);
 | 
					        assertThat(config.getDefaultTTL()).isEqualTo(0L);
 | 
				
			||||||
        assertThat(config.isSkipLatestPersistence()).isFalse();
 | 
					        assertThat(config.getPersistenceSettings()).isInstanceOf(TbMsgTimeseriesNodeConfiguration.PersistenceSettings.OnEveryMessage.class);
 | 
				
			||||||
        assertThat(config.isUseServerTs()).isFalse();
 | 
					        assertThat(config.isUseServerTs()).isFalse();
 | 
				
			||||||
    }
 | 
					    }
 | 
				
			||||||
 | 
					
 | 
				
			||||||
@ -162,7 +163,13 @@ public class TbMsgTimeseriesNodeTest {
 | 
				
			|||||||
    public void givenSkipLatestPersistenceIsTrueAndTtlFromConfig_whenOnMsg_thenSaveTimeseriesUsingTtlFromConfig() throws TbNodeException {
 | 
					    public void givenSkipLatestPersistenceIsTrueAndTtlFromConfig_whenOnMsg_thenSaveTimeseriesUsingTtlFromConfig() throws TbNodeException {
 | 
				
			||||||
        long ttlFromConfig = 5L;
 | 
					        long ttlFromConfig = 5L;
 | 
				
			||||||
        config.setDefaultTTL(ttlFromConfig);
 | 
					        config.setDefaultTTL(ttlFromConfig);
 | 
				
			||||||
        config.setSkipLatestPersistence(true);
 | 
					
 | 
				
			||||||
 | 
					        var timeseriesStrategy = PersistenceStrategy.onEveryMessage();
 | 
				
			||||||
 | 
					        var latestStrategy = PersistenceStrategy.skip();
 | 
				
			||||||
 | 
					        var webSockets = PersistenceStrategy.onEveryMessage();
 | 
				
			||||||
 | 
					        var persistenceSettings = new TbMsgTimeseriesNodeConfiguration.PersistenceSettings.Advanced(timeseriesStrategy, latestStrategy, webSockets);
 | 
				
			||||||
 | 
					        config.setPersistenceSettings(persistenceSettings);
 | 
				
			||||||
 | 
					
 | 
				
			||||||
        init();
 | 
					        init();
 | 
				
			||||||
 | 
					
 | 
				
			||||||
        String data = """
 | 
					        String data = """
 | 
				
			||||||
@ -197,7 +204,9 @@ public class TbMsgTimeseriesNodeTest {
 | 
				
			|||||||
            assertThat(request.getEntityId()).isEqualTo(DEVICE_ID);
 | 
					            assertThat(request.getEntityId()).isEqualTo(DEVICE_ID);
 | 
				
			||||||
            assertThat(request.getEntries()).containsExactlyElementsOf(expectedList);
 | 
					            assertThat(request.getEntries()).containsExactlyElementsOf(expectedList);
 | 
				
			||||||
            assertThat(request.getTtl()).isEqualTo(ttlFromConfig);
 | 
					            assertThat(request.getTtl()).isEqualTo(ttlFromConfig);
 | 
				
			||||||
 | 
					            assertThat(request.isSaveTimeseries()).isTrue();
 | 
				
			||||||
            assertThat(request.isSaveLatest()).isFalse();
 | 
					            assertThat(request.isSaveLatest()).isFalse();
 | 
				
			||||||
 | 
					            assertThat(request.isSendWsUpdate()).isTrue();
 | 
				
			||||||
            assertThat(request.getCallback()).isInstanceOf(TelemetryNodeCallback.class);
 | 
					            assertThat(request.getCallback()).isInstanceOf(TelemetryNodeCallback.class);
 | 
				
			||||||
        }));
 | 
					        }));
 | 
				
			||||||
        verify(ctxMock).tellSuccess(msg);
 | 
					        verify(ctxMock).tellSuccess(msg);
 | 
				
			||||||
 | 
				
			|||||||
		Loading…
	
	
			
			x
			
			
		
	
		Reference in New Issue
	
	Block a user