dendi-wang/thingsboard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #13555 from irynamatveieva/cf-msg-ts

Browse Source 
Replaced 'use message ts' with 'use latest ts'
This commit is contained in:
Viacheslav Klimov 2025-06-11 12:21:33 +03:00 committed by GitHub
commit 9567e9e0ce
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
16 changed files with 108 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java
View File

@ -244,7 +244,7 @@ public class CalculatedFieldController extends BaseController {
);
Object[] args = new Object[ctxAndArgNames.size()];
args[0] = new TbelCfCtx(arguments, getLastUpdateTimestamp(arguments));
args[0] = new TbelCfCtx(arguments, getLatestTimestamp(arguments));
for (int i = 1; i < ctxAndArgNames.size(); i++) {
var arg = arguments.get(ctxAndArgNames.get(i));
if (arg instanceof TbelCfSingleValueArg svArg) {
@ -267,7 +267,7 @@ public class CalculatedFieldController extends BaseController {
return result;
}
private long getLastUpdateTimestamp(Map<String, TbelCfArg> arguments) {
private long getLatestTimestamp(Map<String, TbelCfArg> arguments) {
long lastUpdateTimestamp = -1;
for (TbelCfArg entry : arguments.values()) {
if (entry instanceof TbelCfSingleValueArg singleValueArg) {

2
application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java
View File

@ -177,7 +177,7 @@ public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueS
for (int i = 0; i < entries.size(); i++) {
TsKvProto.Builder tsProtoBuilder = toTsKvProto(entries.get(i)).toBuilder();
if (result != null) {
if (versions != null && !versions.isEmpty() && versions.get(i) != null) {
tsProtoBuilder.setVersion(versions.get(i));
}
telemetryMsg.addTsData(tsProtoBuilder.build());

8
application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java
View File

@ -35,7 +35,7 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState {
protected Map<String, ArgumentEntry> arguments;
protected boolean sizeExceedsLimit;
protected long lastUpdateTimestamp = -1;
protected long latestTimestamp = -1;
public BaseCalculatedFieldState(List<String> requiredArguments) {
this.requiredArguments = requiredArguments;
@ -110,12 +110,14 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState {
protected abstract void validateNewEntry(ArgumentEntry newEntry);
private void updateLastUpdateTimestamp(ArgumentEntry entry) {
long newTs = this.latestTimestamp;
if (entry instanceof SingleValueArgumentEntry singleValueArgumentEntry) {
this.lastUpdateTimestamp = singleValueArgumentEntry.getTs();
newTs = singleValueArgumentEntry.getTs();
} else if (entry instanceof TsRollingArgumentEntry tsRollingArgumentEntry) {
Map.Entry<Long, Double> lastEntry = tsRollingArgumentEntry.getTsRecords().lastEntry();
this.lastUpdateTimestamp = (lastEntry != null) ? lastEntry.getKey() : System.currentTimeMillis();
newTs = (lastEntry != null) ? lastEntry.getKey() : System.currentTimeMillis();
}
this.latestTimestamp = Math.max(this.latestTimestamp, newTs);
}
}

4
application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java
View File

@ -62,7 +62,7 @@ public class CalculatedFieldCtx {
private final List<String> argNames;
private Output output;
private String expression;
private boolean preserveMsgTs;
private boolean useLatestTs;
private TbelInvokeService tbelInvokeService;
private CalculatedFieldScriptEngine calculatedFieldScriptEngine;
private ThreadLocal<Expression> customExpression;
@ -96,7 +96,7 @@ public class CalculatedFieldCtx {
this.argNames = new ArrayList<>(arguments.keySet());
this.output = configuration.getOutput();
this.expression = configuration.getExpression();
this.preserveMsgTs = CalculatedFieldType.SIMPLE.equals(calculatedField.getType()) && ((SimpleCalculatedFieldConfiguration) configuration).isPreserveMsgTs();
this.useLatestTs = CalculatedFieldType.SIMPLE.equals(calculatedField.getType()) && ((SimpleCalculatedFieldConfiguration) configuration).isUseLatestTs();
this.tbelInvokeService = tbelInvokeService;
this.maxDataPointsPerRollingArg = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxDataPointsPerRollingArg);

2
application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java
View File

@ -42,7 +42,7 @@ public interface CalculatedFieldState {
Map<String, ArgumentEntry> getArguments();
long getLastUpdateTimestamp();
long getLatestTimestamp();
void setRequiredArguments(List<String> requiredArguments);

2
application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java
View File

@ -66,7 +66,7 @@ public class ScriptCalculatedFieldState extends BaseCalculatedFieldState {
args.add(arg);
}
}
args.set(0, new TbelCfCtx(arguments, getLastUpdateTimestamp()));
args.set(0, new TbelCfCtx(arguments, getLatestTimestamp()));
ListenableFuture<JsonNode> resultFuture = ctx.getCalculatedFieldScriptEngine().executeJsonAsync(args.toArray());
Output output = ctx.getOutput();
return Futures.transform(resultFuture,

10
application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java
View File

@ -68,7 +68,7 @@ public class SimpleCalculatedFieldState extends BaseCalculatedFieldState {
Output output = ctx.getOutput();
Object result = formatResult(expressionResult, output.getDecimalsByDefault());
JsonNode outputResult = createResultJson(ctx.isPreserveMsgTs(), output.getName(), result);
JsonNode outputResult = createResultJson(ctx.isUseLatestTs(), output.getName(), result);
return Futures.immediateFuture(new CalculatedFieldResult(output.getType(), output.getScope(), outputResult));
}
@ -83,14 +83,14 @@ public class SimpleCalculatedFieldState extends BaseCalculatedFieldState {
return TbUtils.toFixed(expressionResult, decimals);
}
private JsonNode createResultJson(boolean preserveMsgTs, String outputName, Object result) {
private JsonNode createResultJson(boolean useLatestTs, String outputName, Object result) {
ObjectNode valuesNode = JacksonUtil.newObjectNode();
valuesNode.set(outputName, JacksonUtil.valueToTree(result));
long lastTimestamp = getLastUpdateTimestamp();
if (preserveMsgTs && lastTimestamp != -1) {
long latestTs = getLatestTimestamp();
if (useLatestTs && latestTs != -1) {
ObjectNode resultNode = JacksonUtil.newObjectNode();
resultNode.put("ts", lastTimestamp);
resultNode.put("ts", latestTs);
resultNode.set("values", valuesNode);
return resultNode;
} else {

70
application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java
View File

@ -464,7 +464,7 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes
}
@Test
public void testSimpleCalculatedFieldWhenPreserveMsgTsIsTrue() throws Exception {
public void testSimpleCalculatedFieldWhenUseLatestTsIsTrue() throws Exception {
Device testDevice = createDevice("Test device", "1234567890");
long ts = System.currentTimeMillis() - 300000L;
doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode(String.format("{\"ts\": %s, \"values\": {\"temperature\":30}}", ts)));
@ -489,7 +489,7 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes
output.setType(OutputType.TIME_SERIES);
config.setOutput(output);
config.setPreserveMsgTs(true);
config.setUseLatestTs(true);
calculatedField.setConfiguration(config);
@ -506,7 +506,69 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes
}
@Test
public void testScriptCalculatedFieldWhenUsedMsgTsInScript() throws Exception {
public void testSimpleCalculatedFieldWhenUseLatestTsIsTrueAndTelemetryBeforeLatest() throws Exception {
Device testDevice = createDevice("Test device", "1234567890");
long ts = System.currentTimeMillis();
long tsA = ts - 300000L;
doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode(String.format("{\"ts\": %s, \"values\": {\"a\":1}}", tsA)));
long tsB = ts - 300L;
doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode(String.format("{\"ts\": %s, \"values\": {\"b\":5}}", tsB)));
CalculatedField calculatedField = new CalculatedField();
calculatedField.setEntityId(testDevice.getId());
calculatedField.setType(CalculatedFieldType.SIMPLE);
calculatedField.setName("a + b");
calculatedField.setDebugSettings(DebugSettings.all());
calculatedField.setConfigurationVersion(1);
SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration();
Argument argument1 = new Argument();
ReferencedEntityKey refEntityKey1 = new ReferencedEntityKey("a", ArgumentType.TS_LATEST, null);
argument1.setRefEntityKey(refEntityKey1);
Argument argument2 = new Argument();
ReferencedEntityKey refEntityKey2 = new ReferencedEntityKey("b", ArgumentType.TS_LATEST, null);
argument2.setRefEntityKey(refEntityKey2);
config.setArguments(Map.of("a", argument1, "b", argument2));
config.setExpression("a + b");
Output output = new Output();
output.setName("c");
output.setType(OutputType.TIME_SERIES);
config.setOutput(output);
config.setUseLatestTs(true);
calculatedField.setConfiguration(config);
CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class);
await().alias("create CF -> perform initial calculation").atMost(TIMEOUT, TimeUnit.SECONDS)
.pollInterval(POLL_INTERVAL, TimeUnit.SECONDS)
.untilAsserted(() -> {
ObjectNode c = getLatestTelemetry(testDevice.getId(), "c");
assertThat(c).isNotNull();
assertThat(c.get("c").get(0).get("ts").asText()).isEqualTo(Long.toString(tsB));
assertThat(c.get("c").get(0).get("value").asText()).isEqualTo("6.0");
});
long tsABeforeTsB = tsB - 300L;
doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode(String.format("{\"ts\": %s, \"values\": {\"b\":10}}", tsABeforeTsB)));
await().alias("update telemetry with ts less than latest -> save result with latest ts").atMost(TIMEOUT, TimeUnit.SECONDS)
.pollInterval(POLL_INTERVAL, TimeUnit.SECONDS)
.untilAsserted(() -> {
ObjectNode c = getLatestTelemetry(testDevice.getId(), "c");
assertThat(c).isNotNull();
assertThat(c.get("c").get(0).get("ts").asText()).isEqualTo(Long.toString(tsB));// also tsB, since this is the latest timestamp
assertThat(c.get("c").get(0).get("value").asText()).isEqualTo("11.0");
});
}
@Test
public void testScriptCalculatedFieldWhenUsedLatestTsInScript() throws Exception {
Device testDevice = createDevice("Test device", "1234567890");
long ts = System.currentTimeMillis() - 300000L;
doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode(String.format("{\"ts\": %s, \"values\": {\"temperature\":30}}", ts)));
@ -524,7 +586,7 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes
ReferencedEntityKey refEntityKey = new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null);
argument.setRefEntityKey(refEntityKey);
config.setArguments(Map.of("T", argument));
config.setExpression("return {\"ts\": ctx.msgTs, \"values\": {\"fahrenheitTemp\": (T * 1.8) + 32}};");
config.setExpression("return {\"ts\": ctx.latestTs, \"values\": {\"fahrenheitTemp\": (T * 1.8) + 32}};");
Output output = new Output();
output.setType(OutputType.TIME_SERIES);

2
common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/SimpleCalculatedFieldConfiguration.java
View File

@ -23,7 +23,7 @@ import org.thingsboard.server.common.data.cf.CalculatedFieldType;
@EqualsAndHashCode(callSuper = true)
public class SimpleCalculatedFieldConfiguration extends BaseCalculatedFieldConfiguration implements CalculatedFieldConfiguration {
private boolean preserveMsgTs;
private boolean useLatestTs;
@Override
public CalculatedFieldType getType() {

6
common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfCtx.java
View File

@ -25,11 +25,11 @@ public class TbelCfCtx implements TbelCfObject {
@Getter
private final Map<String, TbelCfArg> args;
@Getter
private final long msgTs;
private final long latestTs;
public TbelCfCtx(Map<String, TbelCfArg> args, long lastUpdateTs) {
public TbelCfCtx(Map<String, TbelCfArg> args, long latestTs) {
this.args = Collections.unmodifiableMap(args);
this.msgTs = lastUpdateTs != -1 ? lastUpdateTs : System.currentTimeMillis();
this.latestTs = latestTs != -1 ? latestTs : System.currentTimeMillis();
}
@Override

6
ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html
View File

@ -190,9 +190,9 @@
</mat-form-field>
</div>
<div class="tb-form-row" [formGroup]="configFormGroup" *ngIf="outputFormGroup.get('type').value === OutputType.Timeseries">
<mat-slide-toggle class="mat-slide" formControlName="preserveMsgTs">
<div tb-hint-tooltip-icon="{{ 'calculated-fields.hint.use-message-timestamp' | translate }}" translate>
calculated-fields.use-message-timestamp
<mat-slide-toggle class="mat-slide" formControlName="useLatestTs">
<div tb-hint-tooltip-icon="{{ 'calculated-fields.hint.use-latest-timestamp' | translate }}" translate>
calculated-fields.use-latest-timestamp
</div>
</mat-slide-toggle>
</div>

2
ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.scss
View File

@ -45,7 +45,7 @@
&-key {
color: #c24c1a;
}
&-time-window, &-values, &-func, &-value, &-ts, &-msgTs {
&-time-window, &-values, &-func, &-value, &-ts, &-latestTs {
color: #7214D0;
}
&-start-ts, &-end-ts {

14
ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts
View File

@ -77,7 +77,7 @@ export class CalculatedFieldDialogComponent extends DialogComponent<CalculatedFi
type: [OutputType.Timeseries],
decimalsByDefault: [null as number, [Validators.min(0), Validators.max(15), Validators.pattern(digitsRegex)]],
}),
preserveMsgTs: [false]
useLatestTs: [false]
}),
});
@ -212,12 +212,12 @@ export class CalculatedFieldDialogComponent extends DialogComponent<CalculatedFi
}
if (this.fieldFormGroup.get('type').value === CalculatedFieldType.SIMPLE) {
if (type === OutputType.Attribute) {
this.configFormGroup.get('preserveMsgTs').disable({emitEvent: false});
this.configFormGroup.get('useLatestTs').disable({emitEvent: false});
} else {
this.configFormGroup.get('preserveMsgTs').enable({emitEvent: false});
this.configFormGroup.get('useLatestTs').enable({emitEvent: false});
}
} else {
this.configFormGroup.get('preserveMsgTs').disable({emitEvent: false});
this.configFormGroup.get('useLatestTs').disable({emitEvent: false});
}
}
@ -227,13 +227,13 @@ export class CalculatedFieldDialogComponent extends DialogComponent<CalculatedFi
this.configFormGroup.get('expressionSIMPLE').enable({emitEvent: false});
this.configFormGroup.get('expressionSCRIPT').disable({emitEvent: false});
if (this.outputFormGroup.get('type').value === OutputType.Attribute) {
this.configFormGroup.get('preserveMsgTs').disable({emitEvent: false});
this.configFormGroup.get('useLatestTs').disable({emitEvent: false});
} else {
this.configFormGroup.get('preserveMsgTs').enable({emitEvent: false});
this.configFormGroup.get('useLatestTs').enable({emitEvent: false});
}
} else {
this.outputFormGroup.get('name').disable({emitEvent: false});
this.configFormGroup.get('preserveMsgTs').disable({emitEvent: false});
this.configFormGroup.get('useLatestTs').disable({emitEvent: false});
this.configFormGroup.get('expressionSIMPLE').disable({emitEvent: false});
this.configFormGroup.get('expressionSCRIPT').enable({emitEvent: false});
}

8
ui-ngx/src/app/shared/models/calculated-field.models.ts
View File

@ -526,10 +526,10 @@ export const getCalculatedFieldArgumentsEditorCompleter = (argumentsObj: Record<
description: 'Calculated field context arguments.',
children: {}
},
msgTs: {
latestTs: {
meta: 'constant',
type: 'number',
description: 'Timestamp (ms) of the telemetry message that triggered the calculated field execution.'
description: 'Latest timestamp (ms) of the arguments telemetry.'
}
}
}
@ -582,8 +582,8 @@ const calculatedFieldArgumentsContextValueHighlightRules: AceHighlightRules = {
next: 'calculatedFieldCtxArgs'
},
{
token: 'tb.calculated-field-msgTs',
regex: /msgTs/,
token: 'tb.calculated-field-latestTs',
regex: /latestTs/,
next: 'no_regex'
},
endGroupHighlightRule

8
ui-ngx/src/assets/help/en_US/calculated-field/expression_fn.md
View File

@ -1,7 +1,7 @@
## Calculated Field TBEL Script Function
The **calculate()** function is a user-defined script that enables custom calculations using [TBEL](${siteBaseUrl}/docs${docPlatformPrefix}/user-guide/tbel/) on telemetry and attribute data.
It receives arguments configured in the calculated field setup, along with an additional `ctx` object that stores `msgTs` and provides access to all arguments.
It receives arguments configured in the calculated field setup, along with an additional `ctx` object that stores `latestTs` and provides access to all arguments.
### Function Signature
@ -216,14 +216,14 @@ The return format depends on the output type configured in the calculated field
### Message timestamp
The `ctx` object also includes property `msgTs`, which represents the timestamp of the incoming telemetry message that triggered the calculated field execution in milliseconds.
The `ctx` object also includes property `latestTs`, which represents the latest timestamp of the arguments telemetry in milliseconds.
You can use `ctx.msgTs` to set the timestamp of the resulting output explicitly when returning a time series object.
You can use `ctx.latestTs` to set the timestamp of the resulting output explicitly when returning a time series object.
```javascript
var temperatureC = (temperatureF - 32) / 1.8;
return {
ts: ctx.msgTs,
ts: ctx.latestTs,
values: {
"temperatureC": toFixed(temperatureC, 2)
}

4
ui-ngx/src/assets/locale/locale.constant-en_US.json
View File

@ -1069,7 +1069,7 @@
"delete-multiple-title": "Are you sure you want to delete { count, plural, =1 {1 calculated field} other {# calculated fields} }?",
"delete-multiple-text": "Be careful, after the confirmation all selected calculated fields will be removed and all related data will become unrecoverable.",
"test-with-this-message": "Test with this message",
"use-message-timestamp": "Use message timestamp",
"use-latest-timestamp": "Use latest timestamp",
"hint": {
"arguments-simple-with-rolling": "Simple type calculated field should not contain keys with time series rolling type.",
"arguments-empty": "Arguments should not be empty.",
@ -1086,7 +1086,7 @@
"decimals-range": "Decimals by default should be a number between 0 and 15.",
"expression": "Default expression demonstrates how to transform a temperature from Fahrenheit to Celsius.",
"arguments-entity-not-found": "Argument target entity not found.",
"use-message-timestamp": "If enabled, the calculated value will be persisted using the timestamp of the telemetry that triggered the calculation, instead of the server time."
"use-latest-timestamp": "If enabled, the calculated value will be persisted using the most recent timestamp from the arguments telemetry, instead of the server time."
}
},
"confirm-on-exit": {