dendi-wang/thingsboard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MVEL executor

Browse Source
This commit is contained in:
Andrii Shvaika 2022-10-17 19:03:44 +03:00
parent 9769f95c27
commit 813f632deb
21 changed files with 695 additions and 302 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
application/src/main/java/org/thingsboard/server/ThingsboardServerApplication.java
View File

@ -26,7 +26,7 @@ import java.util.Arrays;
@SpringBootConfiguration
@EnableAsync
@EnableScheduling
@ComponentScan({"org.thingsboard.server"})
@ComponentScan({"org.thingsboard.server", "org.thingsboard.script"})
public class ThingsboardServerApplication {
private static final String SPRING_CONFIG_NAME_KEY = "--spring.config.name";

4
application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java
View File

@ -90,7 +90,7 @@ import org.thingsboard.server.service.profile.TbDeviceProfileCache;
import org.thingsboard.server.service.rpc.TbCoreDeviceRpcService;
import org.thingsboard.server.service.rpc.TbRpcService;
import org.thingsboard.server.service.rpc.TbRuleEngineDeviceRpcService;
import org.thingsboard.script.api.JsInvokeService;
import org.thingsboard.script.api.ScriptInvokeService;
import org.thingsboard.server.service.session.DeviceSessionCacheService;
import org.thingsboard.server.service.sms.SmsExecutorService;
import org.thingsboard.server.service.state.DeviceStateService;
@ -267,7 +267,7 @@ public class ActorSystemContext {
@Autowired
@Getter
private JsInvokeService jsSandbox;
private ScriptInvokeService jsSandbox;
@Autowired
@Getter

6
application/src/main/java/org/thingsboard/server/controller/RuleChainController.java
View File

@ -65,7 +65,7 @@ import org.thingsboard.server.common.msg.TbMsgMetaData;
import org.thingsboard.server.dao.event.EventService;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.rule.TbRuleChainService;
import org.thingsboard.script.api.JsInvokeService;
import org.thingsboard.script.api.ScriptInvokeService;
import org.thingsboard.server.service.script.RuleNodeJsScriptEngine;
import org.thingsboard.server.service.security.permission.Operation;
import org.thingsboard.server.service.security.permission.Resource;
@ -140,7 +140,7 @@ public class RuleChainController extends BaseController {
private EventService eventService;
@Autowired
private JsInvokeService jsInvokeService;
private ScriptInvokeService scriptInvokeService;
@Autowired(required = false)
private ActorSystemContext actorContext;
@ -393,7 +393,7 @@ public class RuleChainController extends BaseController {
String errorText = "";
ScriptEngine engine = null;
try {
engine = new RuleNodeJsScriptEngine(getTenantId(), jsInvokeService, getCurrentUser().getId(), script, argNames);
engine = new RuleNodeJsScriptEngine(getTenantId(), scriptInvokeService, getCurrentUser().getId(), script, argNames);
TbMsg inMsg = TbMsg.newMsg(msgType, null, new TbMsgMetaData(metadata), TbMsgDataType.JSON, data);
switch (scriptType) {
case "update":

10
application/src/test/java/org/thingsboard/server/service/script/MockJsInvokeService.java
View File

@ -20,8 +20,8 @@ import com.google.common.util.concurrent.ListenableFuture;
import lombok.extern.slf4j.Slf4j;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Service;
import org.thingsboard.script.api.JsInvokeService;
import org.thingsboard.script.api.JsScriptType;
import org.thingsboard.script.api.ScriptInvokeService;
import org.thingsboard.script.api.ScriptType;
import org.thingsboard.server.common.data.id.CustomerId;
import org.thingsboard.server.common.data.id.TenantId;
@ -30,16 +30,16 @@ import java.util.UUID;
@Slf4j
@Service
@ConditionalOnProperty(prefix = "js", value = "evaluator", havingValue = "mock")
public class MockJsInvokeService implements JsInvokeService {
public class MockJsInvokeService implements ScriptInvokeService {
@Override
public ListenableFuture<UUID> eval(TenantId tenantId, JsScriptType scriptType, String scriptBody, String... argNames) {
public ListenableFuture<UUID> eval(TenantId tenantId, ScriptType scriptType, String scriptBody, String... argNames) {
log.warn("eval {} {} {} {}", tenantId, scriptType, scriptBody, argNames);
return Futures.immediateFuture(UUID.randomUUID());
}
@Override
public ListenableFuture<String> invokeFunction(TenantId tenantId, CustomerId customerId, UUID scriptId, Object... args) {
public ListenableFuture<String> invokeScript(TenantId tenantId, CustomerId customerId, UUID scriptId, Object... args) {
log.warn("invokeFunction {} {} {} {}", tenantId, customerId, scriptId, args);
return Futures.immediateFuture("{}");
}

10
application/src/test/java/org/thingsboard/server/service/script/JsInvokeServiceTest.java → application/src/test/java/org/thingsboard/server/service/script/ScriptInvokeServiceTest.java
View File

@ -19,8 +19,8 @@ import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.test.context.TestPropertySource;
import org.thingsboard.script.api.JsScriptType;
import org.thingsboard.script.api.NashornJsInvokeService;
import org.thingsboard.script.api.ScriptType;
import org.thingsboard.script.api.js.NashornJsInvokeService;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.controller.AbstractControllerTest;
import org.thingsboard.server.dao.service.DaoSqlTest;
@ -37,7 +37,7 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy;
"js.max_result_size=50",
"js.local.max_errors=2"
})
class JsInvokeServiceTest extends AbstractControllerTest {
class ScriptInvokeServiceTest extends AbstractControllerTest {
@Autowired
private NashornJsInvokeService jsInvokeService;
@ -88,11 +88,11 @@ class JsInvokeServiceTest extends AbstractControllerTest {
}
private UUID evalScript(String script) throws ExecutionException, InterruptedException {
return jsInvokeService.eval(TenantId.SYS_TENANT_ID, JsScriptType.RULE_NODE_SCRIPT, script).get();
return jsInvokeService.eval(TenantId.SYS_TENANT_ID, ScriptType.RULE_NODE_SCRIPT, script).get();
}
private String invokeScript(UUID scriptId, String msg) throws ExecutionException, InterruptedException {
return jsInvokeService.invokeFunction(TenantId.SYS_TENANT_ID, null, scriptId, msg, "{}", "POST_TELEMETRY_REQUEST").get();
return jsInvokeService.invokeScript(TenantId.SYS_TENANT_ID, null, scriptId, msg, "{}", "POST_TELEMETRY_REQUEST").get();
}
}

107
common/script/remote-js-client/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java
View File

@ -15,7 +15,6 @@
*/
package org.thingsboard.server.service.script;
import com.google.common.util.concurrent.FutureCallback;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import lombok.Getter;
@ -27,7 +26,7 @@ import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import org.springframework.util.StopWatch;
import org.thingsboard.common.util.ThingsBoardThreadFactory;
import org.thingsboard.script.api.AbstractJsInvokeService;
import org.thingsboard.script.api.js.AbstractJsInvokeService;
import org.thingsboard.server.common.stats.TbApiUsageReportClient;
import org.thingsboard.server.common.stats.TbApiUsageStateClient;
import org.thingsboard.server.gen.js.JsInvokeProtos;
@ -35,29 +34,29 @@ import org.thingsboard.server.queue.TbQueueRequestTemplate;
import org.thingsboard.server.queue.common.TbProtoJsQueueMsg;
import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import javax.annotation.Nullable;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import java.util.Map;
import java.util.Optional;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Executor;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicInteger;
@Slf4j
@ConditionalOnExpression("'${js.evaluator:null}'=='remote' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core' || '${service.type:null}'=='tb-rule-engine')")
@Service
public class RemoteJsInvokeService extends AbstractJsInvokeService {
@Getter
@Value("${queue.js.max_eval_requests_timeout}")
private long maxEvalRequestsTimeout;
@Getter
@Value("${queue.js.max_requests_timeout}")
private long maxRequestsTimeout;
private long maxInvokeRequestsTimeout;
@Value("${queue.js.max_exec_requests_timeout:2000}")
private long maxExecRequestsTimeout;
@ -66,17 +65,14 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService {
@Value("${js.remote.max_errors}")
private int maxErrors;
@Getter
@Value("${js.remote.max_black_list_duration_sec:60}")
private int maxBlackListDurationSec;
@Getter
@Value("${js.remote.stats.enabled:false}")
private boolean statsEnabled;
private final AtomicInteger queuePushedMsgs = new AtomicInteger(0);
private final AtomicInteger queueInvokeMsgs = new AtomicInteger(0);
private final AtomicInteger queueEvalMsgs = new AtomicInteger(0);
private final AtomicInteger queueFailedMsgs = new AtomicInteger(0);
private final AtomicInteger queueTimeoutMsgs = new AtomicInteger(0);
private final ExecutorService callbackExecutor = Executors.newFixedThreadPool(
Runtime.getRuntime().availableProcessors(), ThingsBoardThreadFactory.forName("js-executor-remote-callback"));
@ -84,19 +80,19 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService {
super(apiUsageStateClient, apiUsageClient);
}
@Override
protected Executor getCallbackExecutor() {
return callbackExecutor;
}
@Override
protected String getStatsName() {
return "Queue JS Invoke Stats";
}
@Scheduled(fixedDelayString = "${js.remote.stats.print_interval_ms}")
public void printStats() {
if (statsEnabled) {
int pushedMsgs = queuePushedMsgs.getAndSet(0);
int invokeMsgs = queueInvokeMsgs.getAndSet(0);
int evalMsgs = queueEvalMsgs.getAndSet(0);
int failed = queueFailedMsgs.getAndSet(0);
int timedOut = queueTimeoutMsgs.getAndSet(0);
if (pushedMsgs > 0 || invokeMsgs > 0 || evalMsgs > 0 || failed > 0 || timedOut > 0) {
log.info("Queue JS Invoke Stats: pushed [{}] received [{}] invoke [{}] eval [{}] failed [{}] timedOut [{}]",
pushedMsgs, invokeMsgs + evalMsgs, invokeMsgs, evalMsgs, failed, timedOut);
}
}
super.printStats();
}
@Autowired
@ -106,7 +102,7 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService {
@PostConstruct
public void init() {
super.init(maxRequestsTimeout);
super.init();
requestTemplate.init();
}
@ -132,24 +128,6 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService {
log.trace("Post compile request for scriptId [{}]", scriptId);
ListenableFuture<TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse>> future = requestTemplate.send(new TbProtoJsQueueMsg<>(UUID.randomUUID(), jsRequestWrapper));
if (maxEvalRequestsTimeout > 0) {
future = Futures.withTimeout(future, maxEvalRequestsTimeout, TimeUnit.MILLISECONDS, timeoutExecutorService);
}
queuePushedMsgs.incrementAndGet();
Futures.addCallback(future, new FutureCallback<TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse>>() {
@Override
public void onSuccess(@Nullable TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse> result) {
queueEvalMsgs.incrementAndGet();
}
@Override
public void onFailure(Throwable t) {
if (t instanceof TimeoutException || (t.getCause() != null && t.getCause() instanceof TimeoutException)) {
queueTimeoutMsgs.incrementAndGet();
}
queueFailedMsgs.incrementAndGet();
}
}, callbackExecutor);
return Futures.transform(future, response -> {
JsInvokeProtos.JsCompileResponse compilationResult = response.getValue().getCompileResponse();
UUID compiledScriptId = new UUID(compilationResult.getScriptIdMSB(), compilationResult.getScriptIdLSB());
@ -166,7 +144,6 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService {
@Override
protected ListenableFuture<Object> doInvokeFunction(UUID scriptId, String functionName, Object[] args) {
log.trace("doInvokeFunction js-request for uuid {} with timeout {}ms", scriptId, maxRequestsTimeout);
final String scriptBody = scriptIdToBodysMap.get(scriptId);
if (scriptBody == null) {
return Futures.immediateFailedFuture(new RuntimeException("No script body found for scriptId: [" + scriptId + "]!"));
@ -186,31 +163,20 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService {
.setInvokeRequest(jsRequestBuilder.build())
.build();
StopWatch stopWatch = new StopWatch();
stopWatch.start();
StopWatch stopWatch;
if (log.isTraceEnabled()) {
stopWatch = new StopWatch();
stopWatch.start();
} else {
stopWatch = null;
}
ListenableFuture<TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse>> future = requestTemplate.send(new TbProtoJsQueueMsg<>(UUID.randomUUID(), jsRequestWrapper));
if (maxRequestsTimeout > 0) {
future = Futures.withTimeout(future, maxRequestsTimeout, TimeUnit.MILLISECONDS, timeoutExecutorService);
}
queuePushedMsgs.incrementAndGet();
Futures.addCallback(future, new FutureCallback<TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse>>() {
@Override
public void onSuccess(@Nullable TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse> result) {
queueInvokeMsgs.incrementAndGet();
}
@Override
public void onFailure(Throwable t) {
if (t instanceof TimeoutException || (t.getCause() != null && t.getCause() instanceof TimeoutException)) {
queueTimeoutMsgs.incrementAndGet();
}
queueFailedMsgs.incrementAndGet();
}
}, callbackExecutor);
return Futures.transform(future, response -> {
stopWatch.stop();
log.trace("doInvokeFunction js-response took {}ms for uuid {}", stopWatch.getTotalTimeMillis(), response.getKey());
if (log.isTraceEnabled()) {
stopWatch.stop();
log.trace("doInvokeFunction js-response took {}ms for uuid {}", stopWatch.getTotalTimeMillis(), response.getKey());
}
JsInvokeProtos.JsInvokeResponse invokeResult = response.getValue().getInvokeResponse();
if (invokeResult.getSuccess()) {
return invokeResult.getResult();
@ -218,11 +184,11 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService {
final RuntimeException e = new RuntimeException(invokeResult.getErrorDetails());
if (JsInvokeProtos.JsInvokeErrorCode.TIMEOUT_ERROR.equals(invokeResult.getErrorCode())) {
onScriptExecutionError(scriptId, e, scriptBody);
queueTimeoutMsgs.incrementAndGet();
timeoutMsgs.incrementAndGet();
} else if (JsInvokeProtos.JsInvokeErrorCode.COMPILATION_ERROR.equals(invokeResult.getErrorCode())) {
onScriptExecutionError(scriptId, e, scriptBody);
}
queueFailedMsgs.incrementAndGet();
failedMsgs.incrementAndGet();
log.debug("[{}] Failed to invoke function due to [{}]: {}", scriptId, invokeResult.getErrorCode().name(), invokeResult.getErrorDetails());
throw e;
}
@ -241,8 +207,8 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService {
.build();
ListenableFuture<TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse>> future = requestTemplate.send(new TbProtoJsQueueMsg<>(UUID.randomUUID(), jsRequestWrapper));
if (maxRequestsTimeout > 0) {
future = Futures.withTimeout(future, maxRequestsTimeout, TimeUnit.MILLISECONDS, timeoutExecutorService);
if (getMaxInvokeRequestsTimeout() > 0) {
future = Futures.withTimeout(future, getMaxInvokeRequestsTimeout(), TimeUnit.MILLISECONDS, timeoutExecutorService);
}
JsInvokeProtos.RemoteJsResponse response = future.get().getValue();
@ -255,9 +221,4 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService {
}
}
@Override
protected long getMaxBlacklistDuration() {
return TimeUnit.SECONDS.toMillis(maxBlackListDurationSec);
}
}

12
common/script/remote-js-client/src/main/java/org/thingsboard/server/service/script/RuleNodeJsScriptEngine.java
View File

@ -22,8 +22,8 @@ import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.MoreExecutors;
import lombok.extern.slf4j.Slf4j;
import org.thingsboard.script.api.JsInvokeService;
import org.thingsboard.script.api.JsScriptType;
import org.thingsboard.script.api.ScriptInvokeService;
import org.thingsboard.script.api.ScriptType;
import org.thingsboard.script.api.RuleNodeScriptFactory;
import org.thingsboard.server.common.data.StringUtils;
import org.thingsboard.server.common.data.id.CustomerId;
@ -47,18 +47,18 @@ import java.util.concurrent.ExecutionException;
public class RuleNodeJsScriptEngine implements org.thingsboard.rule.engine.api.ScriptEngine {
private static final ObjectMapper mapper = new ObjectMapper();
private final JsInvokeService sandboxService;
private final ScriptInvokeService sandboxService;
private final UUID scriptId;
private final TenantId tenantId;
private final EntityId entityId;
public RuleNodeJsScriptEngine(TenantId tenantId, JsInvokeService sandboxService, EntityId entityId, String script, String... argNames) {
public RuleNodeJsScriptEngine(TenantId tenantId, ScriptInvokeService sandboxService, EntityId entityId, String script, String... argNames) {
this.tenantId = tenantId;
this.sandboxService = sandboxService;
this.entityId = entityId;
try {
this.scriptId = this.sandboxService.eval(tenantId, JsScriptType.RULE_NODE_SCRIPT, script, argNames).get();
this.scriptId = this.sandboxService.eval(tenantId, ScriptType.RULE_NODE_SCRIPT, script, argNames).get();
} catch (Exception e) {
Throwable t = e;
if (e instanceof ExecutionException) {
@ -214,7 +214,7 @@ public class RuleNodeJsScriptEngine implements org.thingsboard.rule.engine.api.S
}
ListenableFuture<JsonNode> executeScriptAsync(CustomerId customerId, String... args) {
return Futures.transformAsync(sandboxService.invokeFunction(tenantId, customerId, this.scriptId, args),
return Futures.transformAsync(sandboxService.invokeScript(tenantId, customerId, this.scriptId, args),
o -> {
try {
return Futures.immediateFuture(mapper.readTree(o));

6
common/script/script-api/pom.xml
View File

@ -16,7 +16,7 @@
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.thingsboard.common</groupId>
@ -84,6 +84,10 @@
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
</dependency>
<dependency>
<groupId>org.mvel</groupId>
<artifactId>mvel2</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>

180
common/script/script-api/src/main/java/org/thingsboard/script/api/AbstractJsInvokeService.java → common/script/script-api/src/main/java/org/thingsboard/script/api/AbstractScriptInvokeService.java
View File

@ -15,12 +15,11 @@
*/
package org.thingsboard.script.api;
import com.google.common.util.concurrent.FutureCallback;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.MoreExecutors;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.thingsboard.common.util.ThingsBoardThreadFactory;
import org.thingsboard.server.common.data.ApiUsageRecordKey;
import org.thingsboard.server.common.data.id.CustomerId;
@ -32,42 +31,70 @@ import java.util.Map;
import java.util.Optional;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Executor;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import static java.lang.String.format;
/**
* Created by ashvayka on 26.09.18.
*/
@Slf4j
public abstract class AbstractJsInvokeService implements JsInvokeService {
public abstract class AbstractScriptInvokeService implements ScriptInvokeService {
protected Map<UUID, DisableListInfo> disabledScripts = new ConcurrentHashMap<>();
private final Optional<TbApiUsageStateClient> apiUsageStateClient;
private final Optional<TbApiUsageReportClient> apiUsageReportClient;
private final AtomicInteger pushedMsgs = new AtomicInteger(0);
private final AtomicInteger invokeMsgs = new AtomicInteger(0);
private final AtomicInteger evalMsgs = new AtomicInteger(0);
protected final AtomicInteger failedMsgs = new AtomicInteger(0);
protected final AtomicInteger timeoutMsgs = new AtomicInteger(0);
private final FutureCallback<UUID> evalCallback = new ScriptStatCallback<>(evalMsgs, timeoutMsgs, failedMsgs);
private final FutureCallback<Object> invokeCallback = new ScriptStatCallback<>(invokeMsgs, timeoutMsgs, failedMsgs);
protected ScheduledExecutorService timeoutExecutorService;
protected Map<UUID, String> scriptIdToNameMap = new ConcurrentHashMap<>();
protected Map<UUID, DisableListInfo> disabledFunctions = new ConcurrentHashMap<>();
@Getter
@Value("${js.max_total_args_size:100000}")
private long maxTotalArgsSize;
@Getter
@Value("${js.max_result_size:300000}")
private long maxResultSize;
@Getter
@Value("${js.max_script_body_size:50000}")
private long maxScriptBodySize;
protected AbstractJsInvokeService(Optional<TbApiUsageStateClient> apiUsageStateClient, Optional<TbApiUsageReportClient> apiUsageReportClient) {
protected AbstractScriptInvokeService(Optional<TbApiUsageStateClient> apiUsageStateClient, Optional<TbApiUsageReportClient> apiUsageReportClient) {
this.apiUsageStateClient = apiUsageStateClient;
this.apiUsageReportClient = apiUsageReportClient;
}
public void init(long maxRequestsTimeout) {
if (maxRequestsTimeout > 0) {
timeoutExecutorService = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("nashorn-js-timeout"));
protected long getMaxEvalRequestsTimeout() {
return getMaxInvokeRequestsTimeout();
}
protected abstract long getMaxInvokeRequestsTimeout();
protected abstract long getMaxScriptBodySize();
protected abstract long getMaxTotalArgsSize();
protected abstract long getMaxResultSize();
protected abstract int getMaxBlackListDurationSec();
protected abstract int getMaxErrors();
protected abstract boolean isStatsEnabled();
protected abstract String getStatsName();
protected abstract Executor getCallbackExecutor();
protected abstract boolean isScriptPresent(UUID scriptId);
protected abstract ListenableFuture<UUID> doEvalScript(ScriptType scriptType, String scriptBody, UUID scriptId, String[] argNames);
protected abstract ListenableFuture<Object> doInvokeFunction(UUID scriptId, Object[] args);
protected abstract void doRelease(UUID scriptId) throws Exception;
public void init() {
if (getMaxEvalRequestsTimeout() > 0 || getMaxInvokeRequestsTimeout() > 0) {
timeoutExecutorService = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("script-timeout"));
}
}
@ -77,26 +104,39 @@ public abstract class AbstractJsInvokeService implements JsInvokeService {
}
}
public void printStats() {
if (isStatsEnabled()) {
int pushed = pushedMsgs.getAndSet(0);
int invoked = invokeMsgs.getAndSet(0);
int evaluated = evalMsgs.getAndSet(0);
int failed = failedMsgs.getAndSet(0);
int timedOut = timeoutMsgs.getAndSet(0);
if (pushed > 0 || invoked > 0 || evaluated > 0 || failed > 0 || timedOut > 0) {
log.info("{}: pushed [{}] received [{}] invoke [{}] eval [{}] failed [{}] timedOut [{}]",
getStatsName(), pushed, invoked + evaluated, invoked, evaluated, failed, timedOut);
}
}
}
@Override
public ListenableFuture<UUID> eval(TenantId tenantId, JsScriptType scriptType, String scriptBody, String... argNames) {
public ListenableFuture<UUID> eval(TenantId tenantId, ScriptType scriptType, String scriptBody, String... argNames) {
if (!apiUsageStateClient.isPresent() || apiUsageStateClient.get().getApiUsageState(tenantId).isJsExecEnabled()) {
if (scriptBodySizeExceeded(scriptBody)) {
return error(format("Script body exceeds maximum allowed size of %s symbols", getMaxScriptBodySize()));
}
UUID scriptId = UUID.randomUUID();
String functionName = "invokeInternal_" + scriptId.toString().replace('-', '_');
String jsScript = generateJsScript(scriptType, functionName, scriptBody, argNames);
return doEval(scriptId, functionName, jsScript);
pushedMsgs.incrementAndGet();
return withTimeoutAndStatsCallback(doEvalScript(scriptType, scriptBody, scriptId, argNames), evalCallback, getMaxEvalRequestsTimeout());
} else {
return error("JS Execution is disabled due to API limits!");
return error("Script Execution is disabled due to API limits!");
}
}
@Override
public ListenableFuture<String> invokeFunction(TenantId tenantId, CustomerId customerId, UUID scriptId, Object... args) {
public ListenableFuture<String> invokeScript(TenantId tenantId, CustomerId customerId, UUID scriptId, Object... args) {
if (!apiUsageStateClient.isPresent() || apiUsageStateClient.get().getApiUsageState(tenantId).isJsExecEnabled()) {
String functionName = scriptIdToNameMap.get(scriptId);
if (functionName == null) {
if (!isScriptPresent(scriptId)) {
return error("No compiled script found for scriptId: [" + scriptId + "]!");
}
if (!isDisabled(scriptId)) {
@ -104,13 +144,17 @@ public abstract class AbstractJsInvokeService implements JsInvokeService {
return scriptExecutionError(scriptId, format("Script input arguments exceed maximum allowed total args size of %s symbols", getMaxTotalArgsSize()));
}
apiUsageReportClient.ifPresent(client -> client.report(tenantId, customerId, ApiUsageRecordKey.JS_EXEC_COUNT, 1));
return Futures.transformAsync(doInvokeFunction(scriptId, functionName, args), output -> {
pushedMsgs.incrementAndGet();
log.trace("invokeScript uuid {} with timeout {}ms", scriptId, getMaxInvokeRequestsTimeout());
var resultFuture = Futures.transformAsync(doInvokeFunction(scriptId, args), output -> {
String result = output.toString();
if (resultSizeExceeded(result)) {
return scriptExecutionError(scriptId, format("Script invocation result exceeds maximum allowed size of %s symbols", getMaxResultSize()));
}
return Futures.immediateFuture(result);
}, MoreExecutors.directExecutor());
return withTimeoutAndStatsCallback(resultFuture, invokeCallback, getMaxInvokeRequestsTimeout());
} else {
String message = "Script invocation is blocked due to maximum error count "
+ getMaxErrors() + ", scriptId " + scriptId + "!";
@ -122,14 +166,20 @@ public abstract class AbstractJsInvokeService implements JsInvokeService {
}
}
private <T extends V, V> ListenableFuture<T> withTimeoutAndStatsCallback(ListenableFuture<T> future, FutureCallback<V> statsCallback, long timeout) {
if (timeout > 0) {
future = Futures.withTimeout(future, timeout, TimeUnit.MILLISECONDS, timeoutExecutorService);
}
Futures.addCallback(future, statsCallback, getCallbackExecutor());
return future;
}
@Override
public ListenableFuture<Void> release(UUID scriptId) {
String functionName = scriptIdToNameMap.get(scriptId);
if (functionName != null) {
if (isScriptPresent(scriptId)) {
try {
scriptIdToNameMap.remove(scriptId);
disabledFunctions.remove(scriptId);
doRelease(scriptId, functionName);
disabledScripts.remove(scriptId);
doRelease(scriptId);
} catch (Exception e) {
return Futures.immediateFailedFuture(e);
}
@ -137,21 +187,18 @@ public abstract class AbstractJsInvokeService implements JsInvokeService {
return Futures.immediateFuture(null);
}
protected abstract ListenableFuture<UUID> doEval(UUID scriptId, String functionName, String scriptBody);
protected abstract ListenableFuture<Object> doInvokeFunction(UUID scriptId, String functionName, Object[] args);
protected abstract void doRelease(UUID scriptId, String functionName) throws Exception;
protected abstract int getMaxErrors();
protected abstract long getMaxBlacklistDuration();
protected void onScriptExecutionError(UUID scriptId, Throwable t, String scriptBody) {
DisableListInfo disableListInfo = disabledFunctions.computeIfAbsent(scriptId, key -> new DisableListInfo());
log.warn("Script has exception and will increment counter {} on disabledFunctions for id {}, exception {}, cause {}, scriptBody {}",
disableListInfo.get(), scriptId, t, t.getCause(), scriptBody);
disableListInfo.incrementAndGet();
private boolean isDisabled(UUID scriptId) {
DisableListInfo errorCount = disabledScripts.get(scriptId);
if (errorCount != null) {
if (errorCount.getExpirationTime() <= System.currentTimeMillis()) {
disabledScripts.remove(scriptId);
return false;
} else {
return errorCount.get() >= getMaxErrors();
}
} else {
return false;
}
}
private boolean scriptBodySizeExceeded(String scriptBody) {
@ -175,31 +222,17 @@ public abstract class AbstractJsInvokeService implements JsInvokeService {
return result.length() > getMaxResultSize();
}
private String generateJsScript(JsScriptType scriptType, String functionName, String scriptBody, String... argNames) {
if (scriptType == JsScriptType.RULE_NODE_SCRIPT) {
return RuleNodeScriptFactory.generateRuleNodeScript(functionName, scriptBody, argNames);
}
throw new RuntimeException("No script factory implemented for scriptType: " + scriptType);
}
private boolean isDisabled(UUID scriptId) {
DisableListInfo errorCount = disabledFunctions.get(scriptId);
if (errorCount != null) {
if (errorCount.getExpirationTime() <= System.currentTimeMillis()) {
disabledFunctions.remove(scriptId);
return false;
} else {
return errorCount.get() >= getMaxErrors();
}
} else {
return false;
}
}
private <T> ListenableFuture<T> error(String message) {
return Futures.immediateFailedFuture(new RuntimeException(message));
}
protected void onScriptExecutionError(UUID scriptId, Throwable t, String scriptBody) {
DisableListInfo disableListInfo = disabledScripts.computeIfAbsent(scriptId, key -> new DisableListInfo());
log.warn("Script has exception and will increment counter {} on disabledFunctions for id {}, exception {}, cause {}, scriptBody {}",
disableListInfo.get(), scriptId, t, t.getCause(), scriptBody);
disableListInfo.incrementAndGet();
}
private <T> ListenableFuture<T> scriptExecutionError(UUID scriptId, String errorMsg) {
RuntimeException error = new RuntimeException(errorMsg);
onScriptExecutionError(scriptId, error, null);
@ -220,7 +253,7 @@ public abstract class AbstractJsInvokeService implements JsInvokeService {
public int incrementAndGet() {
int result = counter.incrementAndGet();
expirationTime = System.currentTimeMillis() + getMaxBlacklistDuration();
expirationTime = System.currentTimeMillis() + TimeUnit.SECONDS.toMillis(getMaxBlackListDurationSec());
return result;
}
@ -228,4 +261,5 @@ public abstract class AbstractJsInvokeService implements JsInvokeService {
return expirationTime;
}
}
}

65
common/script/script-api/src/main/java/org/thingsboard/script/api/NashornJsInvokeService.java
View File

@ -1,65 +0,0 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.script.api;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Service;
import org.thingsboard.server.common.stats.TbApiUsageReportClient;
import org.thingsboard.server.common.stats.TbApiUsageStateClient;
import java.util.Optional;
import java.util.concurrent.TimeUnit;
@Slf4j
@ConditionalOnProperty(prefix = "js", value = "evaluator", havingValue = "local", matchIfMissing = true)
@Service
public class NashornJsInvokeService extends AbstractNashornJsInvokeService {
@Value("${js.local.use_js_sandbox}")
private boolean useJsSandbox;
@Getter
@Value("${js.local.monitor_thread_pool_size}")
private int monitorThreadPoolSize;
@Getter
@Value("${js.local.max_cpu_time}")
private long maxCpuTime;
@Getter
@Value("${js.local.max_errors}")
private int maxErrors;
@Value("${js.local.max_black_list_duration_sec:60}")
private int maxBlackListDurationSec;
public NashornJsInvokeService(Optional<TbApiUsageStateClient> apiUsageStateClient, Optional<TbApiUsageReportClient> apiUsageReportClient) {
super(apiUsageStateClient, apiUsageReportClient);
}
@Override
protected boolean useJsSandbox() {
return useJsSandbox;
}
@Override
protected long getMaxBlacklistDuration() {
return TimeUnit.SECONDS.toMillis(maxBlackListDurationSec);
}
}

6
common/script/script-api/src/main/java/org/thingsboard/script/api/JsInvokeService.java → common/script/script-api/src/main/java/org/thingsboard/script/api/ScriptInvokeService.java
View File

@ -21,11 +21,11 @@ import org.thingsboard.server.common.data.id.TenantId;
import java.util.UUID;
public interface JsInvokeService {
public interface ScriptInvokeService {
ListenableFuture<UUID> eval(TenantId tenantId, JsScriptType scriptType, String scriptBody, String... argNames);
ListenableFuture<UUID> eval(TenantId tenantId, ScriptType scriptType, String scriptBody, String... argNames);
ListenableFuture<String> invokeFunction(TenantId tenantId, CustomerId customerId, UUID scriptId, Object... args);
ListenableFuture<String> invokeScript(TenantId tenantId, CustomerId customerId, UUID scriptId, Object... args);
ListenableFuture<Void> release(UUID scriptId);

15
common/script/script-api/src/main/java/org/thingsboard/script/api/JsStatCallback.java → common/script/script-api/src/main/java/org/thingsboard/script/api/ScriptStatCallback.java
View File

@ -23,24 +23,23 @@ import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicInteger;
@AllArgsConstructor
public class JsStatCallback<T> implements FutureCallback<T> {
private final AtomicInteger jsSuccessMsgs;
private final AtomicInteger jsTimeoutMsgs;
private final AtomicInteger jsFailedMsgs;
public class ScriptStatCallback<T> implements FutureCallback<T> {
private final AtomicInteger successMsgs;
private final AtomicInteger timeoutMsgs;
private final AtomicInteger failedMsgs;
@Override
public void onSuccess(@Nullable T result) {
jsSuccessMsgs.incrementAndGet();
successMsgs.incrementAndGet();
}
@Override
public void onFailure(Throwable t) {
if (t instanceof TimeoutException || (t.getCause() != null && t.getCause() instanceof TimeoutException)) {
jsTimeoutMsgs.incrementAndGet();
timeoutMsgs.incrementAndGet();
} else {
jsFailedMsgs.incrementAndGet();
failedMsgs.incrementAndGet();
}
}
}

2
common/script/script-api/src/main/java/org/thingsboard/script/api/JsScriptType.java → common/script/script-api/src/main/java/org/thingsboard/script/api/ScriptType.java
View File

@ -15,6 +15,6 @@
*/
package org.thingsboard.script.api;
public enum JsScriptType {
public enum ScriptType {
RULE_NODE_SCRIPT
}

107
common/script/script-api/src/main/java/org/thingsboard/script/api/js/AbstractJsInvokeService.java Normal file
View File

@ -0,0 +1,107 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.script.api.js;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.thingsboard.script.api.AbstractScriptInvokeService;
import org.thingsboard.script.api.RuleNodeScriptFactory;
import org.thingsboard.script.api.ScriptType;
import org.thingsboard.server.common.stats.TbApiUsageReportClient;
import org.thingsboard.server.common.stats.TbApiUsageStateClient;
import java.util.Map;
import java.util.Optional;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
/**
* Created by ashvayka on 26.09.18.
*/
@Slf4j
public abstract class AbstractJsInvokeService extends AbstractScriptInvokeService {
protected Map<UUID, String> scriptIdToNameMap = new ConcurrentHashMap<>();
@Getter
@Value("${js.max_total_args_size:100000}")
private long maxTotalArgsSize;
@Getter
@Value("${js.max_result_size:300000}")
private long maxResultSize;
@Getter
@Value("${js.max_script_body_size:50000}")
private long maxScriptBodySize;
protected AbstractJsInvokeService(Optional<TbApiUsageStateClient> apiUsageStateClient, Optional<TbApiUsageReportClient> apiUsageReportClient) {
super(apiUsageStateClient, apiUsageReportClient);
}
@Override
protected boolean isScriptPresent(UUID scriptId) {
return scriptIdToNameMap.containsKey(scriptId);
}
@Override
public ListenableFuture<Void> release(UUID scriptId) {
String functionName = scriptIdToNameMap.get(scriptId);
if (functionName != null) {
try {
scriptIdToNameMap.remove(scriptId);
disabledScripts.remove(scriptId);
doRelease(scriptId, functionName);
} catch (Exception e) {
return Futures.immediateFailedFuture(e);
}
}
return Futures.immediateFuture(null);
}
@Override
protected ListenableFuture<Object> doInvokeFunction(UUID scriptId, Object[] args) {
return doInvokeFunction(scriptId, scriptIdToNameMap.get(scriptId), args);
}
@Override
protected ListenableFuture<UUID> doEvalScript(ScriptType scriptType, String scriptBody, UUID scriptId, String[] argNames) {
String functionName = "invokeInternal_" + scriptId.toString().replace('-', '_');
String jsScript = generateJsScript(scriptType, functionName, scriptBody, argNames);
return doEval(scriptId, functionName, jsScript);
}
@Override
protected void doRelease(UUID scriptId) throws Exception {
String functionName = scriptIdToNameMap.remove(scriptId);
doRelease(scriptId, functionName);
}
protected abstract ListenableFuture<UUID> doEval(UUID scriptId, String functionName, String scriptBody);
protected abstract ListenableFuture<Object> doInvokeFunction(UUID scriptId, String functionName, Object[] args);
protected abstract void doRelease(UUID scriptId, String functionName) throws Exception;
private String generateJsScript(ScriptType scriptType, String functionName, String scriptBody, String... argNames) {
if (scriptType == ScriptType.RULE_NODE_SCRIPT) {
return RuleNodeScriptFactory.generateRuleNodeScript(functionName, scriptBody, argNames);
}
throw new RuntimeException("No script factory implemented for scriptType: " + scriptType);
}
}

105
common/script/script-api/src/main/java/org/thingsboard/script/api/AbstractNashornJsInvokeService.java → common/script/script-api/src/main/java/org/thingsboard/script/api/js/NashornJsInvokeService.java
View File

@ -13,10 +13,8 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.script.api;
package org.thingsboard.script.api.js;
import com.google.common.util.concurrent.FutureCallback;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
@ -25,7 +23,9 @@ import delight.nashornsandbox.NashornSandboxes;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import org.thingsboard.common.util.ThingsBoardExecutors;
import org.thingsboard.server.common.stats.TbApiUsageReportClient;
import org.thingsboard.server.common.stats.TbApiUsageStateClient;
@ -39,67 +39,79 @@ import javax.script.ScriptException;
import java.util.Optional;
import java.util.UUID;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Executor;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.ReentrantLock;
@Slf4j
public abstract class AbstractNashornJsInvokeService extends AbstractJsInvokeService {
@ConditionalOnProperty(prefix = "js", value = "evaluator", havingValue = "local", matchIfMissing = true)
@Service
public class NashornJsInvokeService extends AbstractJsInvokeService {
private NashornSandbox sandbox;
private ScriptEngine engine;
private ExecutorService monitorExecutorService;
private ListeningExecutorService jsExecutor;
private final AtomicInteger jsPushedMsgs = new AtomicInteger(0);
private final AtomicInteger jsInvokeMsgs = new AtomicInteger(0);
private final AtomicInteger jsEvalMsgs = new AtomicInteger(0);
private final AtomicInteger jsFailedMsgs = new AtomicInteger(0);
private final AtomicInteger jsTimeoutMsgs = new AtomicInteger(0);
private final FutureCallback<UUID> evalCallback = new JsStatCallback<>(jsEvalMsgs, jsTimeoutMsgs, jsFailedMsgs);
private final FutureCallback<Object> invokeCallback = new JsStatCallback<>(jsInvokeMsgs, jsTimeoutMsgs, jsFailedMsgs);
private final ReentrantLock evalLock = new ReentrantLock();
@Value("${js.local.max_requests_timeout:0}")
private long maxRequestsTimeout;
@Value("${js.local.use_js_sandbox}")
private boolean useJsSandbox;
@Value("${js.local.monitor_thread_pool_size}")
private int monitorThreadPoolSize;
@Value("${js.local.max_cpu_time}")
private long maxCpuTime;
@Getter
@Value("${js.local.max_errors}")
private int maxErrors;
@Getter
@Value("${js.local.max_black_list_duration_sec:60}")
private int maxBlackListDurationSec;
@Getter
@Value("${js.local.max_requests_timeout:0}")
private long maxInvokeRequestsTimeout;
@Getter
@Value("${js.local.stats.enabled:false}")
private boolean statsEnabled;
@Value("${js.local.js_thread_pool_size:50}")
private int jsExecutorThreadPoolSize;
public AbstractNashornJsInvokeService(Optional<TbApiUsageStateClient> apiUsageStateClient, Optional<TbApiUsageReportClient> apiUsageReportClient) {
public NashornJsInvokeService(Optional<TbApiUsageStateClient> apiUsageStateClient, Optional<TbApiUsageReportClient> apiUsageReportClient) {
super(apiUsageStateClient, apiUsageReportClient);
}
@Override
protected String getStatsName() {
return "Nashorn JS Invoke Stats";
}
@Override
protected Executor getCallbackExecutor() {
return MoreExecutors.directExecutor();
}
@Scheduled(fixedDelayString = "${js.local.stats.print_interval_ms:10000}")
public void printStats() {
if (statsEnabled) {
int pushedMsgs = jsPushedMsgs.getAndSet(0);
int invokeMsgs = jsInvokeMsgs.getAndSet(0);
int evalMsgs = jsEvalMsgs.getAndSet(0);
int failed = jsFailedMsgs.getAndSet(0);
int timedOut = jsTimeoutMsgs.getAndSet(0);
if (pushedMsgs > 0 || invokeMsgs > 0 || evalMsgs > 0 || failed > 0 || timedOut > 0) {
log.info("Nashorn JS Invoke Stats: pushed [{}] received [{}] invoke [{}] eval [{}] failed [{}] timedOut [{}]",
pushedMsgs, invokeMsgs + evalMsgs, invokeMsgs, evalMsgs, failed, timedOut);
}
}
super.printStats();
}
@PostConstruct
public void init() {
super.init(maxRequestsTimeout);
super.init();
jsExecutor = MoreExecutors.listeningDecorator(Executors.newWorkStealingPool(jsExecutorThreadPoolSize));
if (useJsSandbox()) {
if (useJsSandbox) {
sandbox = NashornSandboxes.create();
monitorExecutorService = ThingsBoardExecutors.newWorkStealingPool(getMonitorThreadPoolSize(), "nashorn-js-monitor");
monitorExecutorService = ThingsBoardExecutors.newWorkStealingPool(monitorThreadPoolSize, "nashorn-js-monitor");
sandbox.setExecutor(monitorExecutorService);
sandbox.setMaxCPUTime(getMaxCpuTime());
sandbox.setMaxCPUTime(maxCpuTime);
sandbox.allowNoBraces(false);
sandbox.allowLoadFunctions(true);
sandbox.setMaxPreparedStatements(30);
@ -117,20 +129,13 @@ public abstract class AbstractNashornJsInvokeService extends AbstractJsInvokeSer
}
}
protected abstract boolean useJsSandbox();
protected abstract int getMonitorThreadPoolSize();
protected abstract long getMaxCpuTime();
@Override
protected ListenableFuture<UUID> doEval(UUID scriptId, String functionName, String jsScript) {
jsPushedMsgs.incrementAndGet();
ListenableFuture<UUID> result = jsExecutor.submit(() -> {
return jsExecutor.submit(() -> {
try {
evalLock.lock();
try {
if (useJsSandbox()) {
if (useJsSandbox) {
sandbox.eval(jsScript);
} else {
engine.eval(jsScript);
@ -145,19 +150,13 @@ public abstract class AbstractNashornJsInvokeService extends AbstractJsInvokeSer
throw new ExecutionException(e);
}
});
if (maxRequestsTimeout > 0) {
result = Futures.withTimeout(result, maxRequestsTimeout, TimeUnit.MILLISECONDS, timeoutExecutorService);
}
Futures.addCallback(result, evalCallback, MoreExecutors.directExecutor());
return result;
}
@Override
protected ListenableFuture<Object> doInvokeFunction(UUID scriptId, String functionName, Object[] args) {
jsPushedMsgs.incrementAndGet();
ListenableFuture<Object> result = jsExecutor.submit(() -> {
return jsExecutor.submit(() -> {
try {
if (useJsSandbox()) {
if (useJsSandbox) {
return sandbox.getSandboxedInvocable().invokeFunction(functionName, args);
} else {
return ((Invocable) engine).invokeFunction(functionName, args);
@ -169,16 +168,10 @@ public abstract class AbstractNashornJsInvokeService extends AbstractJsInvokeSer
throw new ExecutionException(e);
}
});
if (maxRequestsTimeout > 0) {
result = Futures.withTimeout(result, maxRequestsTimeout, TimeUnit.MILLISECONDS, timeoutExecutorService);
}
Futures.addCallback(result, invokeCallback, MoreExecutors.directExecutor());
return result;
}
protected void doRelease(UUID scriptId, String functionName) throws ScriptException {
if (useJsSandbox()) {
if (useJsSandbox) {
sandbox.eval(functionName + " = undefined;");
} else {
engine.eval(functionName + " = undefined;");

152
common/script/script-api/src/main/java/org/thingsboard/script/api/mvel/MvelInvokeService.java Normal file
View File

@ -0,0 +1,152 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.script.api.mvel;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.mvel2.MVEL;
import org.mvel2.ParserContext;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.scheduling.annotation.Scheduled;
import org.thingsboard.common.util.ThingsBoardExecutors;
import org.thingsboard.script.api.AbstractScriptInvokeService;
import org.thingsboard.script.api.ScriptType;
import org.thingsboard.server.common.stats.TbApiUsageReportClient;
import org.thingsboard.server.common.stats.TbApiUsageStateClient;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import java.io.Serializable;
import java.util.HashMap;
import java.util.Map;
import java.util.Optional;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Executor;
@Slf4j
public class MvelInvokeService extends AbstractScriptInvokeService {
protected Map<UUID, MvelScript> scriptMap = new ConcurrentHashMap<>();
private ParserContext parserContext;
@Getter
@Value("${mvel.max_total_args_size:100000}")
private long maxTotalArgsSize;
@Getter
@Value("${mvel.max_result_size:300000}")
private long maxResultSize;
@Getter
@Value("${mvel.max_script_body_size:50000}")
private long maxScriptBodySize;
@Getter
@Value("${mvel.max_errors:3}")
private int maxErrors;
@Getter
@Value("${mvel.max_black_list_duration_sec:60}")
private int maxBlackListDurationSec;
@Getter
@Value("${mvel.max_requests_timeout:0}")
private long maxInvokeRequestsTimeout;
@Getter
@Value("${mvel.stats.enabled:false}")
private boolean statsEnabled;
private ListeningExecutorService executor;
protected MvelInvokeService(Optional<TbApiUsageStateClient> apiUsageStateClient, Optional<TbApiUsageReportClient> apiUsageReportClient) {
super(apiUsageStateClient, apiUsageReportClient);
}
@Scheduled(fixedDelayString = "${mvel.stats.print_interval_ms:10000}")
public void printStats() {
super.printStats();
}
@PostConstruct
public void init() {
super.init();
parserContext = new ParserContext(new TbMvelParserConfiguration());
executor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool(2, "mvel-executor"));
}
@PreDestroy
public void destroy() {
if (executor != null) {
executor.shutdownNow();
}
}
@Override
protected String getStatsName() {
return "MVEL Scripts Stats";
}
@Override
protected Executor getCallbackExecutor() {
return MoreExecutors.directExecutor();
}
@Override
protected boolean isScriptPresent(UUID scriptId) {
return scriptMap.containsKey(scriptId);
}
@Override
protected ListenableFuture<UUID> doEvalScript(ScriptType scriptType, String scriptBody, UUID scriptId, String[] argNames) {
//TODO: executor, check expression for "new" and ?
return executor.submit(() -> {
try {
Serializable compiledScript = MVEL.compileExpression(scriptBody, parserContext);
MvelScript script = new MvelScript(compiledScript, argNames);
scriptMap.put(scriptId, script);
return scriptId;
} catch (Exception e) {
log.debug("Failed to compile MVEL script: {}", scriptBody, e);
throw new ExecutionException(e);
}
});
}
@Override
protected ListenableFuture<Object> doInvokeFunction(UUID scriptId, Object[] args) {
return executor.submit(() -> {
MvelScript script = scriptMap.get(scriptId);
if (script == null) {
throw new RuntimeException("Script not found!");
}
try {
return MVEL.executeExpression(script.getCompiledScript(), script.createVars(args));
} catch (OutOfMemoryError e) {
Runtime.getRuntime().gc();
throw new RuntimeException("Memory error!");
}
});
}
@Override
protected void doRelease(UUID scriptId) throws Exception {
scriptMap.remove(scriptId);
}
}

40
common/script/script-api/src/main/java/org/thingsboard/script/api/mvel/MvelScript.java Normal file
View File

@ -0,0 +1,40 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.script.api.mvel;
import lombok.Data;
import java.io.Serializable;
import java.util.HashMap;
import java.util.Map;
@Data
public class MvelScript {
private final Serializable compiledScript;
private final String[] argNames;
public Map createVars(Object[] args) {
if (args == null || args.length != argNames.length) {
throw new IllegalArgumentException("Invalid number of argument values");
}
var result = new HashMap<>();
for (int i = 0; i < argNames.length; i++) {
result.put(argNames[i], args[i]);
}
return result;
}
}

90
common/script/script-api/src/main/java/org/thingsboard/script/api/mvel/TbMvelClassLoader.java Normal file
View File

@ -0,0 +1,90 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.script.api.mvel;
import org.mvel2.compiler.AbstractParser;
import java.net.URL;
import java.net.URLClassLoader;
import java.util.HashSet;
import java.util.Set;
public class TbMvelClassLoader extends URLClassLoader {
private static final Set<String> allowedClasses = new HashSet<>();
private static final Set<String> allowedPackages = new HashSet<>();
static {
AbstractParser.LITERALS.remove("System");
AbstractParser.LITERALS.remove("Runtime");
AbstractParser.LITERALS.remove("Class");
AbstractParser.LITERALS.remove("ClassLoader");
AbstractParser.LITERALS.remove("Thread");
AbstractParser.LITERALS.remove("Compiler");
AbstractParser.LITERALS.remove("ThreadLocal");
AbstractParser.LITERALS.remove("SecurityManager");
AbstractParser.CLASS_LITERALS.remove("System");
AbstractParser.CLASS_LITERALS.remove("Runtime");
AbstractParser.CLASS_LITERALS.remove("Class");
AbstractParser.CLASS_LITERALS.remove("ClassLoader");
AbstractParser.CLASS_LITERALS.remove("Thread");
AbstractParser.CLASS_LITERALS.remove("Compiler");
AbstractParser.CLASS_LITERALS.remove("ThreadLocal");
AbstractParser.CLASS_LITERALS.remove("SecurityManager");
AbstractParser.CLASS_LITERALS.values().forEach(val -> allowedClasses.add(((Class) val).getName()));
}
static {
allowedPackages.add("org.mvel2");
allowedPackages.add("java.util");
}
public TbMvelClassLoader() {
super(new URL[0], Thread.currentThread().getContextClassLoader());
}
@Override
protected Class<?> loadClass(String name, boolean resolve) throws ClassNotFoundException {
if (!classNameAllowed(name)) {
throw new ClassNotFoundException();
}
return super.loadClass(name, resolve);
}
@Override
public Class<?> loadClass(String name) throws ClassNotFoundException {
if (!classNameAllowed(name)) {
throw new ClassNotFoundException();
}
return super.loadClass(name);
}
private boolean classNameAllowed(String name) {
if (allowedClasses.contains(name)) {
return true;
}
for (String pkgName : allowedPackages) {
if (name.startsWith(pkgName)) {
return true;
}
}
return false;
}
}

37
common/script/script-api/src/main/java/org/thingsboard/script/api/mvel/TbMvelParserConfiguration.java Normal file
View File

@ -0,0 +1,37 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.script.api.mvel;
import org.mvel2.ParserConfiguration;
import org.mvel2.integration.VariableResolverFactory;
public class TbMvelParserConfiguration extends ParserConfiguration {
private static final long serialVersionUID = 5558151976348875590L;
TbMvelParserConfiguration() {
setClassLoader(new TbMvelClassLoader());
}
@Override
public VariableResolverFactory getVariableFactory(VariableResolverFactory factory) {
if (Thread.interrupted()) {
throw new RuntimeException("Thread is interrupted!");
}
return new TbMvelResolverFactory(factory);
}
}

35
common/script/script-api/src/main/java/org/thingsboard/script/api/mvel/TbMvelResolverFactory.java Normal file
View File

@ -0,0 +1,35 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.script.api.mvel;
import org.mvel2.integration.VariableResolver;
import org.mvel2.integration.VariableResolverFactory;
import org.mvel2.integration.impl.StackResetResolverFactory;
public class TbMvelResolverFactory extends StackResetResolverFactory {
public TbMvelResolverFactory(VariableResolverFactory delegate) {
super(delegate);
}
@Override
public VariableResolver getVariableResolver(String name) {
if (Thread.interrupted()) {
throw new RuntimeException("Thread is interrupted!");
}
return super.getVariableResolver(name);
}
}

6
pom.xml
View File

@ -77,6 +77,7 @@
<zookeeper.version>3.5.5</zookeeper.version>
<protobuf.version>3.17.2</protobuf.version>
<grpc.version>1.42.1</grpc.version>
<mvel.version>2.4.14.Final</mvel.version>
<lombok.version>1.18.18</lombok.version>
<paho.client.version>1.2.4</paho.client.version>
<netty.version>4.1.75.Final</netty.version>
@ -1573,6 +1574,11 @@
<artifactId>grpc-api</artifactId>
<version>${grpc.version}</version>
</dependency>
<dependency>
<groupId>org.mvel</groupId>
<artifactId>mvel2</artifactId>
<version>${mvel.version}</version>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-test</artifactId>