From 9553a6958b313ae85a463609e25d88c6232e963c Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Wed, 29 Nov 2023 16:46:35 +0100 Subject: [PATCH 1/6] fixed persistent RPC for microservice deployment --- .../java/org/thingsboard/server/service/queue/ProtoUtils.java | 1 + 1 file changed, 1 insertion(+) diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ProtoUtils.java b/application/src/main/java/org/thingsboard/server/service/queue/ProtoUtils.java index 36038568bc..b61bd9b67e 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/ProtoUtils.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/ProtoUtils.java @@ -308,6 +308,7 @@ public class ProtoUtils { .setRequestIdMSB(msg.getMsg().getId().getMostSignificantBits()) .setRequestIdLSB(msg.getMsg().getId().getLeastSignificantBits()) .setOneway(msg.getMsg().isOneway()) + .setPersisted(msg.getMsg().isPersisted()) .build(); return TransportProtos.ToDeviceRpcRequestActorMsgProto.newBuilder() From 07721da41d620b8e07fecb5d01285d11fc9d06ca Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Wed, 29 Nov 2023 18:06:01 +0100 Subject: [PATCH 2/6] added blackbox test for persisted RPC --- .../server/msa/TestRestClient.java | 24 ++++++++ .../msa/connectivity/MqttClientTest.java | 59 +++++++++++++++++++ 2 files changed, 83 insertions(+) diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestRestClient.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestRestClient.java index f04331e01e..3d5e71ae2e 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestRestClient.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestRestClient.java @@ -44,12 +44,14 @@ import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityViewId; +import org.thingsboard.server.common.data.id.RpcId; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.RelationTypeGroup; +import org.thingsboard.server.common.data.rpc.Rpc; import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleChainMetaData; import org.thingsboard.server.common.data.security.DeviceCredentials; @@ -57,6 +59,7 @@ import org.thingsboard.server.common.data.security.DeviceCredentials; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.UUID; import static io.restassured.RestAssured.given; import static java.net.HttpURLConnection.HTTP_NOT_FOUND; @@ -302,6 +305,27 @@ public class TestRestClient { .as(JsonNode.class); } + public Rpc getPersistedRpc(RpcId rpcId) { + return given().spec(requestSpec) + .get("/api/rpc/persistent/{rpcId}", rpcId.toString()) + .then() + .statusCode(HTTP_OK) + .extract() + .as(Rpc.class); + } + + public PageData getPersistedRpcByDevice(DeviceId deviceId, PageLink pageLink) { + Map params = new HashMap<>(); + addPageLinkToParam(params, pageLink); + return given().spec(requestSpec).queryParams(params) + .get("/api/rpc/persistent/device/{deviceId}", deviceId.toString()) + .then() + .statusCode(HTTP_OK) + .extract() + .as(new TypeRef<>() { + }); + } + public PageData getDeviceProfiles(PageLink pageLink) { Map params = new HashMap<>(); addPageLinkToParam(params, pageLink); diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/MqttClientTest.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/MqttClientTest.java index 5a7e9b631d..75414477ab 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/MqttClientTest.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/MqttClientTest.java @@ -26,6 +26,7 @@ import io.netty.buffer.Unpooled; import io.netty.handler.codec.mqtt.MqttQoS; import lombok.Data; import lombok.extern.slf4j.Slf4j; +import org.awaitility.Awaitility; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @@ -39,9 +40,11 @@ import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.DeviceProfileProvisionType; import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.id.RpcId; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.common.data.rpc.Rpc; import org.thingsboard.server.common.data.rule.NodeConnectionInfo; import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleChainMetaData; @@ -59,6 +62,7 @@ import java.util.Arrays; import java.util.Objects; import java.util.Optional; import java.util.Random; +import java.util.UUID; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ExecutionException; @@ -66,6 +70,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import static org.assertj.core.api.Assertions.assertThat; +import static org.testng.Assert.assertNotNull; import static org.testng.Assert.fail; import static org.thingsboard.server.common.data.DataConstants.DEVICE; import static org.thingsboard.server.common.data.DataConstants.SHARED_SCOPE; @@ -293,6 +298,60 @@ public class MqttClientTest extends AbstractContainerTest { assertThat(serverResponse).isEqualTo(mapper.readTree(clientResponse.toString())); } + @Test + public void serverSidePersistedRpc() throws Exception { + DeviceCredentials deviceCredentials = testRestClient.getDeviceCredentialsByDeviceId(device.getId()); + + MqttMessageListener listener = new MqttMessageListener(); + MqttClient mqttClient = getMqttClient(deviceCredentials, listener); + mqttClient.on("v1/devices/me/rpc/request/+", listener, MqttQoS.AT_LEAST_ONCE).get(); + + // Wait until subscription is processed + TimeUnit.SECONDS.sleep(3 * timeoutMultiplier); + + // Send an RPC from the server + JsonObject serverRpcPayload = new JsonObject(); + serverRpcPayload.addProperty("method", "getValue"); + serverRpcPayload.addProperty("params", true); + serverRpcPayload.addProperty("persistent", true); + + JsonNode persistentRpcId = testRestClient.postServerSideRpc(device.getId(), mapper.readTree(serverRpcPayload.toString())); + + assertNotNull(persistentRpcId); + + RpcId rpcId = new RpcId(UUID.fromString(persistentRpcId.get("rpcId").asText())); + + // Wait for RPC call from the server and send the response + MqttEvent requestFromServer = listener.getEvents().poll(10 * timeoutMultiplier, TimeUnit.SECONDS); + + assertThat(Objects.requireNonNull(requestFromServer).getMessage()).isEqualTo("{\"method\":\"getValue\",\"params\":true}"); + + Integer requestId = Integer.valueOf(Objects.requireNonNull(requestFromServer).getTopic().substring("v1/devices/me/rpc/request/".length())); + JsonObject clientResponse = new JsonObject(); + clientResponse.addProperty("response", "someResponse"); + // Send a response to the server's RPC request + mqttClient.publish("v1/devices/me/rpc/response/" + requestId, Unpooled.wrappedBuffer(clientResponse.toString().getBytes())).get(); + + PageLink pageLink = new PageLink(10); + + Awaitility.await() + .pollInterval(500, TimeUnit.MILLISECONDS) + .atMost(5 * timeoutMultiplier, TimeUnit.SECONDS) + .until(() -> { + PageData rpcByDevice = testRestClient.getPersistedRpcByDevice(device.getId(), pageLink); + for (Rpc rpc : rpcByDevice.getData()) { + if (rpc.getId().equals(rpcId)) { + return true; + } + } + return false; + }); + + Rpc persistentRpc = testRestClient.getPersistedRpc(rpcId); + + assertThat(persistentRpc.getResponse()).isEqualTo(mapper.readTree(clientResponse.toString())); + } + @Test public void clientSideRpc() throws Exception { DeviceCredentials deviceCredentials = testRestClient.getDeviceCredentialsByDeviceId(device.getId()); From 3477cc1b79cdf864b845d6ba340f99efcbaf39de Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Thu, 30 Nov 2023 11:23:14 +0100 Subject: [PATCH 3/6] fixed possible NPE --- .../src/main/java/org/thingsboard/common/util/SslUtil.java | 6 +++++- .../rule/engine/credentials/CertPemCredentials.java | 6 +++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/common/util/src/main/java/org/thingsboard/common/util/SslUtil.java b/common/util/src/main/java/org/thingsboard/common/util/SslUtil.java index 889436671a..32ab61a9a1 100644 --- a/common/util/src/main/java/org/thingsboard/common/util/SslUtil.java +++ b/common/util/src/main/java/org/thingsboard/common/util/SslUtil.java @@ -73,7 +73,7 @@ public class SslUtil { @SneakyThrows public static PrivateKey readPrivateKey(String fileContent, String passStr) { - char[] password = StringUtils.isEmpty(passStr) ? EMPTY_PASS : passStr.toCharArray(); + char[] password = getPassword(passStr); PrivateKey privateKey = null; JcaPEMKeyConverter keyConverter = new JcaPEMKeyConverter(); @@ -102,4 +102,8 @@ public class SslUtil { return privateKey; } + public static char[] getPassword(String passStr) { + return StringUtils.isEmpty(passStr) ? EMPTY_PASS : passStr.toCharArray(); + } + } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/credentials/CertPemCredentials.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/credentials/CertPemCredentials.java index 3824e71459..2c545f80a7 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/credentials/CertPemCredentials.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/credentials/CertPemCredentials.java @@ -47,7 +47,7 @@ public class CertPemCredentials implements ClientCredentials { protected String caCert; private String cert; private String privateKey; - private String password = ""; + private String password; @Override public CredentialsType getType() { @@ -87,7 +87,7 @@ public class CertPemCredentials implements ClientCredentials { private KeyManagerFactory createAndInitKeyManagerFactory() throws Exception { KeyManagerFactory kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); - kmf.init(loadKeyStore(), password.toCharArray()); + kmf.init(loadKeyStore(), SslUtil.getPassword(password)); return kmf; } @@ -107,7 +107,7 @@ public class CertPemCredentials implements ClientCredentials { CertPath certPath = factory.generateCertPath(certificates); List path = certPath.getCertificates(); Certificate[] x509Certificates = path.toArray(new Certificate[0]); - keyStore.setKeyEntry(PRIVATE_KEY_ALIAS, privateKey, password.toCharArray(), x509Certificates); + keyStore.setKeyEntry(PRIVATE_KEY_ALIAS, privateKey, SslUtil.getPassword(password), x509Certificates); } return keyStore; } From 37a8d78e0161c7cdd5a23975d118875269231ef8 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Thu, 30 Nov 2023 11:44:04 +0100 Subject: [PATCH 4/6] tests improvements --- .../rule/engine/credentials/CertPemCredentialsTest.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/credentials/CertPemCredentialsTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/credentials/CertPemCredentialsTest.java index 2cecd1490d..3c7d8a30c4 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/credentials/CertPemCredentialsTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/credentials/CertPemCredentialsTest.java @@ -39,7 +39,6 @@ import static org.thingsboard.rule.engine.credentials.CertPemCredentials.PRIVATE public class CertPemCredentialsTest { private static final String PASS = "test"; - private static final String EMPTY_PASS = ""; private static final String RSA = "RSA"; private static final String EC = "EC"; @@ -82,10 +81,10 @@ public class CertPemCredentialsTest { private static Stream testLoadKeyStore() { return Stream.of( - Arguments.of("pem/rsa_cert.pem", "pem/rsa_key.pem", EMPTY_PASS, RSA), + Arguments.of("pem/rsa_cert.pem", "pem/rsa_key.pem", null, RSA), Arguments.of("pem/rsa_encrypted_cert.pem", "pem/rsa_encrypted_key.pem", PASS, RSA), Arguments.of("pem/rsa_encrypted_traditional_cert.pem", "pem/rsa_encrypted_traditional_key.pem", PASS, RSA), - Arguments.of("pem/ec_cert.pem", "pem/ec_key.pem", EMPTY_PASS, EC) + Arguments.of("pem/ec_cert.pem", "pem/ec_key.pem", null, EC) ); } @@ -99,7 +98,7 @@ public class CertPemCredentialsTest { certPemCredentials.setPassword(password); KeyStore keyStore = certPemCredentials.loadKeyStore(); Assertions.assertNotNull(keyStore); - Key key = keyStore.getKey(PRIVATE_KEY_ALIAS, password.toCharArray()); + Key key = keyStore.getKey(PRIVATE_KEY_ALIAS, SslUtil.getPassword(password)); Assertions.assertNotNull(key); Assertions.assertEquals(algorithm, key.getAlgorithm()); From 4285f89665c2aa83e047926f293e89f61218d1b2 Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Thu, 23 Nov 2023 12:22:15 +0200 Subject: [PATCH 5/6] extended widgetTypes access to customer user authority --- .../server/controller/WidgetTypeController.java | 4 ++-- .../controller/WidgetTypeControllerTest.java | 14 ++++++++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/WidgetTypeController.java b/application/src/main/java/org/thingsboard/server/controller/WidgetTypeController.java index f90a196edf..0109a327e8 100644 --- a/application/src/main/java/org/thingsboard/server/controller/WidgetTypeController.java +++ b/application/src/main/java/org/thingsboard/server/controller/WidgetTypeController.java @@ -216,7 +216,7 @@ public class WidgetTypeController extends AutoCommitController { @ApiOperation(value = "Get all Widget types for specified Bundle (getBundleWidgetTypes)", notes = "Returns an array of Widget Type objects that belong to specified Widget Bundle." + WIDGET_TYPE_DESCRIPTION + " " + SYSTEM_OR_TENANT_AUTHORITY_PARAGRAPH) - @PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN')") + @PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN', 'CUSTOMER_USER')") @RequestMapping(value = "/widgetTypes", params = {"widgetsBundleId"}, method = RequestMethod.GET) @ResponseBody public List getBundleWidgetTypes( @@ -248,7 +248,7 @@ public class WidgetTypeController extends AutoCommitController { @ApiOperation(value = "Get all Widget types details for specified Bundle (getBundleWidgetTypes)", notes = "Returns an array of Widget Type Details objects that belong to specified Widget Bundle." + WIDGET_TYPE_DETAILS_DESCRIPTION + " " + SYSTEM_OR_TENANT_AUTHORITY_PARAGRAPH) - @PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN')") + @PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN', 'CUSTOMER_USER')") @RequestMapping(value = "/widgetTypesDetails", params = {"widgetsBundleId"}, method = RequestMethod.GET) @ResponseBody public List getBundleWidgetTypesDetails( diff --git a/application/src/test/java/org/thingsboard/server/controller/WidgetTypeControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/WidgetTypeControllerTest.java index 9cc9fd0596..553b0d6490 100644 --- a/application/src/test/java/org/thingsboard/server/controller/WidgetTypeControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/WidgetTypeControllerTest.java @@ -190,6 +190,20 @@ public class WidgetTypeControllerTest extends AbstractControllerTest { Collections.sort(loadedWidgetTypes, idComparator); Assert.assertEquals(widgetTypes, loadedWidgetTypes); + + loginCustomerUser(); + + List loadedWidgetTypes2 = doGetTyped("/api/widgetTypes?widgetsBundleId={widgetsBundleId}", + new TypeReference<>(){}, widgetsBundle.getId().getId().toString()); + Collections.sort(loadedWidgetTypes2, idComparator); + Assert.assertEquals(widgetTypes, loadedWidgetTypes2); + + List loadedWidgetTypes3 = doGetTyped("/api/widgetTypesDetails?widgetsBundleId={widgetsBundleId}", + new TypeReference<>(){}, widgetsBundle.getId().getId().toString()); + List widgetTypes3 = loadedWidgetTypes3.stream().map(WidgetType::new).collect(Collectors.toList()); + Collections.sort(widgetTypes3, idComparator); + Assert.assertEquals(widgetTypes3, loadedWidgetTypes); + } @Test From d156c246c4fee85f3c6f56b4db8922fd85cb28ca Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Mon, 27 Nov 2023 11:43:18 +0200 Subject: [PATCH 6/6] updated test to cover sysadmin authority --- .../controller/WidgetTypeControllerTest.java | 26 ++++++++++++++----- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/controller/WidgetTypeControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/WidgetTypeControllerTest.java index 553b0d6490..e515250321 100644 --- a/application/src/test/java/org/thingsboard/server/controller/WidgetTypeControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/WidgetTypeControllerTest.java @@ -193,17 +193,29 @@ public class WidgetTypeControllerTest extends AbstractControllerTest { loginCustomerUser(); - List loadedWidgetTypes2 = doGetTyped("/api/widgetTypes?widgetsBundleId={widgetsBundleId}", + List loadedWidgetTypesCustomer = doGetTyped("/api/widgetTypes?widgetsBundleId={widgetsBundleId}", new TypeReference<>(){}, widgetsBundle.getId().getId().toString()); - Collections.sort(loadedWidgetTypes2, idComparator); - Assert.assertEquals(widgetTypes, loadedWidgetTypes2); + Collections.sort(loadedWidgetTypesCustomer, idComparator); + Assert.assertEquals(widgetTypes, loadedWidgetTypesCustomer); - List loadedWidgetTypes3 = doGetTyped("/api/widgetTypesDetails?widgetsBundleId={widgetsBundleId}", + List customerLoadedWidgetTypesDetails = doGetTyped("/api/widgetTypesDetails?widgetsBundleId={widgetsBundleId}", new TypeReference<>(){}, widgetsBundle.getId().getId().toString()); - List widgetTypes3 = loadedWidgetTypes3.stream().map(WidgetType::new).collect(Collectors.toList()); - Collections.sort(widgetTypes3, idComparator); - Assert.assertEquals(widgetTypes3, loadedWidgetTypes); + List widgetTypesFromDetailsListCustomer = customerLoadedWidgetTypesDetails.stream().map(WidgetType::new).collect(Collectors.toList()); + Collections.sort(widgetTypesFromDetailsListCustomer, idComparator); + Assert.assertEquals(widgetTypesFromDetailsListCustomer, loadedWidgetTypes); + loginSysAdmin(); + + List sysAdminLoadedWidgetTypes = doGetTyped("/api/widgetTypes?widgetsBundleId={widgetsBundleId}", + new TypeReference<>(){}, widgetsBundle.getId().getId().toString()); + Collections.sort(sysAdminLoadedWidgetTypes, idComparator); + Assert.assertEquals(widgetTypes, sysAdminLoadedWidgetTypes); + + List sysAdminLoadedWidgetTypesDetails = doGetTyped("/api/widgetTypesDetails?widgetsBundleId={widgetsBundleId}", + new TypeReference<>(){}, widgetsBundle.getId().getId().toString()); + List widgetTypesFromDetailsListSysAdmin = sysAdminLoadedWidgetTypesDetails.stream().map(WidgetType::new).collect(Collectors.toList()); + Collections.sort(widgetTypesFromDetailsListSysAdmin, idComparator); + Assert.assertEquals(widgetTypesFromDetailsListSysAdmin, loadedWidgetTypes); } @Test