From e1496f32846c4288705c993e1499136cd5c3924f Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Mon, 28 Aug 2023 19:20:31 +0200 Subject: [PATCH 1/3] download mqtts cert improvements --- .../DeviceConnectivityControllerTest.java | 65 ++++++++++++++++++- dao/pom.xml | 4 ++ .../device/DeviceConnectivityServiceImpl.java | 64 +++++++++++++++--- 3 files changed, 122 insertions(+), 11 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/controller/DeviceConnectivityControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/DeviceConnectivityControllerTest.java index 36a4365544..383a050bb2 100644 --- a/application/src/test/java/org/thingsboard/server/controller/DeviceConnectivityControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/DeviceConnectivityControllerTest.java @@ -27,6 +27,7 @@ import org.mockito.AdditionalAnswers; import org.mockito.Mockito; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Primary; +import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; import org.thingsboard.common.util.JacksonUtil; @@ -43,13 +44,15 @@ import org.thingsboard.server.common.data.device.profile.DefaultDeviceProfileCon import org.thingsboard.server.common.data.device.profile.DeviceProfileData; import org.thingsboard.server.common.data.device.profile.MqttDeviceProfileTransportConfiguration; import org.thingsboard.server.common.data.id.DeviceProfileId; -import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.data.security.DeviceCredentials; import org.thingsboard.server.common.data.security.DeviceCredentialsType; import org.thingsboard.server.dao.device.DeviceDao; import org.thingsboard.server.dao.service.DaoSqlTest; +import java.nio.file.Files; +import java.nio.file.Path; + import static org.assertj.core.api.Assertions.assertThat; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; import static org.thingsboard.server.dao.util.DeviceConnectivityUtil.COAP; @@ -59,21 +62,43 @@ import static org.thingsboard.server.dao.util.DeviceConnectivityUtil.HTTP; import static org.thingsboard.server.dao.util.DeviceConnectivityUtil.HTTPS; import static org.thingsboard.server.dao.util.DeviceConnectivityUtil.MQTT; import static org.thingsboard.server.dao.util.DeviceConnectivityUtil.MQTTS; +import static org.thingsboard.server.dao.util.DeviceConnectivityUtil.PEM_CERT_FILE_NAME; @TestPropertySource(properties = { "device.connectivity.https.enabled=true", "device.connectivity.mqtts.enabled=true", + "device.connectivity.mqtts.pem_cert_file=/tmp/" + PEM_CERT_FILE_NAME, "device.connectivity.coaps.enabled=true", }) @ContextConfiguration(classes = {DeviceConnectivityControllerTest.Config.class}) @DaoSqlTest public class DeviceConnectivityControllerTest extends AbstractControllerTest { - static final TypeReference> PAGE_DATA_DEVICE_TYPE_REF = new TypeReference<>() { - }; private static final String DEVICE_TELEMETRY_TOPIC = "v1/devices/customTopic"; private static final String CHECK_DOCUMENTATION = "Check documentation"; + private static final String CERT = "-----BEGIN CERTIFICATE-----\n" + + "MIIBfzCCASmgAwIBAgIUC1dtaskm/SJLmFE2Ae+YojArg+swDQYJKoZIhvcNAQEL\n" + + "BQAwFDESMBAGA1UEAwwJbG9jYWxob3N0MB4XDTIzMDgyODEzMTAzM1oXDTI0MDgy\n" + + "NzEzMTAzM1owFDESMBAGA1UEAwwJbG9jYWxob3N0MFwwDQYJKoZIhvcNAQEBBQAD\n" + + "SwAwSAJBANpcs46MavFdv7onsxH178YgK5XbpMqzx8AKaLMP2X6UEXN0nlt5mpX5\n" + + "uCJmSwVaFn6lwTm8ThXFYOBydOQImIsCAwEAAaNTMFEwHQYDVR0OBBYEFDvN49bI\n" + + "LaWMmUZ+cMboWAaozfXTMB8GA1UdIwQYMBaAFDvN49bILaWMmUZ+cMboWAaozfXT\n" + + "MA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADQQAhIQL8zPvIhQvHJocU\n" + + "tnSmDAE0iR2rJVkousA+LiORE9BnuBtBUEv5SvFUv3VYUWA0eYFoyatpDHByIm6e\n" + + "/+1c\n" + + "-----END CERTIFICATE-----\n"; + private static final String P_KEY = "-----BEGIN PRIVATE KEY-----\n" + + "MIIBVgIBADANBgkqhkiG9w0BAQEFAASCAUAwggE8AgEAAkEA2lyzjoxq8V2/uiez\n" + + "EfXvxiArldukyrPHwAposw/ZfpQRc3SeW3malfm4ImZLBVoWfqXBObxOFcVg4HJ0\n" + + "5AiYiwIDAQABAkEA1DYhPljSmc2dRcHNMphLtMWQ9iumpGRBrS2wgMzXdz2NF2+0\n" + + "4cicaaL06/Cw6XXx43s8cn7e1xZAkGtNRQuqMQIhAPbrqrcYsropURpI5HSemeha\n" + + "MJA3i67ZFaom39VSrNKJAiEA4mQ0qFKxFSh2xAOqDWDRkiCgdOS00J6hgrYJRPcI\n" + + "nXMCIQDBHGjkT72gGKYkT3PUvSGTdc3bTIXDFmZ6L3MJTGJ7OQIhAKO+6r9coCy3\n" + + "ib+ZDuSCRNK2upgR3B6Qvi020VmKfDa1AiBhCgpBlClv5OjnmC42EGxxFOaZtNQQ\n" + + "C3swkUdrR3pezg==\n" + + "-----END PRIVATE KEY-----\n"; + ListeningExecutorService executor; private Tenant savedTenant; @@ -337,4 +362,38 @@ public class DeviceConnectivityControllerTest extends AbstractControllerTest { assertThat(commands).hasSize(1); assertThat(commands.get(COAP).get(COAPS).asText()).isEqualTo(CHECK_DOCUMENTATION); } + + @Test + @DirtiesContext + public void testDownloadMqttCert() throws Exception { + Path path = Files.createFile(Path.of("/tmp/" + PEM_CERT_FILE_NAME)); + Files.writeString(path, CERT); + + try { + String downloadedCert = doGet("/api/device-connectivity/mqtts/certificate/download", String.class); + Assert.assertEquals(CERT, downloadedCert); + } finally { + Files.deleteIfExists(path); + } + } + + @Test + @DirtiesContext + public void testDownloadMqttCertFromFileWithPrivateKey() throws Exception { + Path path = Files.createFile(Path.of("/tmp/" + PEM_CERT_FILE_NAME)); + Files.writeString(path, CERT + P_KEY); + + try { + String downloadedCert = doGet("/api/device-connectivity/mqtts/certificate/download", String.class); + Assert.assertEquals(CERT, downloadedCert); + } finally { + Files.deleteIfExists(path); + } + } + + @Test + @DirtiesContext + public void testDownloadMqttCertWithoutCertFile() throws Exception { + doGet("/api/device-connectivity/mqtts/certificate/download").andExpect(status().isNotFound()); + } } diff --git a/dao/pom.xml b/dao/pom.xml index ece2215327..0d6d4b4c02 100644 --- a/dao/pom.xml +++ b/dao/pom.xml @@ -79,6 +79,10 @@ org.postgresql postgresql + + org.bouncycastle + bcpkix-jdk15on + org.springframework.boot spring-boot-starter-test diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceConnectivityServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceConnectivityServiceImpl.java index c06103d8f3..d7c5c8c767 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceConnectivityServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceConnectivityServiceImpl.java @@ -19,8 +19,10 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import lombok.extern.slf4j.Slf4j; +import org.bouncycastle.cert.X509CertificateHolder; +import org.bouncycastle.openssl.PEMParser; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.core.io.ClassPathResource; +import org.springframework.core.io.ByteArrayResource; import org.springframework.core.io.Resource; import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; @@ -35,11 +37,17 @@ import org.thingsboard.server.common.data.security.DeviceCredentials; import org.thingsboard.server.common.data.security.DeviceCredentialsType; import org.thingsboard.server.dao.util.DeviceConnectivityUtil; +import java.io.InputStream; +import java.io.InputStreamReader; import java.net.URI; import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Base64; import java.util.List; +import java.util.Map; import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; import static org.thingsboard.server.dao.service.Validator.validateId; import static org.thingsboard.server.dao.util.DeviceConnectivityUtil.CHECK_DOCUMENTATION; @@ -59,6 +67,8 @@ public class DeviceConnectivityServiceImpl implements DeviceConnectivityService public static final String INCORRECT_DEVICE_ID = "Incorrect deviceId "; public static final String DEFAULT_DEVICE_TELEMETRY_TOPIC = "v1/devices/me/telemetry"; + private final Map certs = new ConcurrentHashMap<>(); + @Autowired private DeviceCredentialsService deviceCredentialsService; @@ -68,6 +78,9 @@ public class DeviceConnectivityServiceImpl implements DeviceConnectivityService @Autowired private DeviceConnectivityConfiguration deviceConnectivityConfiguration; + @Autowired + private DeviceConnectivityServiceImpl deviceConnectivityService; + @Override public JsonNode findDevicePublishTelemetryCommands(String baseUrl, Device device) throws URISyntaxException { DeviceId deviceId = device.getId(); @@ -115,15 +128,50 @@ public class DeviceConnectivityServiceImpl implements DeviceConnectivityService @Override public Resource getPemCertFile(String protocol) { - String certFilePath = deviceConnectivityConfiguration.getConnectivity() - .get(protocol) - .getPemCertFile(); + return certs.computeIfAbsent(protocol, key -> { + String certFilePath = deviceConnectivityConfiguration.getConnectivity() + .get(protocol) + .getPemCertFile(); + if (StringUtils.isNotBlank(certFilePath) && ResourceUtils.resourceExists(this, certFilePath)) { + try { + return getCert(certFilePath); + } catch (Exception e) { + String msg = String.format("Failed to read %s server certificate!", protocol); + log.warn(msg); + throw new RuntimeException(msg, e); + } + } else { + return null; + } + }); + } - if (StringUtils.isNotBlank(certFilePath) && ResourceUtils.resourceExists(this, certFilePath)) { - return new ClassPathResource(certFilePath); - } else { - return null; + private Resource getCert(String path) throws Exception { + StringBuilder pemContentBuilder = new StringBuilder(); + + try (InputStream inStream = ResourceUtils.getInputStream(this, path); + PEMParser pemParser = new PEMParser(new InputStreamReader(inStream))) { + + Object object; + + while ((object = pemParser.readObject()) != null) { + if (object instanceof X509CertificateHolder) { + var certHolder = (X509CertificateHolder) object; + String certBase64 = Base64.getEncoder().encodeToString(certHolder.getEncoded()); + + pemContentBuilder.append("-----BEGIN CERTIFICATE-----\n"); + int index = 0; + while (index < certBase64.length()) { + pemContentBuilder.append(certBase64, index, Math.min(index + 64, certBase64.length())); + pemContentBuilder.append("\n"); + index += 64; + } + pemContentBuilder.append("-----END CERTIFICATE-----\n"); + } + } } + + return new ByteArrayResource(pemContentBuilder.toString().getBytes(StandardCharsets.UTF_8)); } private JsonNode getHttpTransportPublishCommands(String defaultHostname, DeviceCredentials deviceCredentials) throws URISyntaxException { From 1c87dabac2965803195367b62d147fd71bdc25d6 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Tue, 29 Aug 2023 10:56:37 +0200 Subject: [PATCH 2/3] improvements --- .../device/DeviceConnectivityConfiguration.java | 3 ++- .../dao/device/DeviceConnectivityServiceImpl.java | 15 +++++++++++++-- .../test/resources/application-test.properties | 2 ++ 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceConnectivityConfiguration.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceConnectivityConfiguration.java index 033aa4e0bc..99a4014f48 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceConnectivityConfiguration.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceConnectivityConfiguration.java @@ -19,13 +19,14 @@ import lombok.Data; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.context.annotation.Configuration; +import java.util.HashMap; import java.util.Map; @Configuration @ConfigurationProperties(prefix = "device") @Data public class DeviceConnectivityConfiguration { - private Map connectivity; + private Map connectivity = new HashMap<>(); public boolean isEnabled(String protocol) { var info = connectivity.get(protocol); diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceConnectivityServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceConnectivityServiceImpl.java index d7c5c8c767..995e3798b0 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceConnectivityServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceConnectivityServiceImpl.java @@ -37,6 +37,7 @@ import org.thingsboard.server.common.data.security.DeviceCredentials; import org.thingsboard.server.common.data.security.DeviceCredentialsType; import org.thingsboard.server.dao.util.DeviceConnectivityUtil; +import javax.annotation.PostConstruct; import java.io.InputStream; import java.io.InputStreamReader; import java.net.URI; @@ -78,8 +79,18 @@ public class DeviceConnectivityServiceImpl implements DeviceConnectivityService @Autowired private DeviceConnectivityConfiguration deviceConnectivityConfiguration; - @Autowired - private DeviceConnectivityServiceImpl deviceConnectivityService; + @PostConstruct + private void init() { + DeviceConnectivityInfo mqtts = deviceConnectivityConfiguration.getConnectivity().get(MQTTS); + if (mqtts != null && mqtts.isEnabled()) { + String certFilePath = mqtts.getPemCertFile(); + if (StringUtils.isBlank(certFilePath) || !ResourceUtils.resourceExists(this, certFilePath)) { + String error = StringUtils.isBlank(certFilePath) ? "path is empty" : "file is not exists"; + log.error("MQTTS is enabled but cert {}!", error); + } + } + + } @Override public JsonNode findDevicePublishTelemetryCommands(String baseUrl, Device device) throws URISyntaxException { diff --git a/dao/src/test/resources/application-test.properties b/dao/src/test/resources/application-test.properties index 98f9091318..447a44a163 100644 --- a/dao/src/test/resources/application-test.properties +++ b/dao/src/test/resources/application-test.properties @@ -136,3 +136,5 @@ queue.rule-engine.queues[2].partitions=2 queue.rule-engine.queues[2].processing-strategy.retries=1 queue.rule-engine.queues[2].processing-strategy.pause-between-retries=0 queue.rule-engine.queues[2].processing-strategy.max-pause-between-retries=0 + +#device.connectivity= From 58859208ec730185a99c3ed7c8a6512034a1f620 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Mon, 4 Sep 2023 15:55:40 +0200 Subject: [PATCH 3/3] refactoring --- dao/src/test/resources/application-test.properties | 2 -- 1 file changed, 2 deletions(-) diff --git a/dao/src/test/resources/application-test.properties b/dao/src/test/resources/application-test.properties index 447a44a163..98f9091318 100644 --- a/dao/src/test/resources/application-test.properties +++ b/dao/src/test/resources/application-test.properties @@ -136,5 +136,3 @@ queue.rule-engine.queues[2].partitions=2 queue.rule-engine.queues[2].processing-strategy.retries=1 queue.rule-engine.queues[2].processing-strategy.pause-between-retries=0 queue.rule-engine.queues[2].processing-strategy.max-pause-between-retries=0 - -#device.connectivity=