From 54e55d51ada9e90339dd81f14ed9e303ef3dfeef Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Wed, 16 Nov 2022 16:38:36 +0200 Subject: [PATCH 1/9] Use id instread of createdtime for sort order of edge events --- .../service/edge/rpc/fetch/GeneralEdgeEventFetcher.java | 2 +- .../server/dao/service/BaseEdgeEventServiceTest.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/fetch/GeneralEdgeEventFetcher.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/fetch/GeneralEdgeEventFetcher.java index ed5e039b62..8741426b42 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/fetch/GeneralEdgeEventFetcher.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/fetch/GeneralEdgeEventFetcher.java @@ -37,7 +37,7 @@ public class GeneralEdgeEventFetcher implements EdgeEventFetcher { pageSize, 0, null, - new SortOrder("createdTime", SortOrder.Direction.ASC), + new SortOrder("id", SortOrder.Direction.ASC), queueStartTs, null); } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/BaseEdgeEventServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/BaseEdgeEventServiceTest.java index 73b26df9e8..11a2423ba7 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/BaseEdgeEventServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/BaseEdgeEventServiceTest.java @@ -110,7 +110,7 @@ public abstract class BaseEdgeEventServiceTest extends AbstractServiceTest { Futures.allAsList(futures).get(); - TimePageLink pageLink = new TimePageLink(2, 0, "", new SortOrder("createdTime", SortOrder.Direction.DESC), startTime, endTime); + TimePageLink pageLink = new TimePageLink(2, 0, "", new SortOrder("id", SortOrder.Direction.DESC), startTime, endTime); PageData edgeEvents = edgeEventService.findEdgeEvents(tenantId, edgeId, pageLink, true); Assert.assertNotNull(edgeEvents.getData()); @@ -135,7 +135,7 @@ public abstract class BaseEdgeEventServiceTest extends AbstractServiceTest { EdgeId edgeId = new EdgeId(Uuids.timeBased()); DeviceId deviceId = new DeviceId(Uuids.timeBased()); TenantId tenantId = TenantId.fromUUID(Uuids.timeBased()); - TimePageLink pageLink = new TimePageLink(1, 0, null, new SortOrder("createdTime", SortOrder.Direction.ASC)); + TimePageLink pageLink = new TimePageLink(1, 0, null, new SortOrder("id", SortOrder.Direction.ASC)); EdgeEvent edgeEventWithTsUpdate = generateEdgeEvent(tenantId, edgeId, deviceId, EdgeEventActionType.TIMESERIES_UPDATED); edgeEventService.saveAsync(edgeEventWithTsUpdate).get(); From b5dbc7321cd33721541d3e710a5b1a42c0a9b7e6 Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Fri, 18 Nov 2022 16:27:36 +0200 Subject: [PATCH 2/9] Increase timeout between batches. Improve cancel and interruption of send downlink msgs task --- .../service/edge/rpc/EdgeGrpcSession.java | 50 +++++++++++++------ .../service/edge/rpc/EdgeSessionState.java | 6 +-- .../src/main/resources/thingsboard.yml | 2 +- 3 files changed, 38 insertions(+), 20 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java index 8ada917234..9571319226 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java @@ -187,10 +187,7 @@ public final class EdgeGrpcSession implements Closeable { public void startSyncProcess(TenantId tenantId, EdgeId edgeId, boolean fullSync) { log.trace("[{}][{}] Staring edge sync process", tenantId, edgeId); syncCompleted = false; - if (sessionState.getSendDownlinkMsgsFuture() != null && sessionState.getSendDownlinkMsgsFuture().isDone()) { - String errorMsg = String.format("[%s][%s] Sync process started. General processing interrupted!", tenantId, edgeId); - sessionState.getSendDownlinkMsgsFuture().setException(new RuntimeException(errorMsg)); - } + interruptGeneralProcessingOnSync(tenantId, edgeId); doSync(new EdgeSyncCursor(ctx, edge, fullSync)); } @@ -265,10 +262,7 @@ public final class EdgeGrpcSession implements Closeable { } if (sessionState.getPendingMsgsMap().isEmpty()) { log.debug("[{}] Pending msgs map is empty. Stopping current iteration", edge.getRoutingKey()); - if (sessionState.getScheduledSendDownlinkTask() != null) { - sessionState.getScheduledSendDownlinkTask().cancel(false); - } - sessionState.getSendDownlinkMsgsFuture().set(null); + stopCurrentSendDownlinkMsgsTask(null); } } catch (Exception e) { log.error("[{}] Can't process downlink response message [{}]", this.sessionId, msg, e); @@ -391,15 +385,14 @@ public final class EdgeGrpcSession implements Closeable { } private ListenableFuture sendDownlinkMsgsPack(List downlinkMsgsPack) { - if (sessionState.getSendDownlinkMsgsFuture() != null && !sessionState.getSendDownlinkMsgsFuture().isDone()) { - String errorMsg = "[" + this.sessionId + "] Previous send downlink future was not properly completed, stopping it now"; - log.error(errorMsg); - sessionState.getSendDownlinkMsgsFuture().setException(new RuntimeException(errorMsg)); - } + interruptPreviousSendDownlinkMsgsTask(); + sessionState.setSendDownlinkMsgsFuture(SettableFuture.create()); sessionState.getPendingMsgsMap().clear(); + downlinkMsgsPack.forEach(msg -> sessionState.getPendingMsgsMap().put(msg.getDownlinkMsgId(), msg)); scheduleDownlinkMsgsPackSend(1); + return sessionState.getSendDownlinkMsgsFuture(); } @@ -422,13 +415,13 @@ public final class EdgeGrpcSession implements Closeable { } else { log.warn("[{}] Failed to deliver the batch after {} attempts. Next messages are going to be discarded {}", this.sessionId, MAX_DOWNLINK_ATTEMPTS, copy); - sessionState.getSendDownlinkMsgsFuture().set(null); + stopCurrentSendDownlinkMsgsTask(null); } } else { - sessionState.getSendDownlinkMsgsFuture().set(null); + stopCurrentSendDownlinkMsgsTask(null); } } catch (Exception e) { - sessionState.getSendDownlinkMsgsFuture().setException(e); + stopCurrentSendDownlinkMsgsTask(e); } }; @@ -673,4 +666,29 @@ public final class EdgeGrpcSession implements Closeable { log.debug("[{}] Failed to close output stream: {}", sessionId, e.getMessage()); } } + + private void interruptPreviousSendDownlinkMsgsTask() { + String msg = String.format("[%s] Previous send downlink future was not properly completed, stopping it now!", this.sessionId); + stopCurrentSendDownlinkMsgsTask(new RuntimeException(msg)); + } + + private void interruptGeneralProcessingOnSync(TenantId tenantId, EdgeId edgeId) { + String msg = String.format("[%s][%s] Sync process started. General processing interrupted!", tenantId, edgeId); + stopCurrentSendDownlinkMsgsTask(new RuntimeException(msg)); + } + + public void stopCurrentSendDownlinkMsgsTask(Exception e) { + if (sessionState.getSendDownlinkMsgsFuture() != null && !sessionState.getSendDownlinkMsgsFuture().isDone()) { + if (e != null) { + log.warn(e.getMessage(), e); + sessionState.getSendDownlinkMsgsFuture().setException(e); + } else { + sessionState.getSendDownlinkMsgsFuture().set(null); + } + } + if (sessionState.getScheduledSendDownlinkTask() != null) { + sessionState.getScheduledSendDownlinkTask().cancel(true); + } + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeSessionState.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeSessionState.java index 98673c9cf5..f36ea73750 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeSessionState.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeSessionState.java @@ -19,14 +19,14 @@ import com.google.common.util.concurrent.SettableFuture; import lombok.Data; import org.thingsboard.server.gen.edge.v1.DownlinkMsg; -import java.util.LinkedHashMap; -import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ScheduledFuture; @Data public class EdgeSessionState { - private final Map pendingMsgsMap = new LinkedHashMap<>(); + private final ConcurrentMap pendingMsgsMap = new ConcurrentHashMap<>(); private SettableFuture sendDownlinkMsgsFuture; private ScheduledFuture scheduledSendDownlinkTask; } diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 0f110da90c..35c613cae6 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -928,7 +928,7 @@ edges: storage: max_read_records_count: "${EDGES_STORAGE_MAX_READ_RECORDS_COUNT:50}" no_read_records_sleep: "${EDGES_NO_READ_RECORDS_SLEEP:1000}" - sleep_between_batches: "${EDGES_SLEEP_BETWEEN_BATCHES:1000}" + sleep_between_batches: "${EDGES_SLEEP_BETWEEN_BATCHES:10000}" scheduler_pool_size: "${EDGES_SCHEDULER_POOL_SIZE:1}" send_scheduler_pool_size: "${EDGES_SEND_SCHEDULER_POOL_SIZE:1}" grpc_callback_thread_pool_size: "${EDGES_GRPC_CALLBACK_POOL_SIZE:1}" From a0205472e7b471642a53d7b41fc79e38cadce72b Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Fri, 18 Nov 2022 16:35:00 +0100 Subject: [PATCH 3/9] added max_old_space_size to the build:prod ui --- ui-ngx/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui-ngx/package.json b/ui-ngx/package.json index ea410747b2..6cacbda8bb 100644 --- a/ui-ngx/package.json +++ b/ui-ngx/package.json @@ -5,7 +5,7 @@ "ng": "ng", "start": "node --max_old_space_size=8048 ./node_modules/@angular/cli/bin/ng serve --configuration development --host 0.0.0.0 --open", "build": "ng build", - "build:prod": "ng build --configuration production --vendor-chunk", + "build:prod": "node --max_old_space_size=3072 ./node_modules/@angular/cli/bin/ng build --configuration production --vendor-chunk", "build:types": "node generate-types.js", "test": "ng test", "lint": "ng lint", From 833ea1b3976fb56e2dd7445988ab2792fdbdf62c Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Fri, 18 Nov 2022 19:23:21 +0200 Subject: [PATCH 4/9] Decreased number of message in edge random timeseries failure to speed up tests execution --- .../java/org/thingsboard/server/edge/BaseTelemetryEdgeTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/application/src/test/java/org/thingsboard/server/edge/BaseTelemetryEdgeTest.java b/application/src/test/java/org/thingsboard/server/edge/BaseTelemetryEdgeTest.java index 25280de666..8930166396 100644 --- a/application/src/test/java/org/thingsboard/server/edge/BaseTelemetryEdgeTest.java +++ b/application/src/test/java/org/thingsboard/server/edge/BaseTelemetryEdgeTest.java @@ -34,7 +34,7 @@ abstract public class BaseTelemetryEdgeTest extends AbstractEdgeTest { @Test public void testTimeseriesWithFailures() throws Exception { - int numberOfTimeseriesToSend = 1000; + int numberOfTimeseriesToSend = 333; Device device = findDeviceByName("Edge Device 1"); From a094b71a245059102d11e3e233f087b5413d35b2 Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Mon, 21 Nov 2022 18:30:25 +0200 Subject: [PATCH 5/9] Fix stability of testSendDeviceToCloudWithNameThatAlreadyExistsOnCloud --- .../thingsboard/server/edge/BaseDeviceEdgeTest.java | 12 ++++++------ .../server/edge/imitator/EdgeImitator.java | 6 +----- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/edge/BaseDeviceEdgeTest.java b/application/src/test/java/org/thingsboard/server/edge/BaseDeviceEdgeTest.java index ffe21ae961..a3162ebc5f 100644 --- a/application/src/test/java/org/thingsboard/server/edge/BaseDeviceEdgeTest.java +++ b/application/src/test/java/org/thingsboard/server/edge/BaseDeviceEdgeTest.java @@ -439,9 +439,9 @@ abstract public class BaseDeviceEdgeTest extends AbstractEdgeTest { Assert.assertTrue(edgeImitator.waitForResponses()); Assert.assertTrue(edgeImitator.waitForMessages()); - AbstractMessage latestMessage = edgeImitator.getMessageFromTail(2); - Assert.assertTrue(latestMessage instanceof DeviceUpdateMsg); - DeviceUpdateMsg latestDeviceUpdateMsg = (DeviceUpdateMsg) latestMessage; + Optional deviceUpdateMsgOpt = edgeImitator.findMessageByType(DeviceUpdateMsg.class); + Assert.assertTrue(deviceUpdateMsgOpt.isPresent()); + DeviceUpdateMsg latestDeviceUpdateMsg = deviceUpdateMsgOpt.get(); Assert.assertNotEquals(deviceOnCloudName, latestDeviceUpdateMsg.getName()); Assert.assertEquals(deviceOnCloudName, latestDeviceUpdateMsg.getConflictName()); @@ -453,9 +453,9 @@ abstract public class BaseDeviceEdgeTest extends AbstractEdgeTest { Assert.assertNotNull(device); Assert.assertNotEquals(deviceOnCloudName, device.getName()); - latestMessage = edgeImitator.getLatestMessage(); - Assert.assertTrue(latestMessage instanceof DeviceCredentialsRequestMsg); - DeviceCredentialsRequestMsg latestDeviceCredentialsRequestMsg = (DeviceCredentialsRequestMsg) latestMessage; + Optional deviceCredentialsUpdateMsgOpt = edgeImitator.findMessageByType(DeviceCredentialsRequestMsg.class); + Assert.assertTrue(deviceCredentialsUpdateMsgOpt.isPresent()); + DeviceCredentialsRequestMsg latestDeviceCredentialsRequestMsg = deviceCredentialsUpdateMsgOpt.get(); Assert.assertEquals(uuid.getMostSignificantBits(), latestDeviceCredentialsRequestMsg.getDeviceIdMSB()); Assert.assertEquals(uuid.getLeastSignificantBits(), latestDeviceCredentialsRequestMsg.getDeviceIdLSB()); diff --git a/application/src/test/java/org/thingsboard/server/edge/imitator/EdgeImitator.java b/application/src/test/java/org/thingsboard/server/edge/imitator/EdgeImitator.java index 668f702a87..4fa3b5e723 100644 --- a/application/src/test/java/org/thingsboard/server/edge/imitator/EdgeImitator.java +++ b/application/src/test/java/org/thingsboard/server/edge/imitator/EdgeImitator.java @@ -365,11 +365,7 @@ public class EdgeImitator { } public AbstractMessage getLatestMessage() { - return getMessageFromTail(1); - } - - public AbstractMessage getMessageFromTail(int offset) { - return downlinkMsgs.get(downlinkMsgs.size() - offset); + return downlinkMsgs.get(downlinkMsgs.size() - 1); } public void ignoreType(Class type) { From 263605ed52741ae1a110696cca3adff0a725d313 Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Mon, 21 Nov 2022 22:14:17 +0200 Subject: [PATCH 6/9] Revert Id to createdTime --- .../service/edge/rpc/fetch/GeneralEdgeEventFetcher.java | 2 +- .../server/dao/service/BaseEdgeEventServiceTest.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/fetch/GeneralEdgeEventFetcher.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/fetch/GeneralEdgeEventFetcher.java index 8741426b42..ed5e039b62 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/fetch/GeneralEdgeEventFetcher.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/fetch/GeneralEdgeEventFetcher.java @@ -37,7 +37,7 @@ public class GeneralEdgeEventFetcher implements EdgeEventFetcher { pageSize, 0, null, - new SortOrder("id", SortOrder.Direction.ASC), + new SortOrder("createdTime", SortOrder.Direction.ASC), queueStartTs, null); } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/BaseEdgeEventServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/BaseEdgeEventServiceTest.java index 11a2423ba7..73b26df9e8 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/BaseEdgeEventServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/BaseEdgeEventServiceTest.java @@ -110,7 +110,7 @@ public abstract class BaseEdgeEventServiceTest extends AbstractServiceTest { Futures.allAsList(futures).get(); - TimePageLink pageLink = new TimePageLink(2, 0, "", new SortOrder("id", SortOrder.Direction.DESC), startTime, endTime); + TimePageLink pageLink = new TimePageLink(2, 0, "", new SortOrder("createdTime", SortOrder.Direction.DESC), startTime, endTime); PageData edgeEvents = edgeEventService.findEdgeEvents(tenantId, edgeId, pageLink, true); Assert.assertNotNull(edgeEvents.getData()); @@ -135,7 +135,7 @@ public abstract class BaseEdgeEventServiceTest extends AbstractServiceTest { EdgeId edgeId = new EdgeId(Uuids.timeBased()); DeviceId deviceId = new DeviceId(Uuids.timeBased()); TenantId tenantId = TenantId.fromUUID(Uuids.timeBased()); - TimePageLink pageLink = new TimePageLink(1, 0, null, new SortOrder("id", SortOrder.Direction.ASC)); + TimePageLink pageLink = new TimePageLink(1, 0, null, new SortOrder("createdTime", SortOrder.Direction.ASC)); EdgeEvent edgeEventWithTsUpdate = generateEdgeEvent(tenantId, edgeId, deviceId, EdgeEventActionType.TIMESERIES_UPDATED); edgeEventService.saveAsync(edgeEventWithTsUpdate).get(); From a0a5549b26bfb4f3dc351b01994525b3a35c60af Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Mon, 21 Nov 2022 22:56:48 +0200 Subject: [PATCH 7/9] Revert LinkedHashMap --- .../server/service/edge/rpc/EdgeSessionState.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeSessionState.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeSessionState.java index f36ea73750..5a3d8dc0f8 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeSessionState.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeSessionState.java @@ -19,14 +19,15 @@ import com.google.common.util.concurrent.SettableFuture; import lombok.Data; import org.thingsboard.server.gen.edge.v1.DownlinkMsg; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; import java.util.concurrent.ScheduledFuture; @Data public class EdgeSessionState { - private final ConcurrentMap pendingMsgsMap = new ConcurrentHashMap<>(); + private final Map pendingMsgsMap = Collections.synchronizedMap(new LinkedHashMap<>()); private SettableFuture sendDownlinkMsgsFuture; private ScheduledFuture scheduledSendDownlinkTask; } From 75f7b7319a704e6f4a9a5d4401d929c121a37bb0 Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Wed, 23 Nov 2022 20:50:45 +0200 Subject: [PATCH 8/9] Added base implementation of edge docker install instructions controller --- .../docker/instructions.md | 88 +++++++++++++++++++ .../server/controller/BaseController.java | 4 + .../server/controller/EdgeController.java | 21 +++++ .../edge/DefaultEdgeInstallService.java | 71 +++++++++++++++ .../service/edge/EdgeInstallService.java | 27 ++++++ .../server/controller/AbstractWebTest.java | 8 +- .../controller/BaseEdgeControllerTest.java | 21 ++++- .../docker/expected-install-instructions.md | 88 +++++++++++++++++++ 8 files changed, 321 insertions(+), 7 deletions(-) create mode 100644 application/src/main/data/json/edge/install_instructions/docker/instructions.md create mode 100644 application/src/main/java/org/thingsboard/server/service/edge/DefaultEdgeInstallService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/edge/EdgeInstallService.java create mode 100644 application/src/test/resources/edge/install_instructions/docker/expected-install-instructions.md diff --git a/application/src/main/data/json/edge/install_instructions/docker/instructions.md b/application/src/main/data/json/edge/install_instructions/docker/instructions.md new file mode 100644 index 0000000000..15a7db1d74 --- /dev/null +++ b/application/src/main/data/json/edge/install_instructions/docker/instructions.md @@ -0,0 +1,88 @@ +## Install edge and connect to cloud instructions + +### Localhost warning + +Localhost cannot be used for docker install - please update baseUrl to the IP address of your machine! + +Here is the list of commands, that can be used to quickly install and connect edge to the cloud using docker-compose. + +### Prerequisites + +Install Docker CE and Docker Compose. + +### Create data and logs folders + +Run following commands to create a directory for storing data and logs and then change its owner to docker container user, to be able to change user, chown command is used, which requires sudo permissions (command will request password for a sudo access): + +```bash +mkdir -p ~/.mytb-edge-data && sudo chown -R 799:799 ~/.mytb-edge-data +mkdir -p ~/.mytb-edge-logs && sudo chown -R 799:799 ~/.mytb-edge-logs +{:copy-code} +``` + +### Running ThingsBoard Edge as docker service + +Create docker compose file for ThingsBoard Edge service: + +```bash +nano docker-compose.yml +{:copy-code} +``` + +Add the following lines to the yml file: + +``` +version: '2.2' +services: +mytbedge: +restart: always +image: "thingsboard/tb-edge:3.4.1EDGE" +ports: +- "8080:8080" +- "1883:1883" +- "5683-5688:5683-5688/udp" +environment: +SPRING_DATASOURCE_URL: jdbc:postgresql://postgres:5432/tb-edge +CLOUD_ROUTING_KEY: ${CLOUD_ROUTING_KEY} +CLOUD_ROUTING_SECRET: ${CLOUD_ROUTING_SECRET} +CLOUD_RPC_HOST: ${BASE_URL} +volumes: +- ~/.mytb-edge-data:/data +- ~/.mytb-edge-logs:/var/log/tb-edge +postgres: +restart: always +image: "postgres:12" +ports: +- "5432" +environment: +POSTGRES_DB: tb-edge +POSTGRES_PASSWORD: postgres +volumes: +- ~/.mytb-edge-data/db:/var/lib/postgresql/data +{:copy-code} +``` + +### Ports warning [Optional] +If ThingsBoard Edge is going to be running on the same machine where ThingsBoard server is running you’ll need to update docker compose port mapping. +Please update next lines of docker compose: +ports: +- “18080:8080” +- “11883:1883” +- “15683-15688:5683-5688/udp” + Please make sure ports above are not used by any other application. + + +Execute the following commands to up this docker compose directly: + +```bash +docker-compose pull +docker-compose up +{:copy-code} +``` + +### Open ThingsBoard Edge UI + +Once started, you will be able to open ThingsBoard Edge UI using the following link http://localhost:8080. + +If during installation process you have changed edge HTTP_BIND_PORT please use that port instead for Edge UI URL: +http://localhost:HTTP_BIND_PORT diff --git a/application/src/main/java/org/thingsboard/server/controller/BaseController.java b/application/src/main/java/org/thingsboard/server/controller/BaseController.java index ffc3fcf3cd..1a8db80149 100644 --- a/application/src/main/java/org/thingsboard/server/controller/BaseController.java +++ b/application/src/main/java/org/thingsboard/server/controller/BaseController.java @@ -131,6 +131,7 @@ import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.provider.TbQueueProducerProvider; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.component.ComponentDiscoveryService; +import org.thingsboard.server.service.edge.EdgeInstallService; import org.thingsboard.server.service.edge.EdgeNotificationService; import org.thingsboard.server.service.edge.rpc.EdgeRpcService; import org.thingsboard.server.service.entitiy.TbNotificationEntityService; @@ -281,6 +282,9 @@ public abstract class BaseController { @Autowired(required = false) protected EdgeRpcService edgeRpcService; + @Autowired(required = false) + protected EdgeInstallService edgeInstallService; + @Autowired protected TbNotificationEntityService notificationEntityService; diff --git a/application/src/main/java/org/thingsboard/server/controller/EdgeController.java b/application/src/main/java/org/thingsboard/server/controller/EdgeController.java index 6a5dd44d0b..12e02aa0bd 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EdgeController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EdgeController.java @@ -63,6 +63,7 @@ import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; +import javax.servlet.http.HttpServletRequest; import java.util.ArrayList; import java.util.List; import java.util.UUID; @@ -595,4 +596,24 @@ public class EdgeController extends BaseController { return edgeBulkImportService.processBulkImport(request, user); } + + @ApiOperation(value = "Get Edge Docker Install Instructions (getEdgeDockerInstallInstructions)", + notes = "Get a docker install instructions for provided edge id." + TENANT_AUTHORITY_PARAGRAPH, + produces = MediaType.APPLICATION_JSON_VALUE) + @PreAuthorize("hasAnyAuthority('TENANT_ADMIN')") + @RequestMapping(value = "/edge/instructions/{edgeId}", method = RequestMethod.GET) + @ResponseBody + public String getEdgeDockerInstallInstructions( + @ApiParam(value = EDGE_ID_PARAM_DESCRIPTION, required = true) + @PathVariable("edgeId") String strEdgeId, + HttpServletRequest request) throws ThingsboardException { + try { + EdgeId edgeId = new EdgeId(toUUID(strEdgeId)); + edgeId = checkNotNull(edgeId); + Edge edge = checkEdgeId(edgeId, Operation.READ); + return checkNotNull(edgeInstallService.getDockerInstallInstructions(getTenantId(), edge, request)); + } catch (Exception e) { + throw handleException(e); + } + } } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/DefaultEdgeInstallService.java b/application/src/main/java/org/thingsboard/server/service/edge/DefaultEdgeInstallService.java new file mode 100644 index 0000000000..e281efbf16 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/edge/DefaultEdgeInstallService.java @@ -0,0 +1,71 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.edge; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.edge.Edge; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.service.install.InstallScripts; +import org.thingsboard.server.service.security.system.SystemSecurityService; + +import javax.servlet.http.HttpServletRequest; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +@Service +@Slf4j +@RequiredArgsConstructor +public class DefaultEdgeInstallService implements EdgeInstallService { + + private static final String EDGE_DIR = "edge"; + + private static final String EDGE_INSTALL_INSTRUCTIONS_DIR = "install_instructions"; + + private final InstallScripts installScripts; + private final SystemSecurityService systemSecurityService; + + @Override + public String getDockerInstallInstructions(TenantId tenantId, Edge edge, HttpServletRequest request) { + String baseUrl = request.getServerName(); + String template = readFile(resolveFile("docker", "instructions.md")); + template = template.replace("${BASE_URL}", baseUrl); + template = template.replace("${CLOUD_ROUTING_KEY}", edge.getRoutingKey()); + template = template.replace("${CLOUD_ROUTING_SECRET}", edge.getSecret()); + return template; + } + + private String readFile(Path file) { + try { + return new String(Files.readAllBytes(file), StandardCharsets.UTF_8); + } catch (IOException e) { + log.warn("Failed to read file: {}", file, e); + throw new RuntimeException(e); + } + } + + private Path resolveFile(String subDir, String... subDirs) { + return getEdgeInstallInstructionsDir().resolve(Paths.get(subDir, subDirs)); + } + + private Path getEdgeInstallInstructionsDir() { + return Paths.get(installScripts.getDataDir(), InstallScripts.JSON_DIR, EDGE_DIR, EDGE_INSTALL_INSTRUCTIONS_DIR); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/EdgeInstallService.java b/application/src/main/java/org/thingsboard/server/service/edge/EdgeInstallService.java new file mode 100644 index 0000000000..54ad0de55d --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/edge/EdgeInstallService.java @@ -0,0 +1,27 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.edge; + +import org.thingsboard.server.common.data.edge.Edge; +import org.thingsboard.server.common.data.id.TenantId; + +import javax.servlet.http.HttpServletRequest; + +public interface EdgeInstallService { + + String getDockerInstallInstructions(TenantId tenantId, Edge edge, HttpServletRequest request); + +} diff --git a/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java b/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java index e2b7d700bd..4d8e7c8488 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java @@ -701,16 +701,16 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest { } protected Edge constructEdge(String name, String type) { - return constructEdge(tenantId, name, type); + return constructEdge(tenantId, name, type, StringUtils.randomAlphanumeric(20), StringUtils.randomAlphanumeric(20)); } - protected Edge constructEdge(TenantId tenantId, String name, String type) { + protected Edge constructEdge(TenantId tenantId, String name, String type, String routingKey, String secret) { Edge edge = new Edge(); edge.setTenantId(tenantId); edge.setName(name); edge.setType(type); - edge.setSecret(StringUtils.randomAlphanumeric(20)); - edge.setRoutingKey(StringUtils.randomAlphanumeric(20)); + edge.setRoutingKey(routingKey); + edge.setSecret(secret); return edge; } diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseEdgeControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseEdgeControllerTest.java index fa466ce93f..a6dde57752 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseEdgeControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseEdgeControllerTest.java @@ -57,6 +57,10 @@ import org.thingsboard.server.gen.edge.v1.RuleChainUpdateMsg; import org.thingsboard.server.gen.edge.v1.UserCredentialsUpdateMsg; import org.thingsboard.server.gen.edge.v1.UserUpdateMsg; +import java.io.File; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -92,7 +96,7 @@ public abstract class BaseEdgeControllerTest extends AbstractControllerTest { } } - @Before + @Before public void beforeTest() throws Exception { loginSysAdmin(); @@ -327,7 +331,7 @@ public abstract class BaseEdgeControllerTest extends AbstractControllerTest { String customerIdStr = customerId.getId().toString(); String msgError = msgErrorNoFound("Customer", customerIdStr); - doPost("/api/customer/" + customerIdStr+ "/edge/" + savedEdge.getId().getId().toString()) + doPost("/api/customer/" + customerIdStr + "/edge/" + savedEdge.getId().getId().toString()) .andExpect(status().isNotFound()) .andExpect(statusReason(containsString(msgError))); @@ -876,4 +880,15 @@ public abstract class BaseEdgeControllerTest extends AbstractControllerTest { Edge edge = constructEdge(name, "default"); return doPost("/api/edge", edge, Edge.class); } -} + + @Test + public void testGetEdgeInstallInstructions() throws Exception { + Edge edge = constructEdge(tenantId, "Edge for Test Docker Install Instructions", "default", "7390c3a6-69b0-9910-d155-b90aca4b772e", "l7q4zsjplzwhk16geqxy"); + Edge savedEdge = doPost("/api/edge", edge, Edge.class); + String installInstructions = doGet("/api/edge/instructions/" + savedEdge.getId().getId().toString(), String.class); + URL resource = this.getClass().getClassLoader().getResource("edge/install_instructions/docker/expected-install-instructions.md"); + File file = new File(resource.toURI()); + Assert.assertEquals(new String(Files.readAllBytes(file.toPath()), StandardCharsets.UTF_8), installInstructions); + } + +} \ No newline at end of file diff --git a/application/src/test/resources/edge/install_instructions/docker/expected-install-instructions.md b/application/src/test/resources/edge/install_instructions/docker/expected-install-instructions.md new file mode 100644 index 0000000000..e9a175bef9 --- /dev/null +++ b/application/src/test/resources/edge/install_instructions/docker/expected-install-instructions.md @@ -0,0 +1,88 @@ +## Install edge and connect to cloud instructions + +### Localhost warning + +Localhost cannot be used for docker install - please update baseUrl to the IP address of your machine! + +Here is the list of commands, that can be used to quickly install and connect edge to the cloud using docker-compose. + +### Prerequisites + +Install Docker CE and Docker Compose. + +### Create data and logs folders + +Run following commands to create a directory for storing data and logs and then change its owner to docker container user, to be able to change user, chown command is used, which requires sudo permissions (command will request password for a sudo access): + +```bash +mkdir -p ~/.mytb-edge-data && sudo chown -R 799:799 ~/.mytb-edge-data +mkdir -p ~/.mytb-edge-logs && sudo chown -R 799:799 ~/.mytb-edge-logs +{:copy-code} +``` + +### Running ThingsBoard Edge as docker service + +Create docker compose file for ThingsBoard Edge service: + +```bash +nano docker-compose.yml +{:copy-code} +``` + +Add the following lines to the yml file: + +``` +version: '2.2' +services: +mytbedge: +restart: always +image: "thingsboard/tb-edge:3.4.1EDGE" +ports: +- "8080:8080" +- "1883:1883" +- "5683-5688:5683-5688/udp" +environment: +SPRING_DATASOURCE_URL: jdbc:postgresql://postgres:5432/tb-edge +CLOUD_ROUTING_KEY: 7390c3a6-69b0-9910-d155-b90aca4b772e +CLOUD_ROUTING_SECRET: l7q4zsjplzwhk16geqxy +CLOUD_RPC_HOST: localhost +volumes: +- ~/.mytb-edge-data:/data +- ~/.mytb-edge-logs:/var/log/tb-edge +postgres: +restart: always +image: "postgres:12" +ports: +- "5432" +environment: +POSTGRES_DB: tb-edge +POSTGRES_PASSWORD: postgres +volumes: +- ~/.mytb-edge-data/db:/var/lib/postgresql/data +{:copy-code} +``` + +### Ports warning [Optional] +If ThingsBoard Edge is going to be running on the same machine where ThingsBoard server is running you’ll need to update docker compose port mapping. +Please update next lines of docker compose: +ports: +- “18080:8080” +- “11883:1883” +- “15683-15688:5683-5688/udp” + Please make sure ports above are not used by any other application. + + +Execute the following commands to up this docker compose directly: + +```bash +docker-compose pull +docker-compose up +{:copy-code} +``` + +### Open ThingsBoard Edge UI + +Once started, you will be able to open ThingsBoard Edge UI using the following link http://localhost:8080. + +If during installation process you have changed edge HTTP_BIND_PORT please use that port instead for Edge UI URL: +http://localhost:HTTP_BIND_PORT From 4893980352fe9fe285f7ce501382c805d42918ff Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Wed, 23 Nov 2022 20:54:49 +0200 Subject: [PATCH 9/9] Added ConditionalOnProperty for DefaultEdgeInstallService --- .../server/service/edge/DefaultEdgeInstallService.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/edge/DefaultEdgeInstallService.java b/application/src/main/java/org/thingsboard/server/service/edge/DefaultEdgeInstallService.java index e281efbf16..e183cebca0 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/DefaultEdgeInstallService.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/DefaultEdgeInstallService.java @@ -17,11 +17,12 @@ package org.thingsboard.server.service.edge; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.install.InstallScripts; -import org.thingsboard.server.service.security.system.SystemSecurityService; import javax.servlet.http.HttpServletRequest; import java.io.IOException; @@ -33,6 +34,8 @@ import java.nio.file.Paths; @Service @Slf4j @RequiredArgsConstructor +@ConditionalOnProperty(prefix = "edges", value = "enabled", havingValue = "true") +@TbCoreComponent public class DefaultEdgeInstallService implements EdgeInstallService { private static final String EDGE_DIR = "edge"; @@ -40,7 +43,6 @@ public class DefaultEdgeInstallService implements EdgeInstallService { private static final String EDGE_INSTALL_INSTRUCTIONS_DIR = "install_instructions"; private final InstallScripts installScripts; - private final SystemSecurityService systemSecurityService; @Override public String getDockerInstallInstructions(TenantId tenantId, Edge edge, HttpServletRequest request) {