Merge remote-tracking branch 'voba/edge-install-instructions' into feature/edge-instructions

This commit is contained in:
Artem Babak 2022-11-24 10:37:39 +02:00
commit b650c4646a
15 changed files with 369 additions and 38 deletions

View File

@ -0,0 +1,88 @@
## Install edge and connect to cloud instructions
### Localhost warning
Localhost cannot be used for docker install - please update baseUrl to the IP address of your machine!
Here is the list of commands, that can be used to quickly install and connect edge to the cloud using docker-compose.
### Prerequisites
Install <a href="https://docs.docker.com/engine/install/" target="_blank"> Docker CE</a> and <a href="https://docs.docker.com/compose/install/" target="_blank"> Docker Compose</a>.
### Create data and logs folders
Run following commands to create a directory for storing data and logs and then change its owner to docker container user, to be able to change user, chown command is used, which requires sudo permissions (command will request password for a sudo access):
```bash
mkdir -p ~/.mytb-edge-data && sudo chown -R 799:799 ~/.mytb-edge-data
mkdir -p ~/.mytb-edge-logs && sudo chown -R 799:799 ~/.mytb-edge-logs
{:copy-code}
```
### Running ThingsBoard Edge as docker service
Create docker compose file for ThingsBoard Edge service:
```bash
nano docker-compose.yml
{:copy-code}
```
Add the following lines to the yml file:
```
version: '2.2'
services:
mytbedge:
restart: always
image: "thingsboard/tb-edge:3.4.1EDGE"
ports:
- "8080:8080"
- "1883:1883"
- "5683-5688:5683-5688/udp"
environment:
SPRING_DATASOURCE_URL: jdbc:postgresql://postgres:5432/tb-edge
CLOUD_ROUTING_KEY: ${CLOUD_ROUTING_KEY}
CLOUD_ROUTING_SECRET: ${CLOUD_ROUTING_SECRET}
CLOUD_RPC_HOST: ${BASE_URL}
volumes:
- ~/.mytb-edge-data:/data
- ~/.mytb-edge-logs:/var/log/tb-edge
postgres:
restart: always
image: "postgres:12"
ports:
- "5432"
environment:
POSTGRES_DB: tb-edge
POSTGRES_PASSWORD: postgres
volumes:
- ~/.mytb-edge-data/db:/var/lib/postgresql/data
{:copy-code}
```
### Ports warning [Optional]
If ThingsBoard Edge is going to be running on the same machine where ThingsBoard server is running you’ll need to update docker compose port mapping.
Please update next lines of docker compose:
ports:
- “18080:8080”
- “11883:1883”
- “15683-15688:5683-5688/udp”
Please make sure ports above are not used by any other application.
Execute the following commands to up this docker compose directly:
```bash
docker-compose pull
docker-compose up
{:copy-code}
```
### Open ThingsBoard Edge UI
Once started, you will be able to open ThingsBoard Edge UI using the following link http://localhost:8080.
If during installation process you have changed edge HTTP_BIND_PORT please use that port instead for Edge UI URL:
http://localhost:HTTP_BIND_PORT

View File

@ -131,6 +131,7 @@ import org.thingsboard.server.queue.discovery.PartitionService;
import org.thingsboard.server.queue.provider.TbQueueProducerProvider;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.component.ComponentDiscoveryService;
import org.thingsboard.server.service.edge.EdgeInstallService;
import org.thingsboard.server.service.edge.EdgeNotificationService;
import org.thingsboard.server.service.edge.rpc.EdgeRpcService;
import org.thingsboard.server.service.entitiy.TbNotificationEntityService;
@ -281,6 +282,9 @@ public abstract class BaseController {
@Autowired(required = false)
protected EdgeRpcService edgeRpcService;
@Autowired(required = false)
protected EdgeInstallService edgeInstallService;
@Autowired
protected TbNotificationEntityService notificationEntityService;

View File

@ -63,6 +63,7 @@ import org.thingsboard.server.service.security.model.SecurityUser;
import org.thingsboard.server.service.security.permission.Operation;
import org.thingsboard.server.service.security.permission.Resource;
import javax.servlet.http.HttpServletRequest;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
@ -595,4 +596,24 @@ public class EdgeController extends BaseController {
return edgeBulkImportService.processBulkImport(request, user);
}
@ApiOperation(value = "Get Edge Docker Install Instructions (getEdgeDockerInstallInstructions)",
notes = "Get a docker install instructions for provided edge id." + TENANT_AUTHORITY_PARAGRAPH,
produces = MediaType.APPLICATION_JSON_VALUE)
@PreAuthorize("hasAnyAuthority('TENANT_ADMIN')")
@RequestMapping(value = "/edge/instructions/{edgeId}", method = RequestMethod.GET)
@ResponseBody
public String getEdgeDockerInstallInstructions(
@ApiParam(value = EDGE_ID_PARAM_DESCRIPTION, required = true)
@PathVariable("edgeId") String strEdgeId,
HttpServletRequest request) throws ThingsboardException {
try {
EdgeId edgeId = new EdgeId(toUUID(strEdgeId));
edgeId = checkNotNull(edgeId);
Edge edge = checkEdgeId(edgeId, Operation.READ);
return checkNotNull(edgeInstallService.getDockerInstallInstructions(getTenantId(), edge, request));
} catch (Exception e) {
throw handleException(e);
}
}
}

View File

@ -0,0 +1,73 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.edge;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Service;
import org.thingsboard.server.common.data.edge.Edge;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.install.InstallScripts;
import javax.servlet.http.HttpServletRequest;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
@Service
@Slf4j
@RequiredArgsConstructor
@ConditionalOnProperty(prefix = "edges", value = "enabled", havingValue = "true")
@TbCoreComponent
public class DefaultEdgeInstallService implements EdgeInstallService {
private static final String EDGE_DIR = "edge";
private static final String EDGE_INSTALL_INSTRUCTIONS_DIR = "install_instructions";
private final InstallScripts installScripts;
@Override
public String getDockerInstallInstructions(TenantId tenantId, Edge edge, HttpServletRequest request) {
String baseUrl = request.getServerName();
String template = readFile(resolveFile("docker", "instructions.md"));
template = template.replace("${BASE_URL}", baseUrl);
template = template.replace("${CLOUD_ROUTING_KEY}", edge.getRoutingKey());
template = template.replace("${CLOUD_ROUTING_SECRET}", edge.getSecret());
return template;
}
private String readFile(Path file) {
try {
return new String(Files.readAllBytes(file), StandardCharsets.UTF_8);
} catch (IOException e) {
log.warn("Failed to read file: {}", file, e);
throw new RuntimeException(e);
}
}
private Path resolveFile(String subDir, String... subDirs) {
return getEdgeInstallInstructionsDir().resolve(Paths.get(subDir, subDirs));
}
private Path getEdgeInstallInstructionsDir() {
return Paths.get(installScripts.getDataDir(), InstallScripts.JSON_DIR, EDGE_DIR, EDGE_INSTALL_INSTRUCTIONS_DIR);
}
}

View File

@ -0,0 +1,27 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.edge;
import org.thingsboard.server.common.data.edge.Edge;
import org.thingsboard.server.common.data.id.TenantId;
import javax.servlet.http.HttpServletRequest;
public interface EdgeInstallService {
String getDockerInstallInstructions(TenantId tenantId, Edge edge, HttpServletRequest request);
}

View File

@ -187,10 +187,7 @@ public final class EdgeGrpcSession implements Closeable {
public void startSyncProcess(TenantId tenantId, EdgeId edgeId, boolean fullSync) {
log.trace("[{}][{}] Staring edge sync process", tenantId, edgeId);
syncCompleted = false;
if (sessionState.getSendDownlinkMsgsFuture() != null && sessionState.getSendDownlinkMsgsFuture().isDone()) {
String errorMsg = String.format("[%s][%s] Sync process started. General processing interrupted!", tenantId, edgeId);
sessionState.getSendDownlinkMsgsFuture().setException(new RuntimeException(errorMsg));
}
interruptGeneralProcessingOnSync(tenantId, edgeId);
doSync(new EdgeSyncCursor(ctx, edge, fullSync));
}
@ -265,10 +262,7 @@ public final class EdgeGrpcSession implements Closeable {
}
if (sessionState.getPendingMsgsMap().isEmpty()) {
log.debug("[{}] Pending msgs map is empty. Stopping current iteration", edge.getRoutingKey());
if (sessionState.getScheduledSendDownlinkTask() != null) {
sessionState.getScheduledSendDownlinkTask().cancel(false);
}
sessionState.getSendDownlinkMsgsFuture().set(null);
stopCurrentSendDownlinkMsgsTask(null);
}
} catch (Exception e) {
log.error("[{}] Can't process downlink response message [{}]", this.sessionId, msg, e);
@ -391,15 +385,14 @@ public final class EdgeGrpcSession implements Closeable {
}
private ListenableFuture<Void> sendDownlinkMsgsPack(List<DownlinkMsg> downlinkMsgsPack) {
if (sessionState.getSendDownlinkMsgsFuture() != null && !sessionState.getSendDownlinkMsgsFuture().isDone()) {
String errorMsg = "[" + this.sessionId + "] Previous send downlink future was not properly completed, stopping it now";
log.error(errorMsg);
sessionState.getSendDownlinkMsgsFuture().setException(new RuntimeException(errorMsg));
}
interruptPreviousSendDownlinkMsgsTask();
sessionState.setSendDownlinkMsgsFuture(SettableFuture.create());
sessionState.getPendingMsgsMap().clear();
downlinkMsgsPack.forEach(msg -> sessionState.getPendingMsgsMap().put(msg.getDownlinkMsgId(), msg));
scheduleDownlinkMsgsPackSend(1);
return sessionState.getSendDownlinkMsgsFuture();
}
@ -422,13 +415,13 @@ public final class EdgeGrpcSession implements Closeable {
} else {
log.warn("[{}] Failed to deliver the batch after {} attempts. Next messages are going to be discarded {}",
this.sessionId, MAX_DOWNLINK_ATTEMPTS, copy);
sessionState.getSendDownlinkMsgsFuture().set(null);
stopCurrentSendDownlinkMsgsTask(null);
}
} else {
sessionState.getSendDownlinkMsgsFuture().set(null);
stopCurrentSendDownlinkMsgsTask(null);
}
} catch (Exception e) {
sessionState.getSendDownlinkMsgsFuture().setException(e);
stopCurrentSendDownlinkMsgsTask(e);
}
};
@ -673,4 +666,29 @@ public final class EdgeGrpcSession implements Closeable {
log.debug("[{}] Failed to close output stream: {}", sessionId, e.getMessage());
}
}
private void interruptPreviousSendDownlinkMsgsTask() {
String msg = String.format("[%s] Previous send downlink future was not properly completed, stopping it now!", this.sessionId);
stopCurrentSendDownlinkMsgsTask(new RuntimeException(msg));
}
private void interruptGeneralProcessingOnSync(TenantId tenantId, EdgeId edgeId) {
String msg = String.format("[%s][%s] Sync process started. General processing interrupted!", tenantId, edgeId);
stopCurrentSendDownlinkMsgsTask(new RuntimeException(msg));
}
public void stopCurrentSendDownlinkMsgsTask(Exception e) {
if (sessionState.getSendDownlinkMsgsFuture() != null && !sessionState.getSendDownlinkMsgsFuture().isDone()) {
if (e != null) {
log.warn(e.getMessage(), e);
sessionState.getSendDownlinkMsgsFuture().setException(e);
} else {
sessionState.getSendDownlinkMsgsFuture().set(null);
}
}
if (sessionState.getScheduledSendDownlinkTask() != null) {
sessionState.getScheduledSendDownlinkTask().cancel(true);
}
}
}

View File

@ -19,6 +19,7 @@ import com.google.common.util.concurrent.SettableFuture;
import lombok.Data;
import org.thingsboard.server.gen.edge.v1.DownlinkMsg;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.concurrent.ScheduledFuture;
@ -26,7 +27,7 @@ import java.util.concurrent.ScheduledFuture;
@Data
public class EdgeSessionState {
private final Map<Integer, DownlinkMsg> pendingMsgsMap = new LinkedHashMap<>();
private final Map<Integer, DownlinkMsg> pendingMsgsMap = Collections.synchronizedMap(new LinkedHashMap<>());
private SettableFuture<Void> sendDownlinkMsgsFuture;
private ScheduledFuture<?> scheduledSendDownlinkTask;
}

View File

@ -930,7 +930,7 @@ edges:
storage:
max_read_records_count: "${EDGES_STORAGE_MAX_READ_RECORDS_COUNT:50}"
no_read_records_sleep: "${EDGES_NO_READ_RECORDS_SLEEP:1000}"
sleep_between_batches: "${EDGES_SLEEP_BETWEEN_BATCHES:1000}"
sleep_between_batches: "${EDGES_SLEEP_BETWEEN_BATCHES:10000}"
scheduler_pool_size: "${EDGES_SCHEDULER_POOL_SIZE:1}"
send_scheduler_pool_size: "${EDGES_SEND_SCHEDULER_POOL_SIZE:1}"
grpc_callback_thread_pool_size: "${EDGES_GRPC_CALLBACK_POOL_SIZE:1}"

View File

@ -701,16 +701,16 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest {
}
protected Edge constructEdge(String name, String type) {
return constructEdge(tenantId, name, type);
return constructEdge(tenantId, name, type, StringUtils.randomAlphanumeric(20), StringUtils.randomAlphanumeric(20));
}
protected Edge constructEdge(TenantId tenantId, String name, String type) {
protected Edge constructEdge(TenantId tenantId, String name, String type, String routingKey, String secret) {
Edge edge = new Edge();
edge.setTenantId(tenantId);
edge.setName(name);
edge.setType(type);
edge.setSecret(StringUtils.randomAlphanumeric(20));
edge.setRoutingKey(StringUtils.randomAlphanumeric(20));
edge.setRoutingKey(routingKey);
edge.setSecret(secret);
return edge;
}

View File

@ -57,6 +57,10 @@ import org.thingsboard.server.gen.edge.v1.RuleChainUpdateMsg;
import org.thingsboard.server.gen.edge.v1.UserCredentialsUpdateMsg;
import org.thingsboard.server.gen.edge.v1.UserUpdateMsg;
import java.io.File;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
@ -92,7 +96,7 @@ public abstract class BaseEdgeControllerTest extends AbstractControllerTest {
}
}
@Before
@Before
public void beforeTest() throws Exception {
loginSysAdmin();
@ -327,7 +331,7 @@ public abstract class BaseEdgeControllerTest extends AbstractControllerTest {
String customerIdStr = customerId.getId().toString();
String msgError = msgErrorNoFound("Customer", customerIdStr);
doPost("/api/customer/" + customerIdStr+ "/edge/" + savedEdge.getId().getId().toString())
doPost("/api/customer/" + customerIdStr + "/edge/" + savedEdge.getId().getId().toString())
.andExpect(status().isNotFound())
.andExpect(statusReason(containsString(msgError)));
@ -876,4 +880,15 @@ public abstract class BaseEdgeControllerTest extends AbstractControllerTest {
Edge edge = constructEdge(name, "default");
return doPost("/api/edge", edge, Edge.class);
}
}
@Test
public void testGetEdgeInstallInstructions() throws Exception {
Edge edge = constructEdge(tenantId, "Edge for Test Docker Install Instructions", "default", "7390c3a6-69b0-9910-d155-b90aca4b772e", "l7q4zsjplzwhk16geqxy");
Edge savedEdge = doPost("/api/edge", edge, Edge.class);
String installInstructions = doGet("/api/edge/instructions/" + savedEdge.getId().getId().toString(), String.class);
URL resource = this.getClass().getClassLoader().getResource("edge/install_instructions/docker/expected-install-instructions.md");
File file = new File(resource.toURI());
Assert.assertEquals(new String(Files.readAllBytes(file.toPath()), StandardCharsets.UTF_8), installInstructions);
}
}

View File

@ -439,9 +439,9 @@ abstract public class BaseDeviceEdgeTest extends AbstractEdgeTest {
Assert.assertTrue(edgeImitator.waitForResponses());
Assert.assertTrue(edgeImitator.waitForMessages());
AbstractMessage latestMessage = edgeImitator.getMessageFromTail(2);
Assert.assertTrue(latestMessage instanceof DeviceUpdateMsg);
DeviceUpdateMsg latestDeviceUpdateMsg = (DeviceUpdateMsg) latestMessage;
Optional<DeviceUpdateMsg> deviceUpdateMsgOpt = edgeImitator.findMessageByType(DeviceUpdateMsg.class);
Assert.assertTrue(deviceUpdateMsgOpt.isPresent());
DeviceUpdateMsg latestDeviceUpdateMsg = deviceUpdateMsgOpt.get();
Assert.assertNotEquals(deviceOnCloudName, latestDeviceUpdateMsg.getName());
Assert.assertEquals(deviceOnCloudName, latestDeviceUpdateMsg.getConflictName());
@ -453,9 +453,9 @@ abstract public class BaseDeviceEdgeTest extends AbstractEdgeTest {
Assert.assertNotNull(device);
Assert.assertNotEquals(deviceOnCloudName, device.getName());
latestMessage = edgeImitator.getLatestMessage();
Assert.assertTrue(latestMessage instanceof DeviceCredentialsRequestMsg);
DeviceCredentialsRequestMsg latestDeviceCredentialsRequestMsg = (DeviceCredentialsRequestMsg) latestMessage;
Optional<DeviceCredentialsRequestMsg> deviceCredentialsUpdateMsgOpt = edgeImitator.findMessageByType(DeviceCredentialsRequestMsg.class);
Assert.assertTrue(deviceCredentialsUpdateMsgOpt.isPresent());
DeviceCredentialsRequestMsg latestDeviceCredentialsRequestMsg = deviceCredentialsUpdateMsgOpt.get();
Assert.assertEquals(uuid.getMostSignificantBits(), latestDeviceCredentialsRequestMsg.getDeviceIdMSB());
Assert.assertEquals(uuid.getLeastSignificantBits(), latestDeviceCredentialsRequestMsg.getDeviceIdLSB());

View File

@ -34,7 +34,7 @@ abstract public class BaseTelemetryEdgeTest extends AbstractEdgeTest {
@Test
public void testTimeseriesWithFailures() throws Exception {
int numberOfTimeseriesToSend = 1000;
int numberOfTimeseriesToSend = 333;
Device device = findDeviceByName("Edge Device 1");

View File

@ -365,11 +365,7 @@ public class EdgeImitator {
}
public AbstractMessage getLatestMessage() {
return getMessageFromTail(1);
}
public AbstractMessage getMessageFromTail(int offset) {
return downlinkMsgs.get(downlinkMsgs.size() - offset);
return downlinkMsgs.get(downlinkMsgs.size() - 1);
}
public void ignoreType(Class<? extends AbstractMessage> type) {

View File

@ -0,0 +1,88 @@
## Install edge and connect to cloud instructions
### Localhost warning
Localhost cannot be used for docker install - please update baseUrl to the IP address of your machine!
Here is the list of commands, that can be used to quickly install and connect edge to the cloud using docker-compose.
### Prerequisites
Install <a href="https://docs.docker.com/engine/install/" target="_blank"> Docker CE</a> and <a href="https://docs.docker.com/compose/install/" target="_blank"> Docker Compose</a>.
### Create data and logs folders
Run following commands to create a directory for storing data and logs and then change its owner to docker container user, to be able to change user, chown command is used, which requires sudo permissions (command will request password for a sudo access):
```bash
mkdir -p ~/.mytb-edge-data && sudo chown -R 799:799 ~/.mytb-edge-data
mkdir -p ~/.mytb-edge-logs && sudo chown -R 799:799 ~/.mytb-edge-logs
{:copy-code}
```
### Running ThingsBoard Edge as docker service
Create docker compose file for ThingsBoard Edge service:
```bash
nano docker-compose.yml
{:copy-code}
```
Add the following lines to the yml file:
```
version: '2.2'
services:
mytbedge:
restart: always
image: "thingsboard/tb-edge:3.4.1EDGE"
ports:
- "8080:8080"
- "1883:1883"
- "5683-5688:5683-5688/udp"
environment:
SPRING_DATASOURCE_URL: jdbc:postgresql://postgres:5432/tb-edge
CLOUD_ROUTING_KEY: 7390c3a6-69b0-9910-d155-b90aca4b772e
CLOUD_ROUTING_SECRET: l7q4zsjplzwhk16geqxy
CLOUD_RPC_HOST: localhost
volumes:
- ~/.mytb-edge-data:/data
- ~/.mytb-edge-logs:/var/log/tb-edge
postgres:
restart: always
image: "postgres:12"
ports:
- "5432"
environment:
POSTGRES_DB: tb-edge
POSTGRES_PASSWORD: postgres
volumes:
- ~/.mytb-edge-data/db:/var/lib/postgresql/data
{:copy-code}
```
### Ports warning [Optional]
If ThingsBoard Edge is going to be running on the same machine where ThingsBoard server is running you’ll need to update docker compose port mapping.
Please update next lines of docker compose:
ports:
- “18080:8080”
- “11883:1883”
- “15683-15688:5683-5688/udp”
Please make sure ports above are not used by any other application.
Execute the following commands to up this docker compose directly:
```bash
docker-compose pull
docker-compose up
{:copy-code}
```
### Open ThingsBoard Edge UI
Once started, you will be able to open ThingsBoard Edge UI using the following link http://localhost:8080.
If during installation process you have changed edge HTTP_BIND_PORT please use that port instead for Edge UI URL:
http://localhost:HTTP_BIND_PORT

View File

@ -5,7 +5,7 @@
"ng": "ng",
"start": "node --max_old_space_size=8048 ./node_modules/@angular/cli/bin/ng serve --configuration development --host 0.0.0.0 --open",
"build": "ng build",
"build:prod": "ng build --configuration production --vendor-chunk",
"build:prod": "node --max_old_space_size=3072 ./node_modules/@angular/cli/bin/ng build --configuration production --vendor-chunk",
"build:types": "node generate-types.js",
"test": "ng test",
"lint": "ng lint",