dendi-wang/thingsboard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge with feature branch

Browse Source
This commit is contained in:
Andrii Shvaika 2022-05-24 12:32:59 +03:00
commit 49e91341de
20 changed files with 255 additions and 136 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
application/src/main/java/org/thingsboard/server/controller/AdminController.java
View File

@ -203,6 +203,20 @@ public class AdminController extends BaseController {
}
}
@ApiOperation(value = "Check version control settings exists (versionControlSettingsExists)",
notes = "Check whether the version control settings exists. " + TENANT_AUTHORITY_PARAGRAPH)
@PreAuthorize("hasAuthority('TENANT_ADMIN')")
@GetMapping("/vcSettings/exists")
@ResponseBody
public Boolean versionControlSettingsExists() throws ThingsboardException {
try {
accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.READ);
return versionControlService.getVersionControlSettings(getTenantId()) != null;
} catch (Exception e) {
throw handleException(e);
}
}
@ApiOperation(value = "Creates or Updates the version control settings (saveVersionControlSettings)",
notes = "Creates or Updates the version control settings object. " + TENANT_AUTHORITY_PARAGRAPH)
@PreAuthorize("hasAuthority('TENANT_ADMIN')")

64
application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java
View File

@ -15,17 +15,21 @@
*/
package org.thingsboard.server.controller;
import com.google.common.util.concurrent.FutureCallback;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.MoreExecutors;
import io.swagger.annotations.ApiOperation;
import io.swagger.annotations.ApiParam;
import lombok.Data;
import lombok.RequiredArgsConstructor;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.context.request.async.DeferredResult;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.StringUtils;
@ -33,25 +37,28 @@ import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.EntityIdFactory;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.dao.device.claim.ClaimResponse;
import org.thingsboard.server.dao.device.claim.ClaimResult;
import org.thingsboard.server.service.security.model.SecurityUser;
import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.sync.vc.EntityVersion;
import org.thingsboard.server.common.data.sync.vc.VersionCreationResult;
import org.thingsboard.server.common.data.sync.vc.VersionLoadResult;
import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo;
import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest;
import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadRequest;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.security.model.SecurityUser;
import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService;
import javax.annotation.Nullable;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
import static org.thingsboard.server.controller.ControllerConstants.NEW_LINE;
import static org.thingsboard.server.controller.ControllerConstants.PAGE_NUMBER_DESCRIPTION;
import static org.thingsboard.server.controller.ControllerConstants.PAGE_SIZE_DESCRIPTION;
@RestController
@TbCoreComponent
@RequestMapping("/api/entities/vc")
@PreAuthorize("hasAuthority('TENANT_ADMIN')")
@RequiredArgsConstructor
@ -117,13 +124,18 @@ public class EntitiesVersionControlController extends BaseController {
" \"name\": \"Device profile 1 version 1.0\"\n" +
" }\n" +
"]\n```")
@GetMapping("/version/{branch}/{entityType}/{externalEntityUuid}")
public DeferredResult<List<EntityVersion>> listEntityVersions(@PathVariable String branch,
@PathVariable EntityType entityType,
@PathVariable UUID externalEntityUuid) throws ThingsboardException {
@GetMapping(value = "/version/{branch}/{entityType}/{externalEntityUuid}", params = {"pageSize", "page"})
public DeferredResult<PageData<EntityVersion>> listEntityVersions(@PathVariable String branch,
@PathVariable EntityType entityType,
@PathVariable UUID externalEntityUuid,
@ApiParam(value = PAGE_SIZE_DESCRIPTION, required = true)
@RequestParam int pageSize,
@ApiParam(value = PAGE_NUMBER_DESCRIPTION, required = true)
@RequestParam int page) throws ThingsboardException {
try {
EntityId externalEntityId = EntityIdFactory.getByTypeAndUuid(entityType, externalEntityUuid);
return wrapFuture(versionControlService.listEntityVersions(getTenantId(), branch, externalEntityId));
PageLink pageLink = new PageLink(pageSize, page);
return wrapFuture(versionControlService.listEntityVersions(getTenantId(), branch, externalEntityId, pageLink));
} catch (Exception e) {
throw handleException(e);
}
@ -136,11 +148,16 @@ public class EntitiesVersionControlController extends BaseController {
" \"name\": \"Device profiles from dev\"\n" +
" }\n" +
"]\n```")
@GetMapping("/version/{branch}/{entityType}")
public DeferredResult<List<EntityVersion>> listEntityTypeVersions(@PathVariable String branch,
@PathVariable EntityType entityType) throws ThingsboardException {
@GetMapping(value = "/version/{branch}/{entityType}", params = {"pageSize", "page"})
public DeferredResult<PageData<EntityVersion>> listEntityTypeVersions(@PathVariable String branch,
@PathVariable EntityType entityType,
@ApiParam(value = PAGE_SIZE_DESCRIPTION, required = true)
@RequestParam int pageSize,
@ApiParam(value = PAGE_NUMBER_DESCRIPTION, required = true)
@RequestParam int page) throws ThingsboardException {
try {
return wrapFuture(versionControlService.listEntityTypeVersions(getTenantId(), branch, entityType));
PageLink pageLink = new PageLink(pageSize, page);
return wrapFuture(versionControlService.listEntityTypeVersions(getTenantId(), branch, entityType, pageLink));
} catch (Exception e) {
throw handleException(e);
}
@ -161,10 +178,15 @@ public class EntitiesVersionControlController extends BaseController {
" \"name\": \"Devices added\"\n" +
" }\n" +
"]\n```")
@GetMapping("/version/{branch}")
public DeferredResult<List<EntityVersion>> listVersions(@PathVariable String branch) throws ThingsboardException {
@GetMapping(value = "/version/{branch}", params = {"pageSize", "page"})
public DeferredResult<PageData<EntityVersion>> listVersions(@PathVariable String branch,
@ApiParam(value = PAGE_SIZE_DESCRIPTION, required = true)
@RequestParam int pageSize,
@ApiParam(value = PAGE_NUMBER_DESCRIPTION, required = true)
@RequestParam int page) throws ThingsboardException {
try {
return wrapFuture(versionControlService.listVersions(getTenantId(), branch));
PageLink pageLink = new PageLink(pageSize, page);
return wrapFuture(versionControlService.listVersions(getTenantId(), branch, pageLink));
} catch (Exception e) {
throw handleException(e);
}

27
application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java
View File

@ -37,6 +37,9 @@ import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.EntityIdFactory;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.sync.ThrowingRunnable;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.sync.ThrowingRunnable;
import org.thingsboard.server.common.data.sync.ie.EntityExportData;
import org.thingsboard.server.common.data.sync.ie.EntityExportSettings;
import org.thingsboard.server.common.data.sync.ie.EntityImportResult;
@ -160,18 +163,18 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
}
@Override
public ListenableFuture<List<EntityVersion>> listEntityVersions(TenantId tenantId, String branch, EntityId externalId) throws Exception {
return gitServiceQueue.listVersions(tenantId, branch, externalId);
public ListenableFuture<PageData<EntityVersion>> listEntityVersions(TenantId tenantId, String branch, EntityId externalId, PageLink pageLink) throws Exception {
return gitServiceQueue.listVersions(tenantId, branch, externalId, pageLink);
}
@Override
public ListenableFuture<List<EntityVersion>> listEntityTypeVersions(TenantId tenantId, String branch, EntityType entityType) throws Exception {
return gitServiceQueue.listVersions(tenantId, branch, entityType);
public ListenableFuture<PageData<EntityVersion>> listEntityTypeVersions(TenantId tenantId, String branch, EntityType entityType, PageLink pageLink) throws Exception {
return gitServiceQueue.listVersions(tenantId, branch, entityType, pageLink);
}
@Override
public ListenableFuture<List<EntityVersion>> listVersions(TenantId tenantId, String branch) throws Exception {
return gitServiceQueue.listVersions(tenantId, branch);
public ListenableFuture<PageData<EntityVersion>> listVersions(TenantId tenantId, String branch, PageLink pageLink) throws Exception {
return gitServiceQueue.listVersions(tenantId, branch, pageLink);
}
@Override
@ -336,6 +339,12 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
adminSettings.setKey(SETTINGS_KEY);
adminSettings.setTenantId(tenantId);
}
try {
//TODO: ashvayka: replace future.get with deferred result. Don't forget to call when tenant is deleted.
gitServiceQueue.initRepository(tenantId, versionControlSettings).get();
} catch (Exception e) {
throw new RuntimeException("Failed to init repository!", e);
}
adminSettings.setJsonValue(JacksonUtil.valueToTree(versionControlSettings));
AdminSettings savedAdminSettings = adminSettingsService.saveAdminSettings(tenantId, adminSettings);
EntitiesVersionControlSettings savedVersionControlSettings;
@ -344,12 +353,6 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
} catch (Exception e) {
throw new RuntimeException("Failed to load version control settings!", e);
}
try {
//TODO: ashvayka: replace future.get with deferred result. Don't forget to call when tenant is deleted.
gitServiceQueue.initRepository(tenantId, versionControlSettings).get();
} catch (Exception e) {
throw new RuntimeException("Failed to init repository!", e);
}
return savedVersionControlSettings;
}

47
application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java
View File

@ -16,15 +16,14 @@
package org.thingsboard.server.service.sync.vc;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectWriter;
import com.fasterxml.jackson.databind.SerializationFeature;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.SettableFuture;
import com.google.protobuf.ByteString;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Service;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.cluster.TbClusterService;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.ExportableEntity;
@ -32,6 +31,8 @@ import org.thingsboard.server.common.data.StringUtils;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.EntityIdFactory;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.sync.ie.EntityExportData;
import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings;
import org.thingsboard.server.common.data.sync.vc.EntityVersion;
@ -73,7 +74,7 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu
private final DefaultEntitiesVersionControlService entitiesVersionControlService;
private final Map<UUID, PendingGitRequest<?>> pendingRequestMap = new HashMap<>();
private final ObjectWriter jsonWriter = new ObjectMapper().writer(SerializationFeature.INDENT_OUTPUT);
private final ObjectMapper jsonMapper = new ObjectMapper().enable(SerializationFeature.INDENT_OUTPUT);
public DefaultGitVersionControlQueueService(TbServiceInfoProvider serviceInfoProvider, TbClusterService clusterService,
DataDecodingEncodingService encodingService,
@ -102,7 +103,7 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu
String path = getRelativePath(entityData.getEntityType(), entityData.getEntity().getId());
String entityDataJson;
try {
entityDataJson = jsonWriter.writeValueAsString(entityData);
entityDataJson = jsonMapper.writeValueAsString(entityData);
} catch (IOException e) {
//TODO: analyze and return meaningful exceptions that we can show to the client;
throw new RuntimeException(e);
@ -144,29 +145,36 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu
}
@Override
public ListenableFuture<List<EntityVersion>> listVersions(TenantId tenantId, String branch) {
public ListenableFuture<PageData<EntityVersion>> listVersions(TenantId tenantId, String branch, PageLink pageLink) {
return listVersions(tenantId, ListVersionsRequestMsg.newBuilder()
.setBranchName(branch).build());
}
@Override
public ListenableFuture<List<EntityVersion>> listVersions(TenantId tenantId, String branch, EntityType entityType) {
return listVersions(tenantId, ListVersionsRequestMsg.newBuilder()
.setBranchName(branch).setEntityType(entityType.name())
.setBranchName(branch)
.setPageSize(pageLink.getPageSize())
.setPage(pageLink.getPage())
.build());
}
@Override
public ListenableFuture<List<EntityVersion>> listVersions(TenantId tenantId, String branch, EntityId entityId) {
public ListenableFuture<PageData<EntityVersion>> listVersions(TenantId tenantId, String branch, EntityType entityType, PageLink pageLink) {
return listVersions(tenantId, ListVersionsRequestMsg.newBuilder()
.setBranchName(branch).setEntityType(entityType.name())
.setPageSize(pageLink.getPageSize())
.setPage(pageLink.getPage())
.build());
}
@Override
public ListenableFuture<PageData<EntityVersion>> listVersions(TenantId tenantId, String branch, EntityId entityId, PageLink pageLink) {
return listVersions(tenantId, ListVersionsRequestMsg.newBuilder()
.setBranchName(branch)
.setEntityType(entityId.getEntityType().name())
.setEntityIdMSB(entityId.getId().getMostSignificantBits())
.setEntityIdLSB(entityId.getId().getLeastSignificantBits())
.setPageSize(pageLink.getPageSize())
.setPage(pageLink.getPage())
.build());
}
private ListenableFuture<List<EntityVersion>> listVersions(TenantId tenantId, ListVersionsRequestMsg requestMsg) {
private ListenableFuture<PageData<EntityVersion>> listVersions(TenantId tenantId, ListVersionsRequestMsg requestMsg) {
ListVersionsGitRequest request = new ListVersionsGitRequest(tenantId);
registerAndSend(request, builder -> builder.setListVersionRequest(requestMsg).build(), wrap(request.getFuture()));
@ -318,8 +326,7 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu
listEntitiesResponse.getEntitiesList().stream().map(this::getVersionedEntityInfo).collect(Collectors.toList()));
} else if (vcResponseMsg.hasListVersionsResponse()) {
var listVersionsResponse = vcResponseMsg.getListVersionsResponse();
((ListVersionsGitRequest) request).getFuture().set(
listVersionsResponse.getVersionsList().stream().map(this::getEntityVersion).collect(Collectors.toList()));
((ListVersionsGitRequest) request).getFuture().set(toPageData(listVersionsResponse));
} else if (vcResponseMsg.hasEntityContentResponse()) {
var data = vcResponseMsg.getEntityContentResponse().getData();
((EntityContentGitRequest) request).getFuture().set(toData(data));
@ -331,6 +338,11 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu
}
}
private PageData<EntityVersion> toPageData(TransportProtos.ListVersionsResponseMsg listVersionsResponse) {
var listVersions = listVersionsResponse.getVersionsList().stream().map(this::getEntityVersion).collect(Collectors.toList());
return new PageData<>(listVersions, listVersionsResponse.getTotalPages(), listVersionsResponse.getTotalElements(), listVersionsResponse.getHasNext());
}
private EntityVersion getEntityVersion(TransportProtos.EntityVersionProto proto) {
return new EntityVersion(proto.getId(), proto.getName());
}
@ -340,8 +352,9 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu
}
@SuppressWarnings("rawtypes")
@SneakyThrows
private EntityExportData toData(String data) {
return JacksonUtil.fromString(data, EntityExportData.class);
return jsonMapper.readValue(data, EntityExportData.class);
}
private static <T> TbQueueCallback wrap(SettableFuture<T> future) {

8
application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java
View File

@ -20,6 +20,8 @@ import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.service.security.model.SecurityUser;
import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings;
import org.thingsboard.server.common.data.sync.vc.EntityVersion;
@ -38,11 +40,11 @@ public interface EntitiesVersionControlService {
ListenableFuture<VersionCreationResult> saveEntitiesVersion(SecurityUser user, VersionCreateRequest request) throws Exception;
ListenableFuture<List<EntityVersion>> listEntityVersions(TenantId tenantId, String branch, EntityId externalId) throws Exception;
ListenableFuture<PageData<EntityVersion>> listEntityVersions(TenantId tenantId, String branch, EntityId externalId, PageLink pageLink) throws Exception;
ListenableFuture<List<EntityVersion>> listEntityTypeVersions(TenantId tenantId, String branch, EntityType entityType) throws Exception;
ListenableFuture<PageData<EntityVersion>> listEntityTypeVersions(TenantId tenantId, String branch, EntityType entityType, PageLink pageLink) throws Exception;
ListenableFuture<List<EntityVersion>> listVersions(TenantId tenantId, String branch) throws Exception;
ListenableFuture<PageData<EntityVersion>> listVersions(TenantId tenantId, String branch, PageLink pageLink) throws Exception;
ListenableFuture<List<VersionedEntityInfo>> listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType) throws Exception;

8
application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java
View File

@ -20,6 +20,8 @@ import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.ExportableEntity;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.sync.ie.EntityExportData;
import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings;
import org.thingsboard.server.common.data.sync.vc.EntityVersion;
@ -40,11 +42,11 @@ public interface GitVersionControlQueueService {
ListenableFuture<VersionCreationResult> push(CommitGitRequest commit);
ListenableFuture<List<EntityVersion>> listVersions(TenantId tenantId, String branch);
ListenableFuture<PageData<EntityVersion>> listVersions(TenantId tenantId, String branch, PageLink pageLink);
ListenableFuture<List<EntityVersion>> listVersions(TenantId tenantId, String branch, EntityType entityType);
ListenableFuture<PageData<EntityVersion>> listVersions(TenantId tenantId, String branch, EntityType entityType, PageLink pageLink);
ListenableFuture<List<EntityVersion>> listVersions(TenantId tenantId, String branch, EntityId entityId);
ListenableFuture<PageData<EntityVersion>> listVersions(TenantId tenantId, String branch, EntityId entityId, PageLink pageLink);
ListenableFuture<List<VersionedEntityInfo>> listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType);

3
application/src/main/java/org/thingsboard/server/service/sync/vc/ListVersionsGitRequest.java
View File

@ -16,13 +16,14 @@
package org.thingsboard.server.service.sync.vc;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.sync.vc.EntityVersion;
import org.thingsboard.server.common.data.sync.vc.VersionCreationResult;
import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest;
import java.util.List;
public class ListVersionsGitRequest extends PendingGitRequest<List<EntityVersion>> {
public class ListVersionsGitRequest extends PendingGitRequest<PageData<EntityVersion>> {
public ListVersionsGitRequest(TenantId tenantId) {
super(tenantId);

2
application/src/main/resources/thingsboard.yml
View File

@ -1122,8 +1122,8 @@ metrics:
vc:
thread_pool_size: "${TB_VC_POOL_SIZE:4}"
git:
folder: "${TB_VC_GIT_FOLDER:}"
repos-poll-interval: "${TB_VC_GIT_REPOS_POLL_INTERVAL_SEC:60}"
repositories-folder: "${TB_VC_GIT_REPOSITORIES_FOLDER:${java.io.tmpdir}/repositories}"
management:
endpoints:

5
common/cluster-api/src/main/proto/queue.proto
View File

@ -721,6 +721,8 @@ message ListVersionsRequestMsg {
string entityType = 2;
int64 entityIdMSB = 3;
int64 entityIdLSB = 4;
int32 pageSize = 5;
int32 page = 6;
}
message EntityVersionProto {
@ -731,6 +733,9 @@ message EntityVersionProto {
message ListVersionsResponseMsg {
repeated EntityVersionProto versions = 1;
int32 totalPages = 2;
int64 totalElements = 3;
bool hasNext = 4;
}
message ListEntitiesRequestMsg {

3
common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java
View File

@ -20,6 +20,7 @@ import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonSubTypes;
import com.fasterxml.jackson.annotation.JsonSubTypes.Type;
import com.fasterxml.jackson.annotation.JsonTypeInfo;
import com.fasterxml.jackson.annotation.JsonTypeInfo.As;
import lombok.Data;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.ExportableEntity;
@ -30,7 +31,7 @@ import org.thingsboard.server.common.data.sync.JsonTbEntity;
import java.util.List;
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "entityType", defaultImpl = EntityExportData.class)
@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "entityType", include = As.EXISTING_PROPERTY, visible = true, defaultImpl = EntityExportData.class)
@JsonSubTypes({
@Type(name = "DEVICE", value = DeviceExportData.class),
@Type(name = "RULE_CHAIN", value = RuleChainExportData.class)

6
common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportResult.java
View File

@ -15,26 +15,20 @@
*/
package org.thingsboard.server.common.data.sync.ie;
import com.fasterxml.jackson.annotation.JsonIgnore;
import lombok.Data;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.ExportableEntity;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.sync.JsonTbEntity;
import org.thingsboard.server.common.data.sync.ThrowingRunnable;
@Data
public class EntityImportResult<E extends ExportableEntity<? extends EntityId>> {
@JsonTbEntity
private E savedEntity;
@JsonTbEntity
private E oldEntity;
private EntityType entityType;
@JsonIgnore
private ThrowingRunnable saveReferencesCallback = () -> {};
@JsonIgnore
private ThrowingRunnable sendEventsCallback = () -> {};
public void addSaveReferencesCallback(ThrowingRunnable callback) {

2
common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntitiesVersionControlSettings.java
View File

@ -31,4 +31,4 @@ public class EntitiesVersionControlSettings implements Serializable {
private String privateKey;
private String privateKeyPassword;
private String defaultBranch;
}
}

1
common/version-control/pom.xml
View File

@ -68,7 +68,6 @@
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>

2
common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/ClusterVersionControlService.java
View File

@ -15,8 +15,8 @@
*/
package org.thingsboard.server.service.sync.vc;
import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent;
import org.springframework.context.ApplicationListener;
import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent;
public interface ClusterVersionControlService extends ApplicationListener<PartitionChangeEvent> {
}

34
common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java
View File

@ -26,15 +26,33 @@ import org.thingsboard.common.util.ThingsBoardThreadFactory;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.StringUtils;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings;
import org.thingsboard.server.common.data.sync.vc.VersionCreationResult;
import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo;
import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.gen.transport.TransportProtos.*;
import org.thingsboard.server.gen.transport.TransportProtos.AddMsg;
import org.thingsboard.server.gen.transport.TransportProtos.CommitRequestMsg;
import org.thingsboard.server.gen.transport.TransportProtos.CommitResponseMsg;
import org.thingsboard.server.gen.transport.TransportProtos.DeleteMsg;
import org.thingsboard.server.gen.transport.TransportProtos.EntitiesContentRequestMsg;
import org.thingsboard.server.gen.transport.TransportProtos.EntitiesContentResponseMsg;
import org.thingsboard.server.gen.transport.TransportProtos.EntityContentRequestMsg;
import org.thingsboard.server.gen.transport.TransportProtos.EntityContentResponseMsg;
import org.thingsboard.server.gen.transport.TransportProtos.EntityVersionProto;
import org.thingsboard.server.gen.transport.TransportProtos.ListBranchesRequestMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ListBranchesResponseMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ListEntitiesRequestMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ListEntitiesResponseMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ListVersionsRequestMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ListVersionsResponseMsg;
import org.thingsboard.server.gen.transport.TransportProtos.PrepareMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToVersionControlServiceMsg;
import org.thingsboard.server.gen.transport.TransportProtos.VersionControlResponseMsg;
import org.thingsboard.server.gen.transport.TransportProtos.VersionedEntityInfoProto;
import org.thingsboard.server.queue.TbQueueConsumer;
import org.thingsboard.server.queue.TbQueueProducer;
import org.thingsboard.server.queue.common.TbProtoQueueMsg;
@ -220,12 +238,16 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe
} else {
path = null;
}
var data = vcService.listVersions(ctx.getTenantId(), request.getBranchName(), path);
var data = vcService.listVersions(ctx.getTenantId(), request.getBranchName(), path, new PageLink(request.getPageSize(), request.getPage()));
reply(ctx, Optional.empty(), builder ->
builder.setListVersionsResponse(ListVersionsResponseMsg.newBuilder()
.addAllVersions(data.stream().map(
.setTotalPages(data.getTotalPages())
.setTotalElements(data.getTotalElements())
.setHasNext(data.hasNext())
.addAllVersions(data.getData().stream().map(
v -> EntityVersionProto.newBuilder().setId(v.getId()).setName(v.getName()).build()
).collect(Collectors.toList()))));
).collect(Collectors.toList())))
);
}
private void handleListEntities(VersionControlRequestCtx ctx, ListEntitiesRequestMsg request) throws Exception {
@ -266,7 +288,9 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe
} else if (request.hasDeleteMsg()) {
deleteFromCommit(ctx, current, request.getDeleteMsg());
} else if (request.hasPushMsg()) {
reply(ctx, vcService.push(current));
var result = vcService.push(current);
pendingCommitMap.remove(ctx.getTenantId());
reply(ctx, result);
}
} catch (Exception e) {
doAbortCurrentCommit(tenantId, current, e);

65
common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java
View File

@ -15,12 +15,11 @@
*/
package org.thingsboard.server.service.sync.vc;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.eclipse.jgit.api.errors.JGitInternalException;
import org.eclipse.jgit.api.errors.RefAlreadyExistsException;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Service;
@ -28,6 +27,8 @@ import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.EntityIdFactory;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings;
import org.thingsboard.server.common.data.sync.vc.EntityVersion;
import org.thingsboard.server.common.data.sync.vc.VersionCreationResult;
@ -57,7 +58,7 @@ public class DefaultGitRepositoryService implements GitRepositoryService {
@Value("${java.io.tmpdir}/repositories")
private String defaultFolder;
@Value("${vc.git.folder:${java.io.tmpdir}/repositories}")
@Value("${vc.git.repositories-folder:${java.io.tmpdir}/repositories}")
private String repositoriesFolder;
@Value("${vc.git.repos-poll-interval:60}")
@ -97,22 +98,12 @@ public class DefaultGitRepositoryService implements GitRepositoryService {
String branch = commit.getBranch();
try {
repository.fetch();
if (repository.listBranches().contains(branch)) {
repository.checkout("origin/" + branch, false);
try {
repository.checkout(branch, true);
} catch (RefAlreadyExistsException e) {
repository.checkout(branch, false);
}
repository.createAndCheckoutOrphanBranch(commit.getWorkingBranch());
repository.resetAndClean();
if (repository.listRemoteBranches().contains(branch)) {
repository.merge(branch);
} else { // TODO [viacheslav]: rollback orphan branch on failure
try {
repository.createAndCheckoutOrphanBranch(branch); // FIXME [viacheslav]: Checkout returned unexpected result NO_CHANGE for master branch
} catch (JGitInternalException e) {
if (!e.getMessage().contains("NO_CHANGE")) {
throw e;
}
}
}
} catch (IOException | GitAPIException gitAPIException) {
//TODO: analyze and return meaningful exceptions that we can show to the client;
@ -145,24 +136,44 @@ public class DefaultGitRepositoryService implements GitRepositoryService {
result.setRemoved(status.getRemoved().size());
GitRepository.Commit gitCommit = repository.commit(commit.getVersionName());
repository.push();
repository.push(commit.getWorkingBranch(), commit.getBranch());
result.setVersion(toVersion(gitCommit));
return result;
} catch (GitAPIException gitAPIException) {
//TODO: analyze and return meaningful exceptions that we can show to the client;
throw new RuntimeException(gitAPIException);
} finally {
cleanUp(commit);
}
}
@SneakyThrows
@Override
public void cleanUp(PendingCommit commit) {
GitRepository repository = checkRepository(commit.getTenantId());
try {
repository.createAndCheckoutOrphanBranch(EntityId.NULL_UUID.toString());
} catch (Exception e) {
if (!e.getMessage().contains("NO_CHANGE")) {
throw e;
}
}
repository.resetAndClean();
repository.deleteLocalBranchIfExists(commit.getWorkingBranch());
}
@Override
public void abort(PendingCommit commit) {
//TODO: implement;
cleanUp(commit);
}
@Override
public void fetch(TenantId tenantId) {
//Fetch latest changes on demand.
public void fetch(TenantId tenantId) throws GitAPIException {
var repository = repositories.get(tenantId);
if (repository != null) {
repository.fetch();
}
}
@Override
@ -175,7 +186,7 @@ public class DefaultGitRepositoryService implements GitRepositoryService {
public List<String> listBranches(TenantId tenantId) {
GitRepository repository = checkRepository(tenantId);
try {
return repository.listBranches();
return repository.listRemoteBranches();
} catch (GitAPIException gitAPIException) {
//TODO: analyze and return meaningful exceptions that we can show to the client;
throw new RuntimeException(gitAPIException);
@ -183,7 +194,7 @@ public class DefaultGitRepositoryService implements GitRepositoryService {
}
private EntityVersion checkVersion(TenantId tenantId, String branch, String versionId) throws Exception {
return listVersions(tenantId, branch, null).stream()
return listVersions(tenantId, branch, null, new PageLink(Integer.MAX_VALUE)).getData().stream()
.filter(version -> version.getId().equals(versionId))
.findFirst().orElseThrow(() -> new IllegalArgumentException("Version not found"));
}
@ -194,11 +205,9 @@ public class DefaultGitRepositoryService implements GitRepositoryService {
}
@Override
public List<EntityVersion> listVersions(TenantId tenantId, String branch, String path) throws Exception {
public PageData<EntityVersion> listVersions(TenantId tenantId, String branch, String path, PageLink pageLink) throws Exception {
GitRepository repository = checkRepository(tenantId);
return repository.listCommits(branch, path, Integer.MAX_VALUE).stream()
.map(this::toVersion)
.collect(Collectors.toList());
return repository.listCommits(branch, path, pageLink).mapData(this::toVersion);
}
@Override

80
common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java
View File

@ -15,11 +15,20 @@
*/
package org.thingsboard.server.service.sync.vc;
import com.google.common.collect.Iterables;
import com.google.common.collect.Streams;
import lombok.Data;
import lombok.Getter;
import org.apache.commons.lang3.StringUtils;
import org.apache.sshd.common.util.security.SecurityUtils;
import org.eclipse.jgit.api.*;
import org.eclipse.jgit.api.CloneCommand;
import org.eclipse.jgit.api.Git;
import org.eclipse.jgit.api.GitCommand;
import org.eclipse.jgit.api.ListBranchCommand;
import org.eclipse.jgit.api.LogCommand;
import org.eclipse.jgit.api.LsRemoteCommand;
import org.eclipse.jgit.api.ResetCommand;
import org.eclipse.jgit.api.TransportCommand;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.eclipse.jgit.lib.Constants;
import org.eclipse.jgit.lib.ObjectId;
@ -28,6 +37,7 @@ import org.eclipse.jgit.lib.ObjectReader;
import org.eclipse.jgit.revwalk.RevCommit;
import org.eclipse.jgit.revwalk.filter.RevFilter;
import org.eclipse.jgit.transport.CredentialsProvider;
import org.eclipse.jgit.transport.RefSpec;
import org.eclipse.jgit.transport.SshTransport;
import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider;
import org.eclipse.jgit.transport.sshd.JGitKeyCache;
@ -36,7 +46,8 @@ import org.eclipse.jgit.transport.sshd.SshdSessionFactory;
import org.eclipse.jgit.transport.sshd.SshdSessionFactoryBuilder;
import org.eclipse.jgit.treewalk.TreeWalk;
import org.eclipse.jgit.treewalk.filter.PathFilter;
import org.thingsboard.server.common.data.StringUtils;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings;
import org.thingsboard.server.common.data.sync.vc.VersionControlAuthMethod;
@ -51,6 +62,7 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Set;
import java.util.function.Function;
import java.util.stream.Collectors;
public class GitRepository {
@ -119,15 +131,18 @@ public class GitRepository {
.setRemoveDeletedRefs(true));
}
public void checkout(String branch, boolean createBranch) throws GitAPIException {
execute(git.checkout()
.setCreateBranch(createBranch)
.setName(branch));
public void deleteLocalBranchIfExists(String branch) throws GitAPIException {
execute(git.branchDelete()
.setBranchNames(branch)
.setForce(true));
}
public void reset() throws GitAPIException {
public void resetAndClean() throws GitAPIException {
execute(git.reset()
.setMode(ResetCommand.ResetType.HARD));
execute(git.clean()
.setForce(true)
.setCleanDirectories(true));
}
public void merge(String branch) throws IOException, GitAPIException {
@ -140,36 +155,34 @@ public class GitRepository {
}
public List<String> listBranches() throws GitAPIException {
public List<String> listRemoteBranches() throws GitAPIException {
return execute(git.branchList()
.setListMode(ListBranchCommand.ListMode.ALL)).stream()
.setListMode(ListBranchCommand.ListMode.REMOTE)).stream()
.filter(ref -> !ref.getName().equals(Constants.HEAD))
.map(ref -> org.eclipse.jgit.lib.Repository.shortenRefName(ref.getName()))
.map(name -> StringUtils.removeStart(name, "origin/"))
.distinct().collect(Collectors.toList());
}
public List<Commit> listCommits(String branch, int limit) throws IOException, GitAPIException {
return listCommits(branch, null, limit);
public PageData<Commit> listCommits(String branch, PageLink pageLink) throws IOException, GitAPIException {
return listCommits(branch, null, pageLink);
}
public List<Commit> listCommits(String branch, String path, int limit) throws IOException, GitAPIException {
public PageData<Commit> listCommits(String branch, String path, PageLink pageLink) throws IOException, GitAPIException {
ObjectId branchId = resolve("origin/" + branch);
if (branchId == null) {
throw new IllegalArgumentException("Branch not found");
}
LogCommand command = git.log()
.add(branchId).setMaxCount(limit)
.add(branchId)
.setRevFilter(RevFilter.NO_MERGES);
if (StringUtils.isNotEmpty(path)) {
command.addPath(path);
}
return Streams.stream(execute(command))
.map(this::toCommit)
.collect(Collectors.toList());
Iterable<RevCommit> commits = execute(command);
return iterableToPageData(commits, this::toCommit, pageLink);
}
public List<String> listFilesAtCommit(String commitId) throws IOException {
return listFilesAtCommit(commitId, null);
}
@ -212,16 +225,9 @@ public class GitRepository {
.setOrphan(true)
.setForced(true)
.setName(name));
// Set<String> uncommittedChanges = git.status().call().getUncommittedChanges();
// if (!uncommittedChanges.isEmpty()) {
// RmCommand rm = git.rm();
// uncommittedChanges.forEach(rm::addFilepattern);
// execute(rm);
// }
// execute(git.clean());
}
public void add(String filesPattern) throws GitAPIException { // FIXME [viacheslav]
public void add(String filesPattern) throws GitAPIException {
execute(git.add().setUpdate(true).addFilepattern(filesPattern));
execute(git.add().addFilepattern(filesPattern));
}
@ -233,13 +239,14 @@ public class GitRepository {
public Commit commit(String message) throws GitAPIException {
RevCommit revCommit = execute(git.commit()
.setMessage(message)); // TODO [viacheslav]: set configurable author for commit
.setMessage(message));
return toCommit(revCommit);
}
public void push() throws GitAPIException {
execute(git.push());
public void push(String localBranch, String remoteBranch) throws GitAPIException {
execute(git.push()
.setRefSpecs(new RefSpec(localBranch + ":" + remoteBranch)));
}
@ -293,6 +300,23 @@ public class GitRepository {
return command.call();
}
private static <T,R> PageData<R> iterableToPageData (Iterable<T> iterable, Function<? super T, ? extends R> mapper, PageLink pageLink) {
int totalElements = Iterables.size(iterable);
int totalPages = pageLink.getPageSize() > 0 ? (int) Math.ceil((float) totalElements / pageLink.getPageSize()) : 1;
int startIndex = pageLink.getPageSize() * pageLink.getPage();
int limit = startIndex + pageLink.getPageSize();
iterable = Iterables.limit(iterable, limit);
if (startIndex < totalElements) {
iterable = Iterables.skip(iterable, startIndex);
} else {
iterable = Collections.emptyList();
}
List<R> data = Streams.stream(iterable).map(mapper)
.collect(Collectors.toList());
boolean hasNext = pageLink.getPageSize() > 0 && totalElements > startIndex + data.size();
return new PageData<>(data, totalPages, totalElements, hasNext);
}
private static void configureTransportCommand(TransportCommand transportCommand, CredentialsProvider credentialsProvider, SshdSessionFactory sshSessionFactory) {
if (credentialsProvider != null) {
transportCommand.setCredentialsProvider(credentialsProvider);

9
common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java
View File

@ -15,7 +15,10 @@
*/
package org.thingsboard.server.service.sync.vc;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings;
import org.thingsboard.server.common.data.sync.vc.EntityVersion;
import org.thingsboard.server.common.data.sync.vc.VersionCreationResult;
@ -28,7 +31,7 @@ public interface GitRepositoryService {
void prepareCommit(PendingCommit pendingCommit);
List<EntityVersion> listVersions(TenantId tenantId, String branch, String path) throws Exception;
PageData<EntityVersion> listVersions(TenantId tenantId, String branch, String path, PageLink pageLink) throws Exception;
List<VersionedEntityInfo> listEntitiesAtVersion(TenantId tenantId, String versionId, String path) throws Exception;
@ -46,11 +49,13 @@ public interface GitRepositoryService {
VersionCreationResult push(PendingCommit commit);
void cleanUp(PendingCommit commit);
void abort(PendingCommit commit);
List<String> listBranches(TenantId tenantId);
String getFileContentAtCommit(TenantId tenantId, String relativePath, String versionId) throws IOException;
void fetch(TenantId tenantId);
void fetch(TenantId tenantId) throws GitAPIException;
}

3
common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/PendingCommit.java
View File

@ -17,7 +17,6 @@ package org.thingsboard.server.service.sync.vc;
import lombok.Data;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest;
import java.util.UUID;
@ -27,6 +26,7 @@ public class PendingCommit {
private final UUID txId;
private final String nodeId;
private final TenantId tenantId;
private final String workingBranch;
private String branch;
private String versionName;
@ -36,5 +36,6 @@ public class PendingCommit {
this.txId = txId;
this.branch = branch;
this.versionName = versionName;
this.workingBranch = txId.toString();
}
}

8
dao/src/test/resources/sql/system-data.sql
View File

@ -26,13 +26,13 @@ VALUES ( '61441950-4612-11e7-a919-92ebcb67fe33', 1592576748000, '5a797660-4612-1
'$2a$10$5JTB8/hxWc9WAy62nCGSxeefl3KWmipA9nFpVdDa0/xfIseeBB4Bu' );
/** System settings **/
INSERT INTO admin_settings ( id, created_time, key, json_value )
VALUES ( '6a2266e4-4612-11e7-a919-92ebcb67fe33', 1592576748000, 'general', '{
INSERT INTO admin_settings ( id, created_time, tenant_id, key, json_value )
VALUES ( '6a2266e4-4612-11e7-a919-92ebcb67fe33', 1592576748000, '13814000-1dd2-11b2-8080-808080808080', 'general', '{
"baseUrl": "http://localhost:8080"
}' );
INSERT INTO admin_settings ( id, created_time, key, json_value )
VALUES ( '6eaaefa6-4612-11e7-a919-92ebcb67fe33', 1592576748000, 'mail', '{
INSERT INTO admin_settings ( id, created_time, tenant_id, key, json_value )
VALUES ( '6eaaefa6-4612-11e7-a919-92ebcb67fe33', 1592576748000, '13814000-1dd2-11b2-8080-808080808080', 'mail', '{
"mailFrom": "Thingsboard <sysadmin@localhost.localdomain>",
"smtpProtocol": "smtp",
"smtpHost": "localhost",