Merge with feature branch

This commit is contained in:
Andrii Shvaika 2022-05-24 12:32:59 +03:00
commit 49e91341de
20 changed files with 255 additions and 136 deletions

View File

@ -203,6 +203,20 @@ public class AdminController extends BaseController {
} }
} }
@ApiOperation(value = "Check version control settings exists (versionControlSettingsExists)",
notes = "Check whether the version control settings exists. " + TENANT_AUTHORITY_PARAGRAPH)
@PreAuthorize("hasAuthority('TENANT_ADMIN')")
@GetMapping("/vcSettings/exists")
@ResponseBody
public Boolean versionControlSettingsExists() throws ThingsboardException {
try {
accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.READ);
return versionControlService.getVersionControlSettings(getTenantId()) != null;
} catch (Exception e) {
throw handleException(e);
}
}
@ApiOperation(value = "Creates or Updates the version control settings (saveVersionControlSettings)", @ApiOperation(value = "Creates or Updates the version control settings (saveVersionControlSettings)",
notes = "Creates or Updates the version control settings object. " + TENANT_AUTHORITY_PARAGRAPH) notes = "Creates or Updates the version control settings object. " + TENANT_AUTHORITY_PARAGRAPH)
@PreAuthorize("hasAuthority('TENANT_ADMIN')") @PreAuthorize("hasAuthority('TENANT_ADMIN')")

View File

@ -15,17 +15,21 @@
*/ */
package org.thingsboard.server.controller; package org.thingsboard.server.controller;
import com.google.common.util.concurrent.FutureCallback;
import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.MoreExecutors; import com.google.common.util.concurrent.MoreExecutors;
import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiOperation;
import io.swagger.annotations.ApiParam;
import lombok.Data; import lombok.Data;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.web.bind.annotation.*; import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.context.request.async.DeferredResult; import org.springframework.web.context.request.async.DeferredResult;
import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.StringUtils;
@ -33,25 +37,28 @@ import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.EntityIdFactory;
import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.dao.device.claim.ClaimResponse; import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.dao.device.claim.ClaimResult; import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.service.security.model.SecurityUser;
import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService;
import org.thingsboard.server.common.data.sync.vc.EntityVersion; import org.thingsboard.server.common.data.sync.vc.EntityVersion;
import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult;
import org.thingsboard.server.common.data.sync.vc.VersionLoadResult; import org.thingsboard.server.common.data.sync.vc.VersionLoadResult;
import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo;
import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest;
import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadRequest; import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadRequest;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.security.model.SecurityUser;
import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService;
import javax.annotation.Nullable;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.UUID; import java.util.UUID;
import static org.thingsboard.server.controller.ControllerConstants.NEW_LINE; import static org.thingsboard.server.controller.ControllerConstants.NEW_LINE;
import static org.thingsboard.server.controller.ControllerConstants.PAGE_NUMBER_DESCRIPTION;
import static org.thingsboard.server.controller.ControllerConstants.PAGE_SIZE_DESCRIPTION;
@RestController @RestController
@TbCoreComponent
@RequestMapping("/api/entities/vc") @RequestMapping("/api/entities/vc")
@PreAuthorize("hasAuthority('TENANT_ADMIN')") @PreAuthorize("hasAuthority('TENANT_ADMIN')")
@RequiredArgsConstructor @RequiredArgsConstructor
@ -117,13 +124,18 @@ public class EntitiesVersionControlController extends BaseController {
" \"name\": \"Device profile 1 version 1.0\"\n" + " \"name\": \"Device profile 1 version 1.0\"\n" +
" }\n" + " }\n" +
"]\n```") "]\n```")
@GetMapping("/version/{branch}/{entityType}/{externalEntityUuid}") @GetMapping(value = "/version/{branch}/{entityType}/{externalEntityUuid}", params = {"pageSize", "page"})
public DeferredResult<List<EntityVersion>> listEntityVersions(@PathVariable String branch, public DeferredResult<PageData<EntityVersion>> listEntityVersions(@PathVariable String branch,
@PathVariable EntityType entityType, @PathVariable EntityType entityType,
@PathVariable UUID externalEntityUuid) throws ThingsboardException { @PathVariable UUID externalEntityUuid,
@ApiParam(value = PAGE_SIZE_DESCRIPTION, required = true)
@RequestParam int pageSize,
@ApiParam(value = PAGE_NUMBER_DESCRIPTION, required = true)
@RequestParam int page) throws ThingsboardException {
try { try {
EntityId externalEntityId = EntityIdFactory.getByTypeAndUuid(entityType, externalEntityUuid); EntityId externalEntityId = EntityIdFactory.getByTypeAndUuid(entityType, externalEntityUuid);
return wrapFuture(versionControlService.listEntityVersions(getTenantId(), branch, externalEntityId)); PageLink pageLink = new PageLink(pageSize, page);
return wrapFuture(versionControlService.listEntityVersions(getTenantId(), branch, externalEntityId, pageLink));
} catch (Exception e) { } catch (Exception e) {
throw handleException(e); throw handleException(e);
} }
@ -136,11 +148,16 @@ public class EntitiesVersionControlController extends BaseController {
" \"name\": \"Device profiles from dev\"\n" + " \"name\": \"Device profiles from dev\"\n" +
" }\n" + " }\n" +
"]\n```") "]\n```")
@GetMapping("/version/{branch}/{entityType}") @GetMapping(value = "/version/{branch}/{entityType}", params = {"pageSize", "page"})
public DeferredResult<List<EntityVersion>> listEntityTypeVersions(@PathVariable String branch, public DeferredResult<PageData<EntityVersion>> listEntityTypeVersions(@PathVariable String branch,
@PathVariable EntityType entityType) throws ThingsboardException { @PathVariable EntityType entityType,
@ApiParam(value = PAGE_SIZE_DESCRIPTION, required = true)
@RequestParam int pageSize,
@ApiParam(value = PAGE_NUMBER_DESCRIPTION, required = true)
@RequestParam int page) throws ThingsboardException {
try { try {
return wrapFuture(versionControlService.listEntityTypeVersions(getTenantId(), branch, entityType)); PageLink pageLink = new PageLink(pageSize, page);
return wrapFuture(versionControlService.listEntityTypeVersions(getTenantId(), branch, entityType, pageLink));
} catch (Exception e) { } catch (Exception e) {
throw handleException(e); throw handleException(e);
} }
@ -161,10 +178,15 @@ public class EntitiesVersionControlController extends BaseController {
" \"name\": \"Devices added\"\n" + " \"name\": \"Devices added\"\n" +
" }\n" + " }\n" +
"]\n```") "]\n```")
@GetMapping("/version/{branch}") @GetMapping(value = "/version/{branch}", params = {"pageSize", "page"})
public DeferredResult<List<EntityVersion>> listVersions(@PathVariable String branch) throws ThingsboardException { public DeferredResult<PageData<EntityVersion>> listVersions(@PathVariable String branch,
@ApiParam(value = PAGE_SIZE_DESCRIPTION, required = true)
@RequestParam int pageSize,
@ApiParam(value = PAGE_NUMBER_DESCRIPTION, required = true)
@RequestParam int page) throws ThingsboardException {
try { try {
return wrapFuture(versionControlService.listVersions(getTenantId(), branch)); PageLink pageLink = new PageLink(pageSize, page);
return wrapFuture(versionControlService.listVersions(getTenantId(), branch, pageLink));
} catch (Exception e) { } catch (Exception e) {
throw handleException(e); throw handleException(e);
} }

View File

@ -37,6 +37,9 @@ import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.EntityIdFactory;
import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.sync.ThrowingRunnable; import org.thingsboard.server.common.data.sync.ThrowingRunnable;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.sync.ThrowingRunnable;
import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.common.data.sync.ie.EntityExportData;
import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; import org.thingsboard.server.common.data.sync.ie.EntityExportSettings;
import org.thingsboard.server.common.data.sync.ie.EntityImportResult; import org.thingsboard.server.common.data.sync.ie.EntityImportResult;
@ -160,18 +163,18 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
} }
@Override @Override
public ListenableFuture<List<EntityVersion>> listEntityVersions(TenantId tenantId, String branch, EntityId externalId) throws Exception { public ListenableFuture<PageData<EntityVersion>> listEntityVersions(TenantId tenantId, String branch, EntityId externalId, PageLink pageLink) throws Exception {
return gitServiceQueue.listVersions(tenantId, branch, externalId); return gitServiceQueue.listVersions(tenantId, branch, externalId, pageLink);
} }
@Override @Override
public ListenableFuture<List<EntityVersion>> listEntityTypeVersions(TenantId tenantId, String branch, EntityType entityType) throws Exception { public ListenableFuture<PageData<EntityVersion>> listEntityTypeVersions(TenantId tenantId, String branch, EntityType entityType, PageLink pageLink) throws Exception {
return gitServiceQueue.listVersions(tenantId, branch, entityType); return gitServiceQueue.listVersions(tenantId, branch, entityType, pageLink);
} }
@Override @Override
public ListenableFuture<List<EntityVersion>> listVersions(TenantId tenantId, String branch) throws Exception { public ListenableFuture<PageData<EntityVersion>> listVersions(TenantId tenantId, String branch, PageLink pageLink) throws Exception {
return gitServiceQueue.listVersions(tenantId, branch); return gitServiceQueue.listVersions(tenantId, branch, pageLink);
} }
@Override @Override
@ -336,6 +339,12 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
adminSettings.setKey(SETTINGS_KEY); adminSettings.setKey(SETTINGS_KEY);
adminSettings.setTenantId(tenantId); adminSettings.setTenantId(tenantId);
} }
try {
//TODO: ashvayka: replace future.get with deferred result. Don't forget to call when tenant is deleted.
gitServiceQueue.initRepository(tenantId, versionControlSettings).get();
} catch (Exception e) {
throw new RuntimeException("Failed to init repository!", e);
}
adminSettings.setJsonValue(JacksonUtil.valueToTree(versionControlSettings)); adminSettings.setJsonValue(JacksonUtil.valueToTree(versionControlSettings));
AdminSettings savedAdminSettings = adminSettingsService.saveAdminSettings(tenantId, adminSettings); AdminSettings savedAdminSettings = adminSettingsService.saveAdminSettings(tenantId, adminSettings);
EntitiesVersionControlSettings savedVersionControlSettings; EntitiesVersionControlSettings savedVersionControlSettings;
@ -344,12 +353,6 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
} catch (Exception e) { } catch (Exception e) {
throw new RuntimeException("Failed to load version control settings!", e); throw new RuntimeException("Failed to load version control settings!", e);
} }
try {
//TODO: ashvayka: replace future.get with deferred result. Don't forget to call when tenant is deleted.
gitServiceQueue.initRepository(tenantId, versionControlSettings).get();
} catch (Exception e) {
throw new RuntimeException("Failed to init repository!", e);
}
return savedVersionControlSettings; return savedVersionControlSettings;
} }

View File

@ -16,15 +16,14 @@
package org.thingsboard.server.service.sync.vc; package org.thingsboard.server.service.sync.vc;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectWriter;
import com.fasterxml.jackson.databind.SerializationFeature; import com.fasterxml.jackson.databind.SerializationFeature;
import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.SettableFuture; import com.google.common.util.concurrent.SettableFuture;
import com.google.protobuf.ByteString; import com.google.protobuf.ByteString;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.context.annotation.Lazy; import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.cluster.TbClusterService;
import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.ExportableEntity;
@ -32,6 +31,8 @@ import org.thingsboard.server.common.data.StringUtils;
import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.EntityIdFactory;
import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.common.data.sync.ie.EntityExportData;
import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings;
import org.thingsboard.server.common.data.sync.vc.EntityVersion; import org.thingsboard.server.common.data.sync.vc.EntityVersion;
@ -73,7 +74,7 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu
private final DefaultEntitiesVersionControlService entitiesVersionControlService; private final DefaultEntitiesVersionControlService entitiesVersionControlService;
private final Map<UUID, PendingGitRequest<?>> pendingRequestMap = new HashMap<>(); private final Map<UUID, PendingGitRequest<?>> pendingRequestMap = new HashMap<>();
private final ObjectWriter jsonWriter = new ObjectMapper().writer(SerializationFeature.INDENT_OUTPUT); private final ObjectMapper jsonMapper = new ObjectMapper().enable(SerializationFeature.INDENT_OUTPUT);
public DefaultGitVersionControlQueueService(TbServiceInfoProvider serviceInfoProvider, TbClusterService clusterService, public DefaultGitVersionControlQueueService(TbServiceInfoProvider serviceInfoProvider, TbClusterService clusterService,
DataDecodingEncodingService encodingService, DataDecodingEncodingService encodingService,
@ -102,7 +103,7 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu
String path = getRelativePath(entityData.getEntityType(), entityData.getEntity().getId()); String path = getRelativePath(entityData.getEntityType(), entityData.getEntity().getId());
String entityDataJson; String entityDataJson;
try { try {
entityDataJson = jsonWriter.writeValueAsString(entityData); entityDataJson = jsonMapper.writeValueAsString(entityData);
} catch (IOException e) { } catch (IOException e) {
//TODO: analyze and return meaningful exceptions that we can show to the client; //TODO: analyze and return meaningful exceptions that we can show to the client;
throw new RuntimeException(e); throw new RuntimeException(e);
@ -144,29 +145,36 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu
} }
@Override @Override
public ListenableFuture<List<EntityVersion>> listVersions(TenantId tenantId, String branch) { public ListenableFuture<PageData<EntityVersion>> listVersions(TenantId tenantId, String branch, PageLink pageLink) {
return listVersions(tenantId, ListVersionsRequestMsg.newBuilder() return listVersions(tenantId, ListVersionsRequestMsg.newBuilder()
.setBranchName(branch).build()); .setBranchName(branch)
} .setPageSize(pageLink.getPageSize())
.setPage(pageLink.getPage())
@Override
public ListenableFuture<List<EntityVersion>> listVersions(TenantId tenantId, String branch, EntityType entityType) {
return listVersions(tenantId, ListVersionsRequestMsg.newBuilder()
.setBranchName(branch).setEntityType(entityType.name())
.build()); .build());
} }
@Override @Override
public ListenableFuture<List<EntityVersion>> listVersions(TenantId tenantId, String branch, EntityId entityId) { public ListenableFuture<PageData<EntityVersion>> listVersions(TenantId tenantId, String branch, EntityType entityType, PageLink pageLink) {
return listVersions(tenantId, ListVersionsRequestMsg.newBuilder()
.setBranchName(branch).setEntityType(entityType.name())
.setPageSize(pageLink.getPageSize())
.setPage(pageLink.getPage())
.build());
}
@Override
public ListenableFuture<PageData<EntityVersion>> listVersions(TenantId tenantId, String branch, EntityId entityId, PageLink pageLink) {
return listVersions(tenantId, ListVersionsRequestMsg.newBuilder() return listVersions(tenantId, ListVersionsRequestMsg.newBuilder()
.setBranchName(branch) .setBranchName(branch)
.setEntityType(entityId.getEntityType().name()) .setEntityType(entityId.getEntityType().name())
.setEntityIdMSB(entityId.getId().getMostSignificantBits()) .setEntityIdMSB(entityId.getId().getMostSignificantBits())
.setEntityIdLSB(entityId.getId().getLeastSignificantBits()) .setEntityIdLSB(entityId.getId().getLeastSignificantBits())
.setPageSize(pageLink.getPageSize())
.setPage(pageLink.getPage())
.build()); .build());
} }
private ListenableFuture<List<EntityVersion>> listVersions(TenantId tenantId, ListVersionsRequestMsg requestMsg) { private ListenableFuture<PageData<EntityVersion>> listVersions(TenantId tenantId, ListVersionsRequestMsg requestMsg) {
ListVersionsGitRequest request = new ListVersionsGitRequest(tenantId); ListVersionsGitRequest request = new ListVersionsGitRequest(tenantId);
registerAndSend(request, builder -> builder.setListVersionRequest(requestMsg).build(), wrap(request.getFuture())); registerAndSend(request, builder -> builder.setListVersionRequest(requestMsg).build(), wrap(request.getFuture()));
@ -318,8 +326,7 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu
listEntitiesResponse.getEntitiesList().stream().map(this::getVersionedEntityInfo).collect(Collectors.toList())); listEntitiesResponse.getEntitiesList().stream().map(this::getVersionedEntityInfo).collect(Collectors.toList()));
} else if (vcResponseMsg.hasListVersionsResponse()) { } else if (vcResponseMsg.hasListVersionsResponse()) {
var listVersionsResponse = vcResponseMsg.getListVersionsResponse(); var listVersionsResponse = vcResponseMsg.getListVersionsResponse();
((ListVersionsGitRequest) request).getFuture().set( ((ListVersionsGitRequest) request).getFuture().set(toPageData(listVersionsResponse));
listVersionsResponse.getVersionsList().stream().map(this::getEntityVersion).collect(Collectors.toList()));
} else if (vcResponseMsg.hasEntityContentResponse()) { } else if (vcResponseMsg.hasEntityContentResponse()) {
var data = vcResponseMsg.getEntityContentResponse().getData(); var data = vcResponseMsg.getEntityContentResponse().getData();
((EntityContentGitRequest) request).getFuture().set(toData(data)); ((EntityContentGitRequest) request).getFuture().set(toData(data));
@ -331,6 +338,11 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu
} }
} }
private PageData<EntityVersion> toPageData(TransportProtos.ListVersionsResponseMsg listVersionsResponse) {
var listVersions = listVersionsResponse.getVersionsList().stream().map(this::getEntityVersion).collect(Collectors.toList());
return new PageData<>(listVersions, listVersionsResponse.getTotalPages(), listVersionsResponse.getTotalElements(), listVersionsResponse.getHasNext());
}
private EntityVersion getEntityVersion(TransportProtos.EntityVersionProto proto) { private EntityVersion getEntityVersion(TransportProtos.EntityVersionProto proto) {
return new EntityVersion(proto.getId(), proto.getName()); return new EntityVersion(proto.getId(), proto.getName());
} }
@ -340,8 +352,9 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu
} }
@SuppressWarnings("rawtypes") @SuppressWarnings("rawtypes")
@SneakyThrows
private EntityExportData toData(String data) { private EntityExportData toData(String data) {
return JacksonUtil.fromString(data, EntityExportData.class); return jsonMapper.readValue(data, EntityExportData.class);
} }
private static <T> TbQueueCallback wrap(SettableFuture<T> future) { private static <T> TbQueueCallback wrap(SettableFuture<T> future) {

View File

@ -20,6 +20,8 @@ import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.model.SecurityUser;
import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings;
import org.thingsboard.server.common.data.sync.vc.EntityVersion; import org.thingsboard.server.common.data.sync.vc.EntityVersion;
@ -38,11 +40,11 @@ public interface EntitiesVersionControlService {
ListenableFuture<VersionCreationResult> saveEntitiesVersion(SecurityUser user, VersionCreateRequest request) throws Exception; ListenableFuture<VersionCreationResult> saveEntitiesVersion(SecurityUser user, VersionCreateRequest request) throws Exception;
ListenableFuture<List<EntityVersion>> listEntityVersions(TenantId tenantId, String branch, EntityId externalId) throws Exception; ListenableFuture<PageData<EntityVersion>> listEntityVersions(TenantId tenantId, String branch, EntityId externalId, PageLink pageLink) throws Exception;
ListenableFuture<List<EntityVersion>> listEntityTypeVersions(TenantId tenantId, String branch, EntityType entityType) throws Exception; ListenableFuture<PageData<EntityVersion>> listEntityTypeVersions(TenantId tenantId, String branch, EntityType entityType, PageLink pageLink) throws Exception;
ListenableFuture<List<EntityVersion>> listVersions(TenantId tenantId, String branch) throws Exception; ListenableFuture<PageData<EntityVersion>> listVersions(TenantId tenantId, String branch, PageLink pageLink) throws Exception;
ListenableFuture<List<VersionedEntityInfo>> listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType) throws Exception; ListenableFuture<List<VersionedEntityInfo>> listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType) throws Exception;

View File

@ -20,6 +20,8 @@ import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.ExportableEntity;
import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.common.data.sync.ie.EntityExportData;
import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings;
import org.thingsboard.server.common.data.sync.vc.EntityVersion; import org.thingsboard.server.common.data.sync.vc.EntityVersion;
@ -40,11 +42,11 @@ public interface GitVersionControlQueueService {
ListenableFuture<VersionCreationResult> push(CommitGitRequest commit); ListenableFuture<VersionCreationResult> push(CommitGitRequest commit);
ListenableFuture<List<EntityVersion>> listVersions(TenantId tenantId, String branch); ListenableFuture<PageData<EntityVersion>> listVersions(TenantId tenantId, String branch, PageLink pageLink);
ListenableFuture<List<EntityVersion>> listVersions(TenantId tenantId, String branch, EntityType entityType); ListenableFuture<PageData<EntityVersion>> listVersions(TenantId tenantId, String branch, EntityType entityType, PageLink pageLink);
ListenableFuture<List<EntityVersion>> listVersions(TenantId tenantId, String branch, EntityId entityId); ListenableFuture<PageData<EntityVersion>> listVersions(TenantId tenantId, String branch, EntityId entityId, PageLink pageLink);
ListenableFuture<List<VersionedEntityInfo>> listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType); ListenableFuture<List<VersionedEntityInfo>> listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType);

View File

@ -16,13 +16,14 @@
package org.thingsboard.server.service.sync.vc; package org.thingsboard.server.service.sync.vc;
import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.sync.vc.EntityVersion; import org.thingsboard.server.common.data.sync.vc.EntityVersion;
import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult;
import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest;
import java.util.List; import java.util.List;
public class ListVersionsGitRequest extends PendingGitRequest<List<EntityVersion>> { public class ListVersionsGitRequest extends PendingGitRequest<PageData<EntityVersion>> {
public ListVersionsGitRequest(TenantId tenantId) { public ListVersionsGitRequest(TenantId tenantId) {
super(tenantId); super(tenantId);

View File

@ -1122,8 +1122,8 @@ metrics:
vc: vc:
thread_pool_size: "${TB_VC_POOL_SIZE:4}" thread_pool_size: "${TB_VC_POOL_SIZE:4}"
git: git:
folder: "${TB_VC_GIT_FOLDER:}"
repos-poll-interval: "${TB_VC_GIT_REPOS_POLL_INTERVAL_SEC:60}" repos-poll-interval: "${TB_VC_GIT_REPOS_POLL_INTERVAL_SEC:60}"
repositories-folder: "${TB_VC_GIT_REPOSITORIES_FOLDER:${java.io.tmpdir}/repositories}"
management: management:
endpoints: endpoints:

View File

@ -721,6 +721,8 @@ message ListVersionsRequestMsg {
string entityType = 2; string entityType = 2;
int64 entityIdMSB = 3; int64 entityIdMSB = 3;
int64 entityIdLSB = 4; int64 entityIdLSB = 4;
int32 pageSize = 5;
int32 page = 6;
} }
message EntityVersionProto { message EntityVersionProto {
@ -731,6 +733,9 @@ message EntityVersionProto {
message ListVersionsResponseMsg { message ListVersionsResponseMsg {
repeated EntityVersionProto versions = 1; repeated EntityVersionProto versions = 1;
int32 totalPages = 2;
int64 totalElements = 3;
bool hasNext = 4;
} }
message ListEntitiesRequestMsg { message ListEntitiesRequestMsg {

View File

@ -20,6 +20,7 @@ import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonSubTypes;
import com.fasterxml.jackson.annotation.JsonSubTypes.Type; import com.fasterxml.jackson.annotation.JsonSubTypes.Type;
import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.fasterxml.jackson.annotation.JsonTypeInfo;
import com.fasterxml.jackson.annotation.JsonTypeInfo.As;
import lombok.Data; import lombok.Data;
import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.ExportableEntity;
@ -30,7 +31,7 @@ import org.thingsboard.server.common.data.sync.JsonTbEntity;
import java.util.List; import java.util.List;
@JsonIgnoreProperties(ignoreUnknown = true) @JsonIgnoreProperties(ignoreUnknown = true)
@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "entityType", defaultImpl = EntityExportData.class) @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "entityType", include = As.EXISTING_PROPERTY, visible = true, defaultImpl = EntityExportData.class)
@JsonSubTypes({ @JsonSubTypes({
@Type(name = "DEVICE", value = DeviceExportData.class), @Type(name = "DEVICE", value = DeviceExportData.class),
@Type(name = "RULE_CHAIN", value = RuleChainExportData.class) @Type(name = "RULE_CHAIN", value = RuleChainExportData.class)

View File

@ -15,26 +15,20 @@
*/ */
package org.thingsboard.server.common.data.sync.ie; package org.thingsboard.server.common.data.sync.ie;
import com.fasterxml.jackson.annotation.JsonIgnore;
import lombok.Data; import lombok.Data;
import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.ExportableEntity;
import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.sync.JsonTbEntity;
import org.thingsboard.server.common.data.sync.ThrowingRunnable; import org.thingsboard.server.common.data.sync.ThrowingRunnable;
@Data @Data
public class EntityImportResult<E extends ExportableEntity<? extends EntityId>> { public class EntityImportResult<E extends ExportableEntity<? extends EntityId>> {
@JsonTbEntity
private E savedEntity; private E savedEntity;
@JsonTbEntity
private E oldEntity; private E oldEntity;
private EntityType entityType; private EntityType entityType;
@JsonIgnore
private ThrowingRunnable saveReferencesCallback = () -> {}; private ThrowingRunnable saveReferencesCallback = () -> {};
@JsonIgnore
private ThrowingRunnable sendEventsCallback = () -> {}; private ThrowingRunnable sendEventsCallback = () -> {};
public void addSaveReferencesCallback(ThrowingRunnable callback) { public void addSaveReferencesCallback(ThrowingRunnable callback) {

View File

@ -68,7 +68,6 @@
<dependency> <dependency>
<groupId>com.google.guava</groupId> <groupId>com.google.guava</groupId>
<artifactId>guava</artifactId> <artifactId>guava</artifactId>
<scope>provided</scope>
</dependency> </dependency>
<dependency> <dependency>
<groupId>com.fasterxml.jackson.core</groupId> <groupId>com.fasterxml.jackson.core</groupId>

View File

@ -15,8 +15,8 @@
*/ */
package org.thingsboard.server.service.sync.vc; package org.thingsboard.server.service.sync.vc;
import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent;
import org.springframework.context.ApplicationListener; import org.springframework.context.ApplicationListener;
import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent;
public interface ClusterVersionControlService extends ApplicationListener<PartitionChangeEvent> { public interface ClusterVersionControlService extends ApplicationListener<PartitionChangeEvent> {
} }

View File

@ -26,15 +26,33 @@ import org.thingsboard.common.util.ThingsBoardThreadFactory;
import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.StringUtils;
import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings;
import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult;
import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo;
import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.gen.transport.TransportProtos.*; import org.thingsboard.server.gen.transport.TransportProtos.AddMsg;
import org.thingsboard.server.gen.transport.TransportProtos.CommitRequestMsg;
import org.thingsboard.server.gen.transport.TransportProtos.CommitResponseMsg;
import org.thingsboard.server.gen.transport.TransportProtos.DeleteMsg;
import org.thingsboard.server.gen.transport.TransportProtos.EntitiesContentRequestMsg;
import org.thingsboard.server.gen.transport.TransportProtos.EntitiesContentResponseMsg;
import org.thingsboard.server.gen.transport.TransportProtos.EntityContentRequestMsg;
import org.thingsboard.server.gen.transport.TransportProtos.EntityContentResponseMsg;
import org.thingsboard.server.gen.transport.TransportProtos.EntityVersionProto;
import org.thingsboard.server.gen.transport.TransportProtos.ListBranchesRequestMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ListBranchesResponseMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ListEntitiesRequestMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ListEntitiesResponseMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ListVersionsRequestMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ListVersionsResponseMsg;
import org.thingsboard.server.gen.transport.TransportProtos.PrepareMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToVersionControlServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToVersionControlServiceMsg;
import org.thingsboard.server.gen.transport.TransportProtos.VersionControlResponseMsg;
import org.thingsboard.server.gen.transport.TransportProtos.VersionedEntityInfoProto;
import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueConsumer;
import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueProducer;
import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg;
@ -220,12 +238,16 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe
} else { } else {
path = null; path = null;
} }
var data = vcService.listVersions(ctx.getTenantId(), request.getBranchName(), path); var data = vcService.listVersions(ctx.getTenantId(), request.getBranchName(), path, new PageLink(request.getPageSize(), request.getPage()));
reply(ctx, Optional.empty(), builder -> reply(ctx, Optional.empty(), builder ->
builder.setListVersionsResponse(ListVersionsResponseMsg.newBuilder() builder.setListVersionsResponse(ListVersionsResponseMsg.newBuilder()
.addAllVersions(data.stream().map( .setTotalPages(data.getTotalPages())
.setTotalElements(data.getTotalElements())
.setHasNext(data.hasNext())
.addAllVersions(data.getData().stream().map(
v -> EntityVersionProto.newBuilder().setId(v.getId()).setName(v.getName()).build() v -> EntityVersionProto.newBuilder().setId(v.getId()).setName(v.getName()).build()
).collect(Collectors.toList())))); ).collect(Collectors.toList())))
);
} }
private void handleListEntities(VersionControlRequestCtx ctx, ListEntitiesRequestMsg request) throws Exception { private void handleListEntities(VersionControlRequestCtx ctx, ListEntitiesRequestMsg request) throws Exception {
@ -266,7 +288,9 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe
} else if (request.hasDeleteMsg()) { } else if (request.hasDeleteMsg()) {
deleteFromCommit(ctx, current, request.getDeleteMsg()); deleteFromCommit(ctx, current, request.getDeleteMsg());
} else if (request.hasPushMsg()) { } else if (request.hasPushMsg()) {
reply(ctx, vcService.push(current)); var result = vcService.push(current);
pendingCommitMap.remove(ctx.getTenantId());
reply(ctx, result);
} }
} catch (Exception e) { } catch (Exception e) {
doAbortCurrentCommit(tenantId, current, e); doAbortCurrentCommit(tenantId, current, e);

View File

@ -15,12 +15,11 @@
*/ */
package org.thingsboard.server.service.sync.vc; package org.thingsboard.server.service.sync.vc;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FileUtils; import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.eclipse.jgit.api.errors.GitAPIException; import org.eclipse.jgit.api.errors.GitAPIException;
import org.eclipse.jgit.api.errors.JGitInternalException;
import org.eclipse.jgit.api.errors.RefAlreadyExistsException;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
@ -28,6 +27,8 @@ import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.EntityIdFactory;
import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings;
import org.thingsboard.server.common.data.sync.vc.EntityVersion; import org.thingsboard.server.common.data.sync.vc.EntityVersion;
import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult;
@ -57,7 +58,7 @@ public class DefaultGitRepositoryService implements GitRepositoryService {
@Value("${java.io.tmpdir}/repositories") @Value("${java.io.tmpdir}/repositories")
private String defaultFolder; private String defaultFolder;
@Value("${vc.git.folder:${java.io.tmpdir}/repositories}") @Value("${vc.git.repositories-folder:${java.io.tmpdir}/repositories}")
private String repositoriesFolder; private String repositoriesFolder;
@Value("${vc.git.repos-poll-interval:60}") @Value("${vc.git.repos-poll-interval:60}")
@ -97,22 +98,12 @@ public class DefaultGitRepositoryService implements GitRepositoryService {
String branch = commit.getBranch(); String branch = commit.getBranch();
try { try {
repository.fetch(); repository.fetch();
if (repository.listBranches().contains(branch)) {
repository.checkout("origin/" + branch, false); repository.createAndCheckoutOrphanBranch(commit.getWorkingBranch());
try { repository.resetAndClean();
repository.checkout(branch, true);
} catch (RefAlreadyExistsException e) { if (repository.listRemoteBranches().contains(branch)) {
repository.checkout(branch, false);
}
repository.merge(branch); repository.merge(branch);
} else { // TODO [viacheslav]: rollback orphan branch on failure
try {
repository.createAndCheckoutOrphanBranch(branch); // FIXME [viacheslav]: Checkout returned unexpected result NO_CHANGE for master branch
} catch (JGitInternalException e) {
if (!e.getMessage().contains("NO_CHANGE")) {
throw e;
}
}
} }
} catch (IOException | GitAPIException gitAPIException) { } catch (IOException | GitAPIException gitAPIException) {
//TODO: analyze and return meaningful exceptions that we can show to the client; //TODO: analyze and return meaningful exceptions that we can show to the client;
@ -145,24 +136,44 @@ public class DefaultGitRepositoryService implements GitRepositoryService {
result.setRemoved(status.getRemoved().size()); result.setRemoved(status.getRemoved().size());
GitRepository.Commit gitCommit = repository.commit(commit.getVersionName()); GitRepository.Commit gitCommit = repository.commit(commit.getVersionName());
repository.push(); repository.push(commit.getWorkingBranch(), commit.getBranch());
result.setVersion(toVersion(gitCommit)); result.setVersion(toVersion(gitCommit));
return result; return result;
} catch (GitAPIException gitAPIException) { } catch (GitAPIException gitAPIException) {
//TODO: analyze and return meaningful exceptions that we can show to the client; //TODO: analyze and return meaningful exceptions that we can show to the client;
throw new RuntimeException(gitAPIException); throw new RuntimeException(gitAPIException);
} finally {
cleanUp(commit);
} }
} }
@SneakyThrows
@Override
public void cleanUp(PendingCommit commit) {
GitRepository repository = checkRepository(commit.getTenantId());
try {
repository.createAndCheckoutOrphanBranch(EntityId.NULL_UUID.toString());
} catch (Exception e) {
if (!e.getMessage().contains("NO_CHANGE")) {
throw e;
}
}
repository.resetAndClean();
repository.deleteLocalBranchIfExists(commit.getWorkingBranch());
}
@Override @Override
public void abort(PendingCommit commit) { public void abort(PendingCommit commit) {
//TODO: implement; cleanUp(commit);
} }
@Override @Override
public void fetch(TenantId tenantId) { public void fetch(TenantId tenantId) throws GitAPIException {
//Fetch latest changes on demand. var repository = repositories.get(tenantId);
if (repository != null) {
repository.fetch();
}
} }
@Override @Override
@ -175,7 +186,7 @@ public class DefaultGitRepositoryService implements GitRepositoryService {
public List<String> listBranches(TenantId tenantId) { public List<String> listBranches(TenantId tenantId) {
GitRepository repository = checkRepository(tenantId); GitRepository repository = checkRepository(tenantId);
try { try {
return repository.listBranches(); return repository.listRemoteBranches();
} catch (GitAPIException gitAPIException) { } catch (GitAPIException gitAPIException) {
//TODO: analyze and return meaningful exceptions that we can show to the client; //TODO: analyze and return meaningful exceptions that we can show to the client;
throw new RuntimeException(gitAPIException); throw new RuntimeException(gitAPIException);
@ -183,7 +194,7 @@ public class DefaultGitRepositoryService implements GitRepositoryService {
} }
private EntityVersion checkVersion(TenantId tenantId, String branch, String versionId) throws Exception { private EntityVersion checkVersion(TenantId tenantId, String branch, String versionId) throws Exception {
return listVersions(tenantId, branch, null).stream() return listVersions(tenantId, branch, null, new PageLink(Integer.MAX_VALUE)).getData().stream()
.filter(version -> version.getId().equals(versionId)) .filter(version -> version.getId().equals(versionId))
.findFirst().orElseThrow(() -> new IllegalArgumentException("Version not found")); .findFirst().orElseThrow(() -> new IllegalArgumentException("Version not found"));
} }
@ -194,11 +205,9 @@ public class DefaultGitRepositoryService implements GitRepositoryService {
} }
@Override @Override
public List<EntityVersion> listVersions(TenantId tenantId, String branch, String path) throws Exception { public PageData<EntityVersion> listVersions(TenantId tenantId, String branch, String path, PageLink pageLink) throws Exception {
GitRepository repository = checkRepository(tenantId); GitRepository repository = checkRepository(tenantId);
return repository.listCommits(branch, path, Integer.MAX_VALUE).stream() return repository.listCommits(branch, path, pageLink).mapData(this::toVersion);
.map(this::toVersion)
.collect(Collectors.toList());
} }
@Override @Override

View File

@ -15,11 +15,20 @@
*/ */
package org.thingsboard.server.service.sync.vc; package org.thingsboard.server.service.sync.vc;
import com.google.common.collect.Iterables;
import com.google.common.collect.Streams; import com.google.common.collect.Streams;
import lombok.Data; import lombok.Data;
import lombok.Getter; import lombok.Getter;
import org.apache.commons.lang3.StringUtils;
import org.apache.sshd.common.util.security.SecurityUtils; import org.apache.sshd.common.util.security.SecurityUtils;
import org.eclipse.jgit.api.*; import org.eclipse.jgit.api.CloneCommand;
import org.eclipse.jgit.api.Git;
import org.eclipse.jgit.api.GitCommand;
import org.eclipse.jgit.api.ListBranchCommand;
import org.eclipse.jgit.api.LogCommand;
import org.eclipse.jgit.api.LsRemoteCommand;
import org.eclipse.jgit.api.ResetCommand;
import org.eclipse.jgit.api.TransportCommand;
import org.eclipse.jgit.api.errors.GitAPIException; import org.eclipse.jgit.api.errors.GitAPIException;
import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.Constants;
import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectId;
@ -28,6 +37,7 @@ import org.eclipse.jgit.lib.ObjectReader;
import org.eclipse.jgit.revwalk.RevCommit; import org.eclipse.jgit.revwalk.RevCommit;
import org.eclipse.jgit.revwalk.filter.RevFilter; import org.eclipse.jgit.revwalk.filter.RevFilter;
import org.eclipse.jgit.transport.CredentialsProvider; import org.eclipse.jgit.transport.CredentialsProvider;
import org.eclipse.jgit.transport.RefSpec;
import org.eclipse.jgit.transport.SshTransport; import org.eclipse.jgit.transport.SshTransport;
import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider; import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider;
import org.eclipse.jgit.transport.sshd.JGitKeyCache; import org.eclipse.jgit.transport.sshd.JGitKeyCache;
@ -36,7 +46,8 @@ import org.eclipse.jgit.transport.sshd.SshdSessionFactory;
import org.eclipse.jgit.transport.sshd.SshdSessionFactoryBuilder; import org.eclipse.jgit.transport.sshd.SshdSessionFactoryBuilder;
import org.eclipse.jgit.treewalk.TreeWalk; import org.eclipse.jgit.treewalk.TreeWalk;
import org.eclipse.jgit.treewalk.filter.PathFilter; import org.eclipse.jgit.treewalk.filter.PathFilter;
import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings;
import org.thingsboard.server.common.data.sync.vc.VersionControlAuthMethod; import org.thingsboard.server.common.data.sync.vc.VersionControlAuthMethod;
@ -51,6 +62,7 @@ import java.util.ArrayList;
import java.util.Collections; import java.util.Collections;
import java.util.List; import java.util.List;
import java.util.Set; import java.util.Set;
import java.util.function.Function;
import java.util.stream.Collectors; import java.util.stream.Collectors;
public class GitRepository { public class GitRepository {
@ -119,15 +131,18 @@ public class GitRepository {
.setRemoveDeletedRefs(true)); .setRemoveDeletedRefs(true));
} }
public void checkout(String branch, boolean createBranch) throws GitAPIException { public void deleteLocalBranchIfExists(String branch) throws GitAPIException {
execute(git.checkout() execute(git.branchDelete()
.setCreateBranch(createBranch) .setBranchNames(branch)
.setName(branch)); .setForce(true));
} }
public void reset() throws GitAPIException { public void resetAndClean() throws GitAPIException {
execute(git.reset() execute(git.reset()
.setMode(ResetCommand.ResetType.HARD)); .setMode(ResetCommand.ResetType.HARD));
execute(git.clean()
.setForce(true)
.setCleanDirectories(true));
} }
public void merge(String branch) throws IOException, GitAPIException { public void merge(String branch) throws IOException, GitAPIException {
@ -140,36 +155,34 @@ public class GitRepository {
} }
public List<String> listBranches() throws GitAPIException { public List<String> listRemoteBranches() throws GitAPIException {
return execute(git.branchList() return execute(git.branchList()
.setListMode(ListBranchCommand.ListMode.ALL)).stream() .setListMode(ListBranchCommand.ListMode.REMOTE)).stream()
.filter(ref -> !ref.getName().equals(Constants.HEAD)) .filter(ref -> !ref.getName().equals(Constants.HEAD))
.map(ref -> org.eclipse.jgit.lib.Repository.shortenRefName(ref.getName())) .map(ref -> org.eclipse.jgit.lib.Repository.shortenRefName(ref.getName()))
.map(name -> StringUtils.removeStart(name, "origin/")) .map(name -> StringUtils.removeStart(name, "origin/"))
.distinct().collect(Collectors.toList()); .distinct().collect(Collectors.toList());
} }
public List<Commit> listCommits(String branch, int limit) throws IOException, GitAPIException { public PageData<Commit> listCommits(String branch, PageLink pageLink) throws IOException, GitAPIException {
return listCommits(branch, null, limit); return listCommits(branch, null, pageLink);
} }
public List<Commit> listCommits(String branch, String path, int limit) throws IOException, GitAPIException { public PageData<Commit> listCommits(String branch, String path, PageLink pageLink) throws IOException, GitAPIException {
ObjectId branchId = resolve("origin/" + branch); ObjectId branchId = resolve("origin/" + branch);
if (branchId == null) { if (branchId == null) {
throw new IllegalArgumentException("Branch not found"); throw new IllegalArgumentException("Branch not found");
} }
LogCommand command = git.log() LogCommand command = git.log()
.add(branchId).setMaxCount(limit) .add(branchId)
.setRevFilter(RevFilter.NO_MERGES); .setRevFilter(RevFilter.NO_MERGES);
if (StringUtils.isNotEmpty(path)) { if (StringUtils.isNotEmpty(path)) {
command.addPath(path); command.addPath(path);
} }
return Streams.stream(execute(command)) Iterable<RevCommit> commits = execute(command);
.map(this::toCommit) return iterableToPageData(commits, this::toCommit, pageLink);
.collect(Collectors.toList());
} }
public List<String> listFilesAtCommit(String commitId) throws IOException { public List<String> listFilesAtCommit(String commitId) throws IOException {
return listFilesAtCommit(commitId, null); return listFilesAtCommit(commitId, null);
} }
@ -212,16 +225,9 @@ public class GitRepository {
.setOrphan(true) .setOrphan(true)
.setForced(true) .setForced(true)
.setName(name)); .setName(name));
// Set<String> uncommittedChanges = git.status().call().getUncommittedChanges();
// if (!uncommittedChanges.isEmpty()) {
// RmCommand rm = git.rm();
// uncommittedChanges.forEach(rm::addFilepattern);
// execute(rm);
// }
// execute(git.clean());
} }
public void add(String filesPattern) throws GitAPIException { // FIXME [viacheslav] public void add(String filesPattern) throws GitAPIException {
execute(git.add().setUpdate(true).addFilepattern(filesPattern)); execute(git.add().setUpdate(true).addFilepattern(filesPattern));
execute(git.add().addFilepattern(filesPattern)); execute(git.add().addFilepattern(filesPattern));
} }
@ -233,13 +239,14 @@ public class GitRepository {
public Commit commit(String message) throws GitAPIException { public Commit commit(String message) throws GitAPIException {
RevCommit revCommit = execute(git.commit() RevCommit revCommit = execute(git.commit()
.setMessage(message)); // TODO [viacheslav]: set configurable author for commit .setMessage(message));
return toCommit(revCommit); return toCommit(revCommit);
} }
public void push() throws GitAPIException { public void push(String localBranch, String remoteBranch) throws GitAPIException {
execute(git.push()); execute(git.push()
.setRefSpecs(new RefSpec(localBranch + ":" + remoteBranch)));
} }
@ -293,6 +300,23 @@ public class GitRepository {
return command.call(); return command.call();
} }
private static <T,R> PageData<R> iterableToPageData (Iterable<T> iterable, Function<? super T, ? extends R> mapper, PageLink pageLink) {
int totalElements = Iterables.size(iterable);
int totalPages = pageLink.getPageSize() > 0 ? (int) Math.ceil((float) totalElements / pageLink.getPageSize()) : 1;
int startIndex = pageLink.getPageSize() * pageLink.getPage();
int limit = startIndex + pageLink.getPageSize();
iterable = Iterables.limit(iterable, limit);
if (startIndex < totalElements) {
iterable = Iterables.skip(iterable, startIndex);
} else {
iterable = Collections.emptyList();
}
List<R> data = Streams.stream(iterable).map(mapper)
.collect(Collectors.toList());
boolean hasNext = pageLink.getPageSize() > 0 && totalElements > startIndex + data.size();
return new PageData<>(data, totalPages, totalElements, hasNext);
}
private static void configureTransportCommand(TransportCommand transportCommand, CredentialsProvider credentialsProvider, SshdSessionFactory sshSessionFactory) { private static void configureTransportCommand(TransportCommand transportCommand, CredentialsProvider credentialsProvider, SshdSessionFactory sshSessionFactory) {
if (credentialsProvider != null) { if (credentialsProvider != null) {
transportCommand.setCredentialsProvider(credentialsProvider); transportCommand.setCredentialsProvider(credentialsProvider);

View File

@ -15,7 +15,10 @@
*/ */
package org.thingsboard.server.service.sync.vc; package org.thingsboard.server.service.sync.vc;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings;
import org.thingsboard.server.common.data.sync.vc.EntityVersion; import org.thingsboard.server.common.data.sync.vc.EntityVersion;
import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult;
@ -28,7 +31,7 @@ public interface GitRepositoryService {
void prepareCommit(PendingCommit pendingCommit); void prepareCommit(PendingCommit pendingCommit);
List<EntityVersion> listVersions(TenantId tenantId, String branch, String path) throws Exception; PageData<EntityVersion> listVersions(TenantId tenantId, String branch, String path, PageLink pageLink) throws Exception;
List<VersionedEntityInfo> listEntitiesAtVersion(TenantId tenantId, String versionId, String path) throws Exception; List<VersionedEntityInfo> listEntitiesAtVersion(TenantId tenantId, String versionId, String path) throws Exception;
@ -46,11 +49,13 @@ public interface GitRepositoryService {
VersionCreationResult push(PendingCommit commit); VersionCreationResult push(PendingCommit commit);
void cleanUp(PendingCommit commit);
void abort(PendingCommit commit); void abort(PendingCommit commit);
List<String> listBranches(TenantId tenantId); List<String> listBranches(TenantId tenantId);
String getFileContentAtCommit(TenantId tenantId, String relativePath, String versionId) throws IOException; String getFileContentAtCommit(TenantId tenantId, String relativePath, String versionId) throws IOException;
void fetch(TenantId tenantId); void fetch(TenantId tenantId) throws GitAPIException;
} }

View File

@ -17,7 +17,6 @@ package org.thingsboard.server.service.sync.vc;
import lombok.Data; import lombok.Data;
import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest;
import java.util.UUID; import java.util.UUID;
@ -27,6 +26,7 @@ public class PendingCommit {
private final UUID txId; private final UUID txId;
private final String nodeId; private final String nodeId;
private final TenantId tenantId; private final TenantId tenantId;
private final String workingBranch;
private String branch; private String branch;
private String versionName; private String versionName;
@ -36,5 +36,6 @@ public class PendingCommit {
this.txId = txId; this.txId = txId;
this.branch = branch; this.branch = branch;
this.versionName = versionName; this.versionName = versionName;
this.workingBranch = txId.toString();
} }
} }

View File

@ -26,13 +26,13 @@ VALUES ( '61441950-4612-11e7-a919-92ebcb67fe33', 1592576748000, '5a797660-4612-1
'$2a$10$5JTB8/hxWc9WAy62nCGSxeefl3KWmipA9nFpVdDa0/xfIseeBB4Bu' ); '$2a$10$5JTB8/hxWc9WAy62nCGSxeefl3KWmipA9nFpVdDa0/xfIseeBB4Bu' );
/** System settings **/ /** System settings **/
INSERT INTO admin_settings ( id, created_time, key, json_value ) INSERT INTO admin_settings ( id, created_time, tenant_id, key, json_value )
VALUES ( '6a2266e4-4612-11e7-a919-92ebcb67fe33', 1592576748000, 'general', '{ VALUES ( '6a2266e4-4612-11e7-a919-92ebcb67fe33', 1592576748000, '13814000-1dd2-11b2-8080-808080808080', 'general', '{
"baseUrl": "http://localhost:8080" "baseUrl": "http://localhost:8080"
}' ); }' );
INSERT INTO admin_settings ( id, created_time, key, json_value ) INSERT INTO admin_settings ( id, created_time, tenant_id, key, json_value )
VALUES ( '6eaaefa6-4612-11e7-a919-92ebcb67fe33', 1592576748000, 'mail', '{ VALUES ( '6eaaefa6-4612-11e7-a919-92ebcb67fe33', 1592576748000, '13814000-1dd2-11b2-8080-808080808080', 'mail', '{
"mailFrom": "Thingsboard <sysadmin@localhost.localdomain>", "mailFrom": "Thingsboard <sysadmin@localhost.localdomain>",
"smtpProtocol": "smtp", "smtpProtocol": "smtp",
"smtpHost": "localhost", "smtpHost": "localhost",