diff --git a/application/src/main/java/org/thingsboard/server/controller/AdminController.java b/application/src/main/java/org/thingsboard/server/controller/AdminController.java index 45b8aa5f95..ae9bf18bcf 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AdminController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AdminController.java @@ -203,6 +203,20 @@ public class AdminController extends BaseController { } } + @ApiOperation(value = "Check version control settings exists (versionControlSettingsExists)", + notes = "Check whether the version control settings exists. " + TENANT_AUTHORITY_PARAGRAPH) + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + @GetMapping("/vcSettings/exists") + @ResponseBody + public Boolean versionControlSettingsExists() throws ThingsboardException { + try { + accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.READ); + return versionControlService.getVersionControlSettings(getTenantId()) != null; + } catch (Exception e) { + throw handleException(e); + } + } + @ApiOperation(value = "Creates or Updates the version control settings (saveVersionControlSettings)", notes = "Creates or Updates the version control settings object. " + TENANT_AUTHORITY_PARAGRAPH) @PreAuthorize("hasAuthority('TENANT_ADMIN')") diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java index a2b0bf3859..4086788fea 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java @@ -16,6 +16,7 @@ package org.thingsboard.server.controller; import io.swagger.annotations.ApiOperation; +import io.swagger.annotations.ApiParam; import lombok.Data; import lombok.RequiredArgsConstructor; import org.springframework.security.access.prepost.PreAuthorize; @@ -25,6 +26,8 @@ import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService; @@ -39,7 +42,7 @@ import java.util.ArrayList; import java.util.List; import java.util.UUID; -import static org.thingsboard.server.controller.ControllerConstants.NEW_LINE; +import static org.thingsboard.server.controller.ControllerConstants.*; @RestController @TbCoreComponent @@ -109,13 +112,18 @@ public class EntitiesVersionControlController extends BaseController { " \"name\": \"Device profile 1 version 1.0\"\n" + " }\n" + "]\n```") - @GetMapping("/version/{branch}/{entityType}/{externalEntityUuid}") - public List listEntityVersions(@PathVariable String branch, - @PathVariable EntityType entityType, - @PathVariable UUID externalEntityUuid) throws ThingsboardException { + @GetMapping(value = "/version/{branch}/{entityType}/{externalEntityUuid}", params = {"pageSize", "page"}) + public PageData listEntityVersions(@PathVariable String branch, + @PathVariable EntityType entityType, + @PathVariable UUID externalEntityUuid, + @ApiParam(value = PAGE_SIZE_DESCRIPTION, required = true) + @RequestParam int pageSize, + @ApiParam(value = PAGE_NUMBER_DESCRIPTION, required = true) + @RequestParam int page) throws ThingsboardException { try { EntityId externalEntityId = EntityIdFactory.getByTypeAndUuid(entityType, externalEntityUuid); - return versionControlService.listEntityVersions(getTenantId(), branch, externalEntityId); + PageLink pageLink = new PageLink(pageSize, page); + return versionControlService.listEntityVersions(getTenantId(), branch, externalEntityId, pageLink); } catch (Exception e) { throw handleException(e); } @@ -128,11 +136,16 @@ public class EntitiesVersionControlController extends BaseController { " \"name\": \"Device profiles from dev\"\n" + " }\n" + "]\n```") - @GetMapping("/version/{branch}/{entityType}") - public List listEntityTypeVersions(@PathVariable String branch, - @PathVariable EntityType entityType) throws ThingsboardException { + @GetMapping(value = "/version/{branch}/{entityType}", params = {"pageSize", "page"}) + public PageData listEntityTypeVersions(@PathVariable String branch, + @PathVariable EntityType entityType, + @ApiParam(value = PAGE_SIZE_DESCRIPTION, required = true) + @RequestParam int pageSize, + @ApiParam(value = PAGE_NUMBER_DESCRIPTION, required = true) + @RequestParam int page) throws ThingsboardException { try { - return versionControlService.listEntityTypeVersions(getTenantId(), branch, entityType); + PageLink pageLink = new PageLink(pageSize, page); + return versionControlService.listEntityTypeVersions(getTenantId(), branch, entityType, pageLink); } catch (Exception e) { throw handleException(e); } @@ -153,10 +166,15 @@ public class EntitiesVersionControlController extends BaseController { " \"name\": \"Devices added\"\n" + " }\n" + "]\n```") - @GetMapping("/version/{branch}") - public List listVersions(@PathVariable String branch) throws ThingsboardException { + @GetMapping(value = "/version/{branch}", params = {"pageSize", "page"}) + public PageData listVersions(@PathVariable String branch, + @ApiParam(value = PAGE_SIZE_DESCRIPTION, required = true) + @RequestParam int pageSize, + @ApiParam(value = PAGE_NUMBER_DESCRIPTION, required = true) + @RequestParam int page) throws ThingsboardException { try { - return versionControlService.listVersions(getTenantId(), branch); + PageLink pageLink = new PageLink(pageSize, page); + return versionControlService.listVersions(getTenantId(), branch, pageLink); } catch (Exception e) { throw handleException(e); } @@ -223,9 +241,9 @@ public class EntitiesVersionControlController extends BaseController { try { String versionId = request.getVersionId(); if (versionId == null) { - List versions = versionControlService.listVersions(user.getTenantId(), request.getBranch()); - if (versions.size() > 0) { - versionId = versions.get(0).getId(); + PageData versions = versionControlService.listVersions(user.getTenantId(), request.getBranch(), new PageLink(1)); + if (versions.getData().size() > 0) { + versionId = versions.getData().get(0).getId(); } else { throw new IllegalArgumentException("No versions available in branch"); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index 4aedfb9699..8f0d11068e 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -29,6 +29,8 @@ import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.sync.ThrowingRunnable; import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; @@ -134,18 +136,18 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont @Override - public List listEntityVersions(TenantId tenantId, String branch, EntityId externalId) throws Exception { - return gitService.listVersions(tenantId, branch, externalId); + public PageData listEntityVersions(TenantId tenantId, String branch, EntityId externalId, PageLink pageLink) throws Exception { + return gitService.listVersions(tenantId, branch, externalId, pageLink); } @Override - public List listEntityTypeVersions(TenantId tenantId, String branch, EntityType entityType) throws Exception { - return gitService.listVersions(tenantId, branch, entityType); + public PageData listEntityTypeVersions(TenantId tenantId, String branch, EntityType entityType, PageLink pageLink) throws Exception { + return gitService.listVersions(tenantId, branch, entityType, pageLink); } @Override - public List listVersions(TenantId tenantId, String branch) throws Exception { - return gitService.listVersions(tenantId, branch); + public PageData listVersions(TenantId tenantId, String branch, PageLink pageLink) throws Exception { + return gitService.listVersions(tenantId, branch, pageLink); } @Override @@ -307,6 +309,12 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont adminSettings.setKey(SETTINGS_KEY); adminSettings.setTenantId(tenantId); } + try { + gitService.clearRepository(tenantId); + gitService.initRepository(tenantId, versionControlSettings); + } catch (Exception e) { + throw new RuntimeException("Failed to init repository!", e); + } adminSettings.setJsonValue(JacksonUtil.valueToTree(versionControlSettings)); AdminSettings savedAdminSettings = adminSettingsService.saveAdminSettings(tenantId, adminSettings); EntitiesVersionControlSettings savedVersionControlSettings; @@ -315,12 +323,6 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } catch (Exception e) { throw new RuntimeException("Failed to load version control settings!", e); } - try { - gitService.clearRepository(tenantId); - gitService.initRepository(tenantId, savedVersionControlSettings); - } catch (Exception e) { - throw new RuntimeException("Failed to init repository!", e); - } return savedVersionControlSettings; } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java index 28f095123c..a70b78aaf7 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java @@ -19,6 +19,8 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; import org.thingsboard.server.common.data.sync.vc.EntityVersion; @@ -37,11 +39,11 @@ public interface EntitiesVersionControlService { VersionCreationResult saveEntitiesVersion(SecurityUser user, VersionCreateRequest request) throws Exception; - List listEntityVersions(TenantId tenantId, String branch, EntityId externalId) throws Exception; + PageData listEntityVersions(TenantId tenantId, String branch, EntityId externalId, PageLink pageLink) throws Exception; - List listEntityTypeVersions(TenantId tenantId, String branch, EntityType entityType) throws Exception; + PageData listEntityTypeVersions(TenantId tenantId, String branch, EntityType entityType, PageLink pageLink) throws Exception; - List listVersions(TenantId tenantId, String branch) throws Exception; + PageData listVersions(TenantId tenantId, String branch, PageLink pageLink) throws Exception; List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType) throws Exception; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java index fa767bd1d5..42a1308901 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java @@ -31,6 +31,8 @@ import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; import org.thingsboard.server.common.data.sync.vc.EntityVersion; @@ -183,18 +185,18 @@ public class LocalGitVersionControlService implements GitVersionControlService { } @Override - public List listVersions(TenantId tenantId, String branch) { - return listVersions(tenantId, branch, (String) null); + public PageData listVersions(TenantId tenantId, String branch, PageLink pageLink) { + return listVersions(tenantId, branch, (String) null, pageLink); } @Override - public List listVersions(TenantId tenantId, String branch, EntityType entityType) { - return listVersions(tenantId, branch, getRelativePath(entityType, null)); + public PageData listVersions(TenantId tenantId, String branch, EntityType entityType, PageLink pageLink) { + return listVersions(tenantId, branch, getRelativePath(entityType, null), pageLink); } @Override - public List listVersions(TenantId tenantId, String branch, EntityId entityId) { - return listVersions(tenantId, branch, getRelativePath(entityId.getEntityType(), entityId.getId().toString())); + public PageData listVersions(TenantId tenantId, String branch, EntityId entityId, PageLink pageLink) { + return listVersions(tenantId, branch, getRelativePath(entityId.getEntityType(), entityId.getId().toString()), pageLink); } @Override @@ -249,9 +251,9 @@ public class LocalGitVersionControlService implements GitVersionControlService { return null; } - private List listVersions(TenantId tenantId, String branch, String path) { + private PageData listVersions(TenantId tenantId, String branch, String path, PageLink pageLink) { try { - return gitRepositoryService.listVersions(tenantId, branch, path); + return gitRepositoryService.listVersions(tenantId, branch, path, pageLink); } catch (Exception e) { //TODO: analyze and return meaningful exceptions that we can show to the client; throw new RuntimeException(e); diff --git a/common/version-control/pom.xml b/common/version-control/pom.xml index b01b466cb5..89c36b1c96 100644 --- a/common/version-control/pom.xml +++ b/common/version-control/pom.xml @@ -60,7 +60,6 @@ com.google.guava guava - provided com.fasterxml.jackson.core diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java index e85dee8e2b..b822ff3b5b 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java @@ -27,6 +27,8 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; import org.thingsboard.server.common.data.sync.vc.EntityVersion; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; @@ -178,7 +180,7 @@ public class DefaultGitRepositoryService implements GitRepositoryService { } private EntityVersion checkVersion(TenantId tenantId, String branch, String versionId) throws Exception { - return listVersions(tenantId, branch, null).stream() + return listVersions(tenantId, branch, null, new PageLink(Integer.MAX_VALUE)).getData().stream() .filter(version -> version.getId().equals(versionId)) .findFirst().orElseThrow(() -> new IllegalArgumentException("Version not found")); } @@ -189,11 +191,9 @@ public class DefaultGitRepositoryService implements GitRepositoryService { } @Override - public List listVersions(TenantId tenantId, String branch, String path) throws Exception { + public PageData listVersions(TenantId tenantId, String branch, String path, PageLink pageLink) throws Exception { GitRepository repository = checkRepository(tenantId); - return repository.listCommits(branch, path, Integer.MAX_VALUE).stream() - .map(this::toVersion) - .collect(Collectors.toList()); + return repository.listCommits(branch, path, pageLink).mapData(this::toVersion); } @Override diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java index 5f1a13ec53..22be22d991 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java @@ -15,19 +15,13 @@ */ package org.thingsboard.server.service.sync.vc; +import com.google.common.collect.Iterables; import com.google.common.collect.Streams; import lombok.Data; import lombok.Getter; import org.apache.commons.lang3.StringUtils; import org.apache.sshd.common.util.security.SecurityUtils; -import org.eclipse.jgit.api.CloneCommand; -import org.eclipse.jgit.api.Git; -import org.eclipse.jgit.api.GitCommand; -import org.eclipse.jgit.api.ListBranchCommand; -import org.eclipse.jgit.api.LogCommand; -import org.eclipse.jgit.api.LsRemoteCommand; -import org.eclipse.jgit.api.ResetCommand; -import org.eclipse.jgit.api.TransportCommand; +import org.eclipse.jgit.api.*; import org.eclipse.jgit.api.errors.GitAPIException; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.ObjectId; @@ -45,6 +39,8 @@ import org.eclipse.jgit.transport.sshd.SshdSessionFactory; import org.eclipse.jgit.transport.sshd.SshdSessionFactoryBuilder; import org.eclipse.jgit.treewalk.TreeWalk; import org.eclipse.jgit.treewalk.filter.PathFilter; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; import org.thingsboard.server.common.data.sync.vc.VersionControlAuthMethod; @@ -59,6 +55,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Set; +import java.util.function.Function; import java.util.stream.Collectors; public class GitRepository { @@ -157,27 +154,25 @@ public class GitRepository { .distinct().collect(Collectors.toList()); } - public List listCommits(String branch, int limit) throws IOException, GitAPIException { - return listCommits(branch, null, limit); + public PageData listCommits(String branch, PageLink pageLink) throws IOException, GitAPIException { + return listCommits(branch, null, pageLink); } - public List listCommits(String branch, String path, int limit) throws IOException, GitAPIException { + public PageData listCommits(String branch, String path, PageLink pageLink) throws IOException, GitAPIException { ObjectId branchId = resolve("origin/" + branch); if (branchId == null) { throw new IllegalArgumentException("Branch not found"); } LogCommand command = git.log() - .add(branchId).setMaxCount(limit) + .add(branchId) .setRevFilter(RevFilter.NO_MERGES); if (StringUtils.isNotEmpty(path)) { command.addPath(path); } - return Streams.stream(execute(command)) - .map(this::toCommit) - .collect(Collectors.toList()); + Iterable commits = execute(command); + return iterableToPageData(commits, this::toCommit, pageLink); } - public List listFilesAtCommit(String commitId) throws IOException { return listFilesAtCommit(commitId, null); } @@ -295,6 +290,23 @@ public class GitRepository { return command.call(); } + private static PageData iterableToPageData (Iterable iterable, Function mapper, PageLink pageLink) { + int totalElements = Iterables.size(iterable); + int totalPages = pageLink.getPageSize() > 0 ? (int) Math.ceil((float) totalElements / pageLink.getPageSize()) : 1; + int startIndex = pageLink.getPageSize() * pageLink.getPage(); + int limit = startIndex + pageLink.getPageSize(); + iterable = Iterables.limit(iterable, limit); + if (startIndex < totalElements) { + iterable = Iterables.skip(iterable, startIndex); + } else { + iterable = Collections.emptyList(); + } + List data = Streams.stream(iterable).map(mapper) + .collect(Collectors.toList()); + boolean hasNext = pageLink.getPageSize() > 0 && totalElements > startIndex + data.size(); + return new PageData<>(data, totalPages, totalElements, hasNext); + } + private static void configureTransportCommand(TransportCommand transportCommand, CredentialsProvider credentialsProvider, SshdSessionFactory sshSessionFactory) { if (credentialsProvider != null) { transportCommand.setCredentialsProvider(credentialsProvider); diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java index bbdf976804..8fb0e0f84d 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java @@ -16,6 +16,8 @@ package org.thingsboard.server.service.sync.vc; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; import org.thingsboard.server.common.data.sync.vc.EntityVersion; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; @@ -28,7 +30,7 @@ public interface GitRepositoryService { void prepareCommit(PendingCommit pendingCommit); - List listVersions(TenantId tenantId, String branch, String path) throws Exception; + PageData listVersions(TenantId tenantId, String branch, String path, PageLink pageLink) throws Exception; List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, String path) throws Exception; diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java index f28584c418..c21efbb774 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java @@ -19,6 +19,8 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; import org.thingsboard.server.common.data.sync.vc.EntityVersion; @@ -44,11 +46,11 @@ public interface GitVersionControlService { VersionCreationResult push(PendingCommit commit); - List listVersions(TenantId tenantId, String branch); + PageData listVersions(TenantId tenantId, String branch, PageLink pageLink); - List listVersions(TenantId tenantId, String branch, EntityType entityType); + PageData listVersions(TenantId tenantId, String branch, EntityType entityType, PageLink pageLink); - List listVersions(TenantId tenantId, String branch, EntityId entityId); + PageData listVersions(TenantId tenantId, String branch, EntityId entityId, PageLink pageLink); List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType);