From 421624f3d48f9a0793077582d68d3fdc98197657 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Mon, 23 May 2022 12:41:14 +0300 Subject: [PATCH 1/5] Fixed sql test data --- dao/src/test/resources/sql/system-data.sql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dao/src/test/resources/sql/system-data.sql b/dao/src/test/resources/sql/system-data.sql index 339d6e94ba..8e38b15010 100644 --- a/dao/src/test/resources/sql/system-data.sql +++ b/dao/src/test/resources/sql/system-data.sql @@ -26,13 +26,13 @@ VALUES ( '61441950-4612-11e7-a919-92ebcb67fe33', 1592576748000, '5a797660-4612-1 '$2a$10$5JTB8/hxWc9WAy62nCGSxeefl3KWmipA9nFpVdDa0/xfIseeBB4Bu' ); /** System settings **/ -INSERT INTO admin_settings ( id, created_time, key, json_value ) -VALUES ( '6a2266e4-4612-11e7-a919-92ebcb67fe33', 1592576748000, 'general', '{ +INSERT INTO admin_settings ( id, created_time, tenant_id, key, json_value ) +VALUES ( '6a2266e4-4612-11e7-a919-92ebcb67fe33', 1592576748000, '13814000-1dd2-11b2-8080-808080808080', 'general', '{ "baseUrl": "http://localhost:8080" }' ); -INSERT INTO admin_settings ( id, created_time, key, json_value ) -VALUES ( '6eaaefa6-4612-11e7-a919-92ebcb67fe33', 1592576748000, 'mail', '{ +INSERT INTO admin_settings ( id, created_time, tenant_id, key, json_value ) +VALUES ( '6eaaefa6-4612-11e7-a919-92ebcb67fe33', 1592576748000, '13814000-1dd2-11b2-8080-808080808080', 'mail', '{ "mailFrom": "Thingsboard ", "smtpProtocol": "smtp", "smtpHost": "localhost", From 2a89fc67aea28844c65444e3da579963c4a56254 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Mon, 23 May 2022 13:20:00 +0300 Subject: [PATCH 2/5] Git api usage improvements --- .../DefaultEntitiesVersionControlService.java | 29 ++++++----- .../vc/LocalGitVersionControlService.java | 20 +++++++- .../src/main/resources/thingsboard.yml | 1 + .../vc/EntitiesVersionControlSettings.java | 3 +- .../sync/vc/DefaultGitRepositoryService.java | 51 ++++++++++--------- .../server/service/sync/vc/GitRepository.java | 45 ++++++++-------- .../service/sync/vc/GitRepositoryService.java | 2 + .../server/service/sync/vc/PendingCommit.java | 2 + 8 files changed, 93 insertions(+), 60 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index d46884ca1e..4aedfb9699 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -24,37 +24,39 @@ import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.AdminSettings; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; -import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.sync.vc.*; -import org.thingsboard.server.common.data.sync.vc.request.load.EntityTypeVersionLoadConfig; -import org.thingsboard.server.dao.DaoUtil; -import org.thingsboard.server.dao.entity.EntityService; -import org.thingsboard.server.dao.settings.AdminSettingsService; -import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.security.permission.Operation; - -import org.thingsboard.server.service.sync.ie.EntitiesExportImportService; -import org.thingsboard.server.service.sync.ie.exporting.ExportableEntitiesService; +import org.thingsboard.server.common.data.sync.ThrowingRunnable; import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; import org.thingsboard.server.common.data.sync.ie.EntityImportResult; import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; +import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.common.data.sync.vc.EntityVersion; +import org.thingsboard.server.common.data.sync.vc.VersionControlAuthMethod; +import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; +import org.thingsboard.server.common.data.sync.vc.VersionLoadResult; +import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; import org.thingsboard.server.common.data.sync.vc.request.create.ComplexVersionCreateRequest; import org.thingsboard.server.common.data.sync.vc.request.create.SingleEntityVersionCreateRequest; import org.thingsboard.server.common.data.sync.vc.request.create.SyncStrategy; import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateConfig; import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; +import org.thingsboard.server.common.data.sync.vc.request.load.EntityTypeVersionLoadConfig; import org.thingsboard.server.common.data.sync.vc.request.load.EntityTypeVersionLoadRequest; import org.thingsboard.server.common.data.sync.vc.request.load.SingleEntityVersionLoadRequest; import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadConfig; import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadRequest; -import org.thingsboard.server.common.data.sync.ThrowingRunnable; +import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.settings.AdminSettingsService; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.security.model.SecurityUser; +import org.thingsboard.server.service.security.permission.Operation; +import org.thingsboard.server.service.sync.ie.EntitiesExportImportService; +import org.thingsboard.server.service.sync.ie.exporting.ExportableEntitiesService; import java.util.ArrayList; import java.util.HashMap; @@ -76,7 +78,6 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont private final EntitiesExportImportService exportImportService; private final ExportableEntitiesService exportableEntitiesService; private final AdminSettingsService adminSettingsService; - private final EntityService entityService; private final TransactionTemplate transactionTemplate; @Override diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java index d73157d256..fa767bd1d5 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java @@ -135,7 +135,13 @@ public class LocalGitVersionControlService implements GitVersionControlService { if (old != null) { gitRepositoryService.abort(old); } - gitRepositoryService.prepareCommit(pendingCommit); + try { + gitRepositoryService.prepareCommit(pendingCommit); + } catch (Exception e) { + pendingCommitMap.remove(tenantId); + gitRepositoryService.cleanUp(pendingCommit); + throw e; + } return pendingCommit; } finally { lock.unlock(); @@ -171,7 +177,9 @@ public class LocalGitVersionControlService implements GitVersionControlService { @Override public VersionCreationResult push(PendingCommit commit) { - return executeInsideLock(commit, gitRepositoryService::push); + VersionCreationResult result = executeInsideLock(commit, gitRepositoryService::push); + pendingCommitMap.remove(commit.getTenantId()); + return result; } @Override @@ -256,6 +264,10 @@ public class LocalGitVersionControlService implements GitVersionControlService { try { checkCommit(commit); r.accept(commit); + } catch (Exception e) { + pendingCommitMap.remove(commit.getTenantId()); + gitRepositoryService.cleanUp(commit); + throw e; } finally { lock.unlock(); } @@ -267,6 +279,10 @@ public class LocalGitVersionControlService implements GitVersionControlService { try { checkCommit(commit); return c.apply(commit); + } catch (Exception e) { + pendingCommitMap.remove(commit.getTenantId()); + gitRepositoryService.cleanUp(commit); + throw e; } finally { lock.unlock(); } diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 654929f500..115b4d2ebd 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1119,6 +1119,7 @@ vc: git: service: "${JS_VC_GIT_SERVICE:local}" # local/remote repos-poll-interval: "${TB_VC_GIT_REPOS_POLL_INTERVAL_SEC:60}" + repositories-folder: "${TB_VC_GIT_REPOSITORIES_FOLDER:${java.io.tmpdir}/repositories}" management: endpoints: diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntitiesVersionControlSettings.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntitiesVersionControlSettings.java index 4fdff1f413..308f412228 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntitiesVersionControlSettings.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntitiesVersionControlSettings.java @@ -16,6 +16,7 @@ package org.thingsboard.server.common.data.sync.vc; import lombok.Data; + @Data public class EntitiesVersionControlSettings { private String repositoryUri; @@ -26,4 +27,4 @@ public class EntitiesVersionControlSettings { private String privateKey; private String privateKeyPassword; private String defaultBranch; -} \ No newline at end of file +} diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java index aac40e0dcb..e85dee8e2b 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java @@ -20,8 +20,6 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; import org.eclipse.jgit.api.errors.GitAPIException; -import org.eclipse.jgit.api.errors.JGitInternalException; -import org.eclipse.jgit.api.errors.RefAlreadyExistsException; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Service; @@ -55,7 +53,7 @@ import java.util.stream.Collectors; @Service public class DefaultGitRepositoryService implements GitRepositoryService { - @Value("${vc.git.repos-poll-interval:${java.io.tmpdir}/repositories}") + @Value("${vc.git.repositories-folder}") private String repositoriesFolder; @Value("${vc.git.repos-poll-interval:60}") @@ -92,22 +90,12 @@ public class DefaultGitRepositoryService implements GitRepositoryService { String branch = commit.getRequest().getBranch(); try { repository.fetch(); - if (repository.listBranches().contains(branch)) { - repository.checkout("origin/" + branch, false); - try { - repository.checkout(branch, true); - } catch (RefAlreadyExistsException e) { - repository.checkout(branch, false); - } + + repository.createAndCheckoutOrphanBranch(commit.getWorkingBranch()); + repository.resetAndClean(); + + if (repository.listRemoteBranches().contains(branch)) { repository.merge(branch); - } else { // TODO [viacheslav]: rollback orphan branch on failure - try { - repository.createAndCheckoutOrphanBranch(branch); // FIXME [viacheslav]: Checkout returned unexpected result NO_CHANGE for master branch - } catch (JGitInternalException e) { - if (!e.getMessage().contains("NO_CHANGE")) { - throw e; - } - } } } catch (IOException | GitAPIException gitAPIException) { //TODO: analyze and return meaningful exceptions that we can show to the client; @@ -140,32 +128,49 @@ public class DefaultGitRepositoryService implements GitRepositoryService { result.setRemoved(status.getRemoved().size()); GitRepository.Commit gitCommit = repository.commit(commit.getRequest().getVersionName()); - repository.push(); + repository.push(commit.getWorkingBranch(), commit.getRequest().getBranch()); result.setVersion(toVersion(gitCommit)); return result; } catch (GitAPIException gitAPIException) { //TODO: analyze and return meaningful exceptions that we can show to the client; throw new RuntimeException(gitAPIException); + } finally { + cleanUp(commit); } } + @SneakyThrows + @Override + public void cleanUp(PendingCommit commit) { + GitRepository repository = checkRepository(commit.getTenantId()); + try { + repository.createAndCheckoutOrphanBranch(EntityId.NULL_UUID.toString()); + } catch (Exception e) { + if (!e.getMessage().contains("NO_CHANGE")) { + throw e; + } + } + repository.resetAndClean(); + repository.deleteLocalBranchIfExists(commit.getWorkingBranch()); + } + @Override public void abort(PendingCommit commit) { - //TODO: implement; + cleanUp(commit); } @Override public String getFileContentAtCommit(TenantId tenantId, String relativePath, String versionId) throws IOException { - GitRepository repository = checkRepository(tenantId); - return repository.getFileContentAtCommit(relativePath, versionId); + GitRepository repository = checkRepository(tenantId); + return repository.getFileContentAtCommit(relativePath, versionId); } @Override public List listBranches(TenantId tenantId) { GitRepository repository = checkRepository(tenantId); try { - return repository.listBranches(); + return repository.listRemoteBranches(); } catch (GitAPIException gitAPIException) { //TODO: analyze and return meaningful exceptions that we can show to the client; throw new RuntimeException(gitAPIException); diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java index fa4891d35a..5f1a13ec53 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java @@ -18,8 +18,16 @@ package org.thingsboard.server.service.sync.vc; import com.google.common.collect.Streams; import lombok.Data; import lombok.Getter; +import org.apache.commons.lang3.StringUtils; import org.apache.sshd.common.util.security.SecurityUtils; -import org.eclipse.jgit.api.*; +import org.eclipse.jgit.api.CloneCommand; +import org.eclipse.jgit.api.Git; +import org.eclipse.jgit.api.GitCommand; +import org.eclipse.jgit.api.ListBranchCommand; +import org.eclipse.jgit.api.LogCommand; +import org.eclipse.jgit.api.LsRemoteCommand; +import org.eclipse.jgit.api.ResetCommand; +import org.eclipse.jgit.api.TransportCommand; import org.eclipse.jgit.api.errors.GitAPIException; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.ObjectId; @@ -28,6 +36,7 @@ import org.eclipse.jgit.lib.ObjectReader; import org.eclipse.jgit.revwalk.RevCommit; import org.eclipse.jgit.revwalk.filter.RevFilter; import org.eclipse.jgit.transport.CredentialsProvider; +import org.eclipse.jgit.transport.RefSpec; import org.eclipse.jgit.transport.SshTransport; import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider; import org.eclipse.jgit.transport.sshd.JGitKeyCache; @@ -36,7 +45,6 @@ import org.eclipse.jgit.transport.sshd.SshdSessionFactory; import org.eclipse.jgit.transport.sshd.SshdSessionFactoryBuilder; import org.eclipse.jgit.treewalk.TreeWalk; import org.eclipse.jgit.treewalk.filter.PathFilter; -import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; import org.thingsboard.server.common.data.sync.vc.VersionControlAuthMethod; @@ -116,15 +124,18 @@ public class GitRepository { .setRemoveDeletedRefs(true)); } - public void checkout(String branch, boolean createBranch) throws GitAPIException { - execute(git.checkout() - .setCreateBranch(createBranch) - .setName(branch)); + public void deleteLocalBranchIfExists(String branch) throws GitAPIException { + execute(git.branchDelete() + .setBranchNames(branch) + .setForce(true)); } - public void reset() throws GitAPIException { + public void resetAndClean() throws GitAPIException { execute(git.reset() .setMode(ResetCommand.ResetType.HARD)); + execute(git.clean() + .setForce(true) + .setCleanDirectories(true)); } public void merge(String branch) throws IOException, GitAPIException { @@ -137,9 +148,9 @@ public class GitRepository { } - public List listBranches() throws GitAPIException { + public List listRemoteBranches() throws GitAPIException { return execute(git.branchList() - .setListMode(ListBranchCommand.ListMode.ALL)).stream() + .setListMode(ListBranchCommand.ListMode.REMOTE)).stream() .filter(ref -> !ref.getName().equals(Constants.HEAD)) .map(ref -> org.eclipse.jgit.lib.Repository.shortenRefName(ref.getName())) .map(name -> StringUtils.removeStart(name, "origin/")) @@ -209,16 +220,9 @@ public class GitRepository { .setOrphan(true) .setForced(true) .setName(name)); -// Set uncommittedChanges = git.status().call().getUncommittedChanges(); -// if (!uncommittedChanges.isEmpty()) { -// RmCommand rm = git.rm(); -// uncommittedChanges.forEach(rm::addFilepattern); -// execute(rm); -// } -// execute(git.clean()); } - public void add(String filesPattern) throws GitAPIException { // FIXME [viacheslav] + public void add(String filesPattern) throws GitAPIException { execute(git.add().setUpdate(true).addFilepattern(filesPattern)); execute(git.add().addFilepattern(filesPattern)); } @@ -230,13 +234,14 @@ public class GitRepository { public Commit commit(String message) throws GitAPIException { RevCommit revCommit = execute(git.commit() - .setMessage(message)); // TODO [viacheslav]: set configurable author for commit + .setMessage(message)); return toCommit(revCommit); } - public void push() throws GitAPIException { - execute(git.push()); + public void push(String localBranch, String remoteBranch) throws GitAPIException { + execute(git.push() + .setRefSpecs(new RefSpec(localBranch + ":" + remoteBranch))); } diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java index 19754750c2..bbdf976804 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java @@ -44,6 +44,8 @@ public interface GitRepositoryService { VersionCreationResult push(PendingCommit commit); + void cleanUp(PendingCommit commit); + void abort(PendingCommit commit); List listBranches(TenantId tenantId); diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/PendingCommit.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/PendingCommit.java index a30f7514a2..16f3ce5e40 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/PendingCommit.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/PendingCommit.java @@ -27,10 +27,12 @@ public class PendingCommit { private final UUID txId; private final TenantId tenantId; private final VersionCreateRequest request; + private final String workingBranch; public PendingCommit(TenantId tenantId, VersionCreateRequest request) { this.txId = UUID.randomUUID(); this.tenantId = tenantId; this.request = request; + this.workingBranch = txId.toString(); } } From 55966e033e75e7e7d5d13f4d55452f4698f76189 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Mon, 23 May 2022 14:04:12 +0300 Subject: [PATCH 3/5] Fixes for vc api --- .../EntitiesVersionControlController.java | 2 +- .../sync/vc/LocalGitVersionControlService.java | 14 ++++++-------- .../common/data/sync/ie/EntityExportData.java | 3 ++- .../common/data/sync/ie/EntityImportResult.java | 6 ------ 4 files changed, 9 insertions(+), 16 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java index bba5d20b06..39a8dc464f 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java @@ -223,7 +223,7 @@ public class EntitiesVersionControlController extends BaseController { if (versionId == null) { List versions = versionControlService.listVersions(user.getTenantId(), request.getBranch()); if (versions.size() > 0) { - versionId = versions.get(0).getId(); + request.setVersionId(versions.get(0).getId()); } else { throw new IllegalArgumentException("No versions available in branch"); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java index fa767bd1d5..8aa4b793df 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java @@ -16,21 +16,17 @@ package org.thingsboard.server.service.sync.vc; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.ObjectWriter; import com.fasterxml.jackson.databind.SerializationFeature; import lombok.RequiredArgsConstructor; -import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.AdminSettings; -import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; import org.thingsboard.server.common.data.sync.vc.EntityVersion; @@ -38,7 +34,6 @@ import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; import org.thingsboard.server.dao.DaoUtil; -import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.settings.AdminSettingsService; import org.thingsboard.server.dao.tenant.TenantDao; import org.thingsboard.server.queue.util.AfterStartUp; @@ -62,13 +57,16 @@ import java.util.stream.Collectors; @ConditionalOnProperty(prefix = "vc", value = "git.service", havingValue = "local", matchIfMissing = true) public class LocalGitVersionControlService implements GitVersionControlService { - private final ObjectWriter jsonWriter = new ObjectMapper().writer(SerializationFeature.INDENT_OUTPUT); private final GitRepositoryService gitRepositoryService; private final TenantDao tenantDao; private final AdminSettingsService adminSettingsService; + private final ConcurrentMap tenantRepoLocks = new ConcurrentHashMap<>(); private final Map pendingCommitMap = new HashMap<>(); + private final ObjectMapper jsonMapper = new ObjectMapper() + .enable(SerializationFeature.INDENT_OUTPUT); + @AfterStartUp public void init() { DaoUtil.processInBatches(tenantDao::findTenantsIds, 100, tenantId -> { @@ -165,7 +163,7 @@ public class LocalGitVersionControlService implements GitVersionControlService { doInsideLock(commit, c -> { String entityDataJson; try { - entityDataJson = jsonWriter.writeValueAsString(entityData); + entityDataJson = jsonMapper.writeValueAsString(entityData); gitRepositoryService.add(c, getRelativePath(entityData.getEntityType(), entityData.getEntity().getId().toString()), entityDataJson); } catch (IOException e) { @@ -230,7 +228,7 @@ public class LocalGitVersionControlService implements GitVersionControlService { try { String entityDataJson = gitRepositoryService.getFileContentAtCommit(tenantId, getRelativePath(entityId.getEntityType(), entityId.getId().toString()), versionId); - return JacksonUtil.fromString(entityDataJson, EntityExportData.class); + return jsonMapper.readValue(entityDataJson, EntityExportData.class); } catch (Exception e) { //TODO: analyze and return meaningful exceptions that we can show to the client; throw new RuntimeException(e); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java index b361a302fc..35fceafe43 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java @@ -20,6 +20,7 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonSubTypes.Type; import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.annotation.JsonTypeInfo.As; import lombok.Data; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; @@ -30,7 +31,7 @@ import org.thingsboard.server.common.data.sync.JsonTbEntity; import java.util.List; @JsonIgnoreProperties(ignoreUnknown = true) -@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "entityType", defaultImpl = EntityExportData.class) +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "entityType", include = As.EXISTING_PROPERTY, visible = true, defaultImpl = EntityExportData.class) @JsonSubTypes({ @Type(name = "DEVICE", value = DeviceExportData.class), @Type(name = "RULE_CHAIN", value = RuleChainExportData.class) diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportResult.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportResult.java index ea69425a2c..8d6d9ff7b6 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportResult.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportResult.java @@ -15,26 +15,20 @@ */ package org.thingsboard.server.common.data.sync.ie; -import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Data; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.sync.JsonTbEntity; import org.thingsboard.server.common.data.sync.ThrowingRunnable; @Data public class EntityImportResult> { - @JsonTbEntity private E savedEntity; - @JsonTbEntity private E oldEntity; private EntityType entityType; - @JsonIgnore private ThrowingRunnable saveReferencesCallback = () -> {}; - @JsonIgnore private ThrowingRunnable sendEventsCallback = () -> {}; public void addSaveReferencesCallback(ThrowingRunnable callback) { From 2dcdb4af8e561fedd600b8af5cd86544251e91c1 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Mon, 23 May 2022 15:36:57 +0300 Subject: [PATCH 4/5] Fix Entities Version Control controller - add TbCoreComponent annotation --- .../server/controller/EntitiesVersionControlController.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java index bba5d20b06..a2b0bf3859 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java @@ -25,6 +25,7 @@ import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService; import org.thingsboard.server.common.data.sync.vc.EntityVersion; @@ -41,6 +42,7 @@ import java.util.UUID; import static org.thingsboard.server.controller.ControllerConstants.NEW_LINE; @RestController +@TbCoreComponent @RequestMapping("/api/entities/vc") @PreAuthorize("hasAuthority('TENANT_ADMIN')") @RequiredArgsConstructor From 0d57f2f6c2baf8200066a6c34b67d2bdb87a97ef Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Mon, 23 May 2022 18:19:07 +0300 Subject: [PATCH 5/5] Use page link to get versions list --- .../server/controller/AdminController.java | 14 ++++++ .../EntitiesVersionControlController.java | 50 +++++++++++++------ .../DefaultEntitiesVersionControlService.java | 26 +++++----- .../vc/EntitiesVersionControlService.java | 8 +-- .../vc/LocalGitVersionControlService.java | 18 ++++--- common/version-control/pom.xml | 1 - .../sync/vc/DefaultGitRepositoryService.java | 10 ++-- .../server/service/sync/vc/GitRepository.java | 44 ++++++++++------ .../service/sync/vc/GitRepositoryService.java | 4 +- .../sync/vc/GitVersionControlService.java | 8 +-- 10 files changed, 118 insertions(+), 65 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/AdminController.java b/application/src/main/java/org/thingsboard/server/controller/AdminController.java index 45b8aa5f95..ae9bf18bcf 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AdminController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AdminController.java @@ -203,6 +203,20 @@ public class AdminController extends BaseController { } } + @ApiOperation(value = "Check version control settings exists (versionControlSettingsExists)", + notes = "Check whether the version control settings exists. " + TENANT_AUTHORITY_PARAGRAPH) + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + @GetMapping("/vcSettings/exists") + @ResponseBody + public Boolean versionControlSettingsExists() throws ThingsboardException { + try { + accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.READ); + return versionControlService.getVersionControlSettings(getTenantId()) != null; + } catch (Exception e) { + throw handleException(e); + } + } + @ApiOperation(value = "Creates or Updates the version control settings (saveVersionControlSettings)", notes = "Creates or Updates the version control settings object. " + TENANT_AUTHORITY_PARAGRAPH) @PreAuthorize("hasAuthority('TENANT_ADMIN')") diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java index a2b0bf3859..4086788fea 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java @@ -16,6 +16,7 @@ package org.thingsboard.server.controller; import io.swagger.annotations.ApiOperation; +import io.swagger.annotations.ApiParam; import lombok.Data; import lombok.RequiredArgsConstructor; import org.springframework.security.access.prepost.PreAuthorize; @@ -25,6 +26,8 @@ import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService; @@ -39,7 +42,7 @@ import java.util.ArrayList; import java.util.List; import java.util.UUID; -import static org.thingsboard.server.controller.ControllerConstants.NEW_LINE; +import static org.thingsboard.server.controller.ControllerConstants.*; @RestController @TbCoreComponent @@ -109,13 +112,18 @@ public class EntitiesVersionControlController extends BaseController { " \"name\": \"Device profile 1 version 1.0\"\n" + " }\n" + "]\n```") - @GetMapping("/version/{branch}/{entityType}/{externalEntityUuid}") - public List listEntityVersions(@PathVariable String branch, - @PathVariable EntityType entityType, - @PathVariable UUID externalEntityUuid) throws ThingsboardException { + @GetMapping(value = "/version/{branch}/{entityType}/{externalEntityUuid}", params = {"pageSize", "page"}) + public PageData listEntityVersions(@PathVariable String branch, + @PathVariable EntityType entityType, + @PathVariable UUID externalEntityUuid, + @ApiParam(value = PAGE_SIZE_DESCRIPTION, required = true) + @RequestParam int pageSize, + @ApiParam(value = PAGE_NUMBER_DESCRIPTION, required = true) + @RequestParam int page) throws ThingsboardException { try { EntityId externalEntityId = EntityIdFactory.getByTypeAndUuid(entityType, externalEntityUuid); - return versionControlService.listEntityVersions(getTenantId(), branch, externalEntityId); + PageLink pageLink = new PageLink(pageSize, page); + return versionControlService.listEntityVersions(getTenantId(), branch, externalEntityId, pageLink); } catch (Exception e) { throw handleException(e); } @@ -128,11 +136,16 @@ public class EntitiesVersionControlController extends BaseController { " \"name\": \"Device profiles from dev\"\n" + " }\n" + "]\n```") - @GetMapping("/version/{branch}/{entityType}") - public List listEntityTypeVersions(@PathVariable String branch, - @PathVariable EntityType entityType) throws ThingsboardException { + @GetMapping(value = "/version/{branch}/{entityType}", params = {"pageSize", "page"}) + public PageData listEntityTypeVersions(@PathVariable String branch, + @PathVariable EntityType entityType, + @ApiParam(value = PAGE_SIZE_DESCRIPTION, required = true) + @RequestParam int pageSize, + @ApiParam(value = PAGE_NUMBER_DESCRIPTION, required = true) + @RequestParam int page) throws ThingsboardException { try { - return versionControlService.listEntityTypeVersions(getTenantId(), branch, entityType); + PageLink pageLink = new PageLink(pageSize, page); + return versionControlService.listEntityTypeVersions(getTenantId(), branch, entityType, pageLink); } catch (Exception e) { throw handleException(e); } @@ -153,10 +166,15 @@ public class EntitiesVersionControlController extends BaseController { " \"name\": \"Devices added\"\n" + " }\n" + "]\n```") - @GetMapping("/version/{branch}") - public List listVersions(@PathVariable String branch) throws ThingsboardException { + @GetMapping(value = "/version/{branch}", params = {"pageSize", "page"}) + public PageData listVersions(@PathVariable String branch, + @ApiParam(value = PAGE_SIZE_DESCRIPTION, required = true) + @RequestParam int pageSize, + @ApiParam(value = PAGE_NUMBER_DESCRIPTION, required = true) + @RequestParam int page) throws ThingsboardException { try { - return versionControlService.listVersions(getTenantId(), branch); + PageLink pageLink = new PageLink(pageSize, page); + return versionControlService.listVersions(getTenantId(), branch, pageLink); } catch (Exception e) { throw handleException(e); } @@ -223,9 +241,9 @@ public class EntitiesVersionControlController extends BaseController { try { String versionId = request.getVersionId(); if (versionId == null) { - List versions = versionControlService.listVersions(user.getTenantId(), request.getBranch()); - if (versions.size() > 0) { - versionId = versions.get(0).getId(); + PageData versions = versionControlService.listVersions(user.getTenantId(), request.getBranch(), new PageLink(1)); + if (versions.getData().size() > 0) { + versionId = versions.getData().get(0).getId(); } else { throw new IllegalArgumentException("No versions available in branch"); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index 4aedfb9699..8f0d11068e 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -29,6 +29,8 @@ import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.sync.ThrowingRunnable; import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; @@ -134,18 +136,18 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont @Override - public List listEntityVersions(TenantId tenantId, String branch, EntityId externalId) throws Exception { - return gitService.listVersions(tenantId, branch, externalId); + public PageData listEntityVersions(TenantId tenantId, String branch, EntityId externalId, PageLink pageLink) throws Exception { + return gitService.listVersions(tenantId, branch, externalId, pageLink); } @Override - public List listEntityTypeVersions(TenantId tenantId, String branch, EntityType entityType) throws Exception { - return gitService.listVersions(tenantId, branch, entityType); + public PageData listEntityTypeVersions(TenantId tenantId, String branch, EntityType entityType, PageLink pageLink) throws Exception { + return gitService.listVersions(tenantId, branch, entityType, pageLink); } @Override - public List listVersions(TenantId tenantId, String branch) throws Exception { - return gitService.listVersions(tenantId, branch); + public PageData listVersions(TenantId tenantId, String branch, PageLink pageLink) throws Exception { + return gitService.listVersions(tenantId, branch, pageLink); } @Override @@ -307,6 +309,12 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont adminSettings.setKey(SETTINGS_KEY); adminSettings.setTenantId(tenantId); } + try { + gitService.clearRepository(tenantId); + gitService.initRepository(tenantId, versionControlSettings); + } catch (Exception e) { + throw new RuntimeException("Failed to init repository!", e); + } adminSettings.setJsonValue(JacksonUtil.valueToTree(versionControlSettings)); AdminSettings savedAdminSettings = adminSettingsService.saveAdminSettings(tenantId, adminSettings); EntitiesVersionControlSettings savedVersionControlSettings; @@ -315,12 +323,6 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } catch (Exception e) { throw new RuntimeException("Failed to load version control settings!", e); } - try { - gitService.clearRepository(tenantId); - gitService.initRepository(tenantId, savedVersionControlSettings); - } catch (Exception e) { - throw new RuntimeException("Failed to init repository!", e); - } return savedVersionControlSettings; } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java index 28f095123c..a70b78aaf7 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java @@ -19,6 +19,8 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; import org.thingsboard.server.common.data.sync.vc.EntityVersion; @@ -37,11 +39,11 @@ public interface EntitiesVersionControlService { VersionCreationResult saveEntitiesVersion(SecurityUser user, VersionCreateRequest request) throws Exception; - List listEntityVersions(TenantId tenantId, String branch, EntityId externalId) throws Exception; + PageData listEntityVersions(TenantId tenantId, String branch, EntityId externalId, PageLink pageLink) throws Exception; - List listEntityTypeVersions(TenantId tenantId, String branch, EntityType entityType) throws Exception; + PageData listEntityTypeVersions(TenantId tenantId, String branch, EntityType entityType, PageLink pageLink) throws Exception; - List listVersions(TenantId tenantId, String branch) throws Exception; + PageData listVersions(TenantId tenantId, String branch, PageLink pageLink) throws Exception; List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType) throws Exception; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java index fa767bd1d5..42a1308901 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java @@ -31,6 +31,8 @@ import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; import org.thingsboard.server.common.data.sync.vc.EntityVersion; @@ -183,18 +185,18 @@ public class LocalGitVersionControlService implements GitVersionControlService { } @Override - public List listVersions(TenantId tenantId, String branch) { - return listVersions(tenantId, branch, (String) null); + public PageData listVersions(TenantId tenantId, String branch, PageLink pageLink) { + return listVersions(tenantId, branch, (String) null, pageLink); } @Override - public List listVersions(TenantId tenantId, String branch, EntityType entityType) { - return listVersions(tenantId, branch, getRelativePath(entityType, null)); + public PageData listVersions(TenantId tenantId, String branch, EntityType entityType, PageLink pageLink) { + return listVersions(tenantId, branch, getRelativePath(entityType, null), pageLink); } @Override - public List listVersions(TenantId tenantId, String branch, EntityId entityId) { - return listVersions(tenantId, branch, getRelativePath(entityId.getEntityType(), entityId.getId().toString())); + public PageData listVersions(TenantId tenantId, String branch, EntityId entityId, PageLink pageLink) { + return listVersions(tenantId, branch, getRelativePath(entityId.getEntityType(), entityId.getId().toString()), pageLink); } @Override @@ -249,9 +251,9 @@ public class LocalGitVersionControlService implements GitVersionControlService { return null; } - private List listVersions(TenantId tenantId, String branch, String path) { + private PageData listVersions(TenantId tenantId, String branch, String path, PageLink pageLink) { try { - return gitRepositoryService.listVersions(tenantId, branch, path); + return gitRepositoryService.listVersions(tenantId, branch, path, pageLink); } catch (Exception e) { //TODO: analyze and return meaningful exceptions that we can show to the client; throw new RuntimeException(e); diff --git a/common/version-control/pom.xml b/common/version-control/pom.xml index b01b466cb5..89c36b1c96 100644 --- a/common/version-control/pom.xml +++ b/common/version-control/pom.xml @@ -60,7 +60,6 @@ com.google.guava guava - provided com.fasterxml.jackson.core diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java index e85dee8e2b..b822ff3b5b 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java @@ -27,6 +27,8 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; import org.thingsboard.server.common.data.sync.vc.EntityVersion; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; @@ -178,7 +180,7 @@ public class DefaultGitRepositoryService implements GitRepositoryService { } private EntityVersion checkVersion(TenantId tenantId, String branch, String versionId) throws Exception { - return listVersions(tenantId, branch, null).stream() + return listVersions(tenantId, branch, null, new PageLink(Integer.MAX_VALUE)).getData().stream() .filter(version -> version.getId().equals(versionId)) .findFirst().orElseThrow(() -> new IllegalArgumentException("Version not found")); } @@ -189,11 +191,9 @@ public class DefaultGitRepositoryService implements GitRepositoryService { } @Override - public List listVersions(TenantId tenantId, String branch, String path) throws Exception { + public PageData listVersions(TenantId tenantId, String branch, String path, PageLink pageLink) throws Exception { GitRepository repository = checkRepository(tenantId); - return repository.listCommits(branch, path, Integer.MAX_VALUE).stream() - .map(this::toVersion) - .collect(Collectors.toList()); + return repository.listCommits(branch, path, pageLink).mapData(this::toVersion); } @Override diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java index 5f1a13ec53..22be22d991 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java @@ -15,19 +15,13 @@ */ package org.thingsboard.server.service.sync.vc; +import com.google.common.collect.Iterables; import com.google.common.collect.Streams; import lombok.Data; import lombok.Getter; import org.apache.commons.lang3.StringUtils; import org.apache.sshd.common.util.security.SecurityUtils; -import org.eclipse.jgit.api.CloneCommand; -import org.eclipse.jgit.api.Git; -import org.eclipse.jgit.api.GitCommand; -import org.eclipse.jgit.api.ListBranchCommand; -import org.eclipse.jgit.api.LogCommand; -import org.eclipse.jgit.api.LsRemoteCommand; -import org.eclipse.jgit.api.ResetCommand; -import org.eclipse.jgit.api.TransportCommand; +import org.eclipse.jgit.api.*; import org.eclipse.jgit.api.errors.GitAPIException; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.ObjectId; @@ -45,6 +39,8 @@ import org.eclipse.jgit.transport.sshd.SshdSessionFactory; import org.eclipse.jgit.transport.sshd.SshdSessionFactoryBuilder; import org.eclipse.jgit.treewalk.TreeWalk; import org.eclipse.jgit.treewalk.filter.PathFilter; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; import org.thingsboard.server.common.data.sync.vc.VersionControlAuthMethod; @@ -59,6 +55,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Set; +import java.util.function.Function; import java.util.stream.Collectors; public class GitRepository { @@ -157,27 +154,25 @@ public class GitRepository { .distinct().collect(Collectors.toList()); } - public List listCommits(String branch, int limit) throws IOException, GitAPIException { - return listCommits(branch, null, limit); + public PageData listCommits(String branch, PageLink pageLink) throws IOException, GitAPIException { + return listCommits(branch, null, pageLink); } - public List listCommits(String branch, String path, int limit) throws IOException, GitAPIException { + public PageData listCommits(String branch, String path, PageLink pageLink) throws IOException, GitAPIException { ObjectId branchId = resolve("origin/" + branch); if (branchId == null) { throw new IllegalArgumentException("Branch not found"); } LogCommand command = git.log() - .add(branchId).setMaxCount(limit) + .add(branchId) .setRevFilter(RevFilter.NO_MERGES); if (StringUtils.isNotEmpty(path)) { command.addPath(path); } - return Streams.stream(execute(command)) - .map(this::toCommit) - .collect(Collectors.toList()); + Iterable commits = execute(command); + return iterableToPageData(commits, this::toCommit, pageLink); } - public List listFilesAtCommit(String commitId) throws IOException { return listFilesAtCommit(commitId, null); } @@ -295,6 +290,23 @@ public class GitRepository { return command.call(); } + private static PageData iterableToPageData (Iterable iterable, Function mapper, PageLink pageLink) { + int totalElements = Iterables.size(iterable); + int totalPages = pageLink.getPageSize() > 0 ? (int) Math.ceil((float) totalElements / pageLink.getPageSize()) : 1; + int startIndex = pageLink.getPageSize() * pageLink.getPage(); + int limit = startIndex + pageLink.getPageSize(); + iterable = Iterables.limit(iterable, limit); + if (startIndex < totalElements) { + iterable = Iterables.skip(iterable, startIndex); + } else { + iterable = Collections.emptyList(); + } + List data = Streams.stream(iterable).map(mapper) + .collect(Collectors.toList()); + boolean hasNext = pageLink.getPageSize() > 0 && totalElements > startIndex + data.size(); + return new PageData<>(data, totalPages, totalElements, hasNext); + } + private static void configureTransportCommand(TransportCommand transportCommand, CredentialsProvider credentialsProvider, SshdSessionFactory sshSessionFactory) { if (credentialsProvider != null) { transportCommand.setCredentialsProvider(credentialsProvider); diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java index bbdf976804..8fb0e0f84d 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java @@ -16,6 +16,8 @@ package org.thingsboard.server.service.sync.vc; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; import org.thingsboard.server.common.data.sync.vc.EntityVersion; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; @@ -28,7 +30,7 @@ public interface GitRepositoryService { void prepareCommit(PendingCommit pendingCommit); - List listVersions(TenantId tenantId, String branch, String path) throws Exception; + PageData listVersions(TenantId tenantId, String branch, String path, PageLink pageLink) throws Exception; List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, String path) throws Exception; diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java index f28584c418..c21efbb774 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java @@ -19,6 +19,8 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; import org.thingsboard.server.common.data.sync.vc.EntityVersion; @@ -44,11 +46,11 @@ public interface GitVersionControlService { VersionCreationResult push(PendingCommit commit); - List listVersions(TenantId tenantId, String branch); + PageData listVersions(TenantId tenantId, String branch, PageLink pageLink); - List listVersions(TenantId tenantId, String branch, EntityType entityType); + PageData listVersions(TenantId tenantId, String branch, EntityType entityType, PageLink pageLink); - List listVersions(TenantId tenantId, String branch, EntityId entityId); + PageData listVersions(TenantId tenantId, String branch, EntityId entityId, PageLink pageLink); List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType);