From 2a89fc67aea28844c65444e3da579963c4a56254 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Mon, 23 May 2022 13:20:00 +0300 Subject: [PATCH] Git api usage improvements --- .../DefaultEntitiesVersionControlService.java | 29 ++++++----- .../vc/LocalGitVersionControlService.java | 20 +++++++- .../src/main/resources/thingsboard.yml | 1 + .../vc/EntitiesVersionControlSettings.java | 3 +- .../sync/vc/DefaultGitRepositoryService.java | 51 ++++++++++--------- .../server/service/sync/vc/GitRepository.java | 45 ++++++++-------- .../service/sync/vc/GitRepositoryService.java | 2 + .../server/service/sync/vc/PendingCommit.java | 2 + 8 files changed, 93 insertions(+), 60 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index d46884ca1e..4aedfb9699 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -24,37 +24,39 @@ import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.AdminSettings; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; -import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.sync.vc.*; -import org.thingsboard.server.common.data.sync.vc.request.load.EntityTypeVersionLoadConfig; -import org.thingsboard.server.dao.DaoUtil; -import org.thingsboard.server.dao.entity.EntityService; -import org.thingsboard.server.dao.settings.AdminSettingsService; -import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.security.permission.Operation; - -import org.thingsboard.server.service.sync.ie.EntitiesExportImportService; -import org.thingsboard.server.service.sync.ie.exporting.ExportableEntitiesService; +import org.thingsboard.server.common.data.sync.ThrowingRunnable; import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; import org.thingsboard.server.common.data.sync.ie.EntityImportResult; import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; +import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.common.data.sync.vc.EntityVersion; +import org.thingsboard.server.common.data.sync.vc.VersionControlAuthMethod; +import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; +import org.thingsboard.server.common.data.sync.vc.VersionLoadResult; +import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; import org.thingsboard.server.common.data.sync.vc.request.create.ComplexVersionCreateRequest; import org.thingsboard.server.common.data.sync.vc.request.create.SingleEntityVersionCreateRequest; import org.thingsboard.server.common.data.sync.vc.request.create.SyncStrategy; import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateConfig; import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; +import org.thingsboard.server.common.data.sync.vc.request.load.EntityTypeVersionLoadConfig; import org.thingsboard.server.common.data.sync.vc.request.load.EntityTypeVersionLoadRequest; import org.thingsboard.server.common.data.sync.vc.request.load.SingleEntityVersionLoadRequest; import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadConfig; import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadRequest; -import org.thingsboard.server.common.data.sync.ThrowingRunnable; +import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.settings.AdminSettingsService; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.security.model.SecurityUser; +import org.thingsboard.server.service.security.permission.Operation; +import org.thingsboard.server.service.sync.ie.EntitiesExportImportService; +import org.thingsboard.server.service.sync.ie.exporting.ExportableEntitiesService; import java.util.ArrayList; import java.util.HashMap; @@ -76,7 +78,6 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont private final EntitiesExportImportService exportImportService; private final ExportableEntitiesService exportableEntitiesService; private final AdminSettingsService adminSettingsService; - private final EntityService entityService; private final TransactionTemplate transactionTemplate; @Override diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java index d73157d256..fa767bd1d5 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java @@ -135,7 +135,13 @@ public class LocalGitVersionControlService implements GitVersionControlService { if (old != null) { gitRepositoryService.abort(old); } - gitRepositoryService.prepareCommit(pendingCommit); + try { + gitRepositoryService.prepareCommit(pendingCommit); + } catch (Exception e) { + pendingCommitMap.remove(tenantId); + gitRepositoryService.cleanUp(pendingCommit); + throw e; + } return pendingCommit; } finally { lock.unlock(); @@ -171,7 +177,9 @@ public class LocalGitVersionControlService implements GitVersionControlService { @Override public VersionCreationResult push(PendingCommit commit) { - return executeInsideLock(commit, gitRepositoryService::push); + VersionCreationResult result = executeInsideLock(commit, gitRepositoryService::push); + pendingCommitMap.remove(commit.getTenantId()); + return result; } @Override @@ -256,6 +264,10 @@ public class LocalGitVersionControlService implements GitVersionControlService { try { checkCommit(commit); r.accept(commit); + } catch (Exception e) { + pendingCommitMap.remove(commit.getTenantId()); + gitRepositoryService.cleanUp(commit); + throw e; } finally { lock.unlock(); } @@ -267,6 +279,10 @@ public class LocalGitVersionControlService implements GitVersionControlService { try { checkCommit(commit); return c.apply(commit); + } catch (Exception e) { + pendingCommitMap.remove(commit.getTenantId()); + gitRepositoryService.cleanUp(commit); + throw e; } finally { lock.unlock(); } diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 654929f500..115b4d2ebd 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1119,6 +1119,7 @@ vc: git: service: "${JS_VC_GIT_SERVICE:local}" # local/remote repos-poll-interval: "${TB_VC_GIT_REPOS_POLL_INTERVAL_SEC:60}" + repositories-folder: "${TB_VC_GIT_REPOSITORIES_FOLDER:${java.io.tmpdir}/repositories}" management: endpoints: diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntitiesVersionControlSettings.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntitiesVersionControlSettings.java index 4fdff1f413..308f412228 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntitiesVersionControlSettings.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntitiesVersionControlSettings.java @@ -16,6 +16,7 @@ package org.thingsboard.server.common.data.sync.vc; import lombok.Data; + @Data public class EntitiesVersionControlSettings { private String repositoryUri; @@ -26,4 +27,4 @@ public class EntitiesVersionControlSettings { private String privateKey; private String privateKeyPassword; private String defaultBranch; -} \ No newline at end of file +} diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java index aac40e0dcb..e85dee8e2b 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java @@ -20,8 +20,6 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; import org.eclipse.jgit.api.errors.GitAPIException; -import org.eclipse.jgit.api.errors.JGitInternalException; -import org.eclipse.jgit.api.errors.RefAlreadyExistsException; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Service; @@ -55,7 +53,7 @@ import java.util.stream.Collectors; @Service public class DefaultGitRepositoryService implements GitRepositoryService { - @Value("${vc.git.repos-poll-interval:${java.io.tmpdir}/repositories}") + @Value("${vc.git.repositories-folder}") private String repositoriesFolder; @Value("${vc.git.repos-poll-interval:60}") @@ -92,22 +90,12 @@ public class DefaultGitRepositoryService implements GitRepositoryService { String branch = commit.getRequest().getBranch(); try { repository.fetch(); - if (repository.listBranches().contains(branch)) { - repository.checkout("origin/" + branch, false); - try { - repository.checkout(branch, true); - } catch (RefAlreadyExistsException e) { - repository.checkout(branch, false); - } + + repository.createAndCheckoutOrphanBranch(commit.getWorkingBranch()); + repository.resetAndClean(); + + if (repository.listRemoteBranches().contains(branch)) { repository.merge(branch); - } else { // TODO [viacheslav]: rollback orphan branch on failure - try { - repository.createAndCheckoutOrphanBranch(branch); // FIXME [viacheslav]: Checkout returned unexpected result NO_CHANGE for master branch - } catch (JGitInternalException e) { - if (!e.getMessage().contains("NO_CHANGE")) { - throw e; - } - } } } catch (IOException | GitAPIException gitAPIException) { //TODO: analyze and return meaningful exceptions that we can show to the client; @@ -140,32 +128,49 @@ public class DefaultGitRepositoryService implements GitRepositoryService { result.setRemoved(status.getRemoved().size()); GitRepository.Commit gitCommit = repository.commit(commit.getRequest().getVersionName()); - repository.push(); + repository.push(commit.getWorkingBranch(), commit.getRequest().getBranch()); result.setVersion(toVersion(gitCommit)); return result; } catch (GitAPIException gitAPIException) { //TODO: analyze and return meaningful exceptions that we can show to the client; throw new RuntimeException(gitAPIException); + } finally { + cleanUp(commit); } } + @SneakyThrows + @Override + public void cleanUp(PendingCommit commit) { + GitRepository repository = checkRepository(commit.getTenantId()); + try { + repository.createAndCheckoutOrphanBranch(EntityId.NULL_UUID.toString()); + } catch (Exception e) { + if (!e.getMessage().contains("NO_CHANGE")) { + throw e; + } + } + repository.resetAndClean(); + repository.deleteLocalBranchIfExists(commit.getWorkingBranch()); + } + @Override public void abort(PendingCommit commit) { - //TODO: implement; + cleanUp(commit); } @Override public String getFileContentAtCommit(TenantId tenantId, String relativePath, String versionId) throws IOException { - GitRepository repository = checkRepository(tenantId); - return repository.getFileContentAtCommit(relativePath, versionId); + GitRepository repository = checkRepository(tenantId); + return repository.getFileContentAtCommit(relativePath, versionId); } @Override public List listBranches(TenantId tenantId) { GitRepository repository = checkRepository(tenantId); try { - return repository.listBranches(); + return repository.listRemoteBranches(); } catch (GitAPIException gitAPIException) { //TODO: analyze and return meaningful exceptions that we can show to the client; throw new RuntimeException(gitAPIException); diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java index fa4891d35a..5f1a13ec53 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java @@ -18,8 +18,16 @@ package org.thingsboard.server.service.sync.vc; import com.google.common.collect.Streams; import lombok.Data; import lombok.Getter; +import org.apache.commons.lang3.StringUtils; import org.apache.sshd.common.util.security.SecurityUtils; -import org.eclipse.jgit.api.*; +import org.eclipse.jgit.api.CloneCommand; +import org.eclipse.jgit.api.Git; +import org.eclipse.jgit.api.GitCommand; +import org.eclipse.jgit.api.ListBranchCommand; +import org.eclipse.jgit.api.LogCommand; +import org.eclipse.jgit.api.LsRemoteCommand; +import org.eclipse.jgit.api.ResetCommand; +import org.eclipse.jgit.api.TransportCommand; import org.eclipse.jgit.api.errors.GitAPIException; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.ObjectId; @@ -28,6 +36,7 @@ import org.eclipse.jgit.lib.ObjectReader; import org.eclipse.jgit.revwalk.RevCommit; import org.eclipse.jgit.revwalk.filter.RevFilter; import org.eclipse.jgit.transport.CredentialsProvider; +import org.eclipse.jgit.transport.RefSpec; import org.eclipse.jgit.transport.SshTransport; import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider; import org.eclipse.jgit.transport.sshd.JGitKeyCache; @@ -36,7 +45,6 @@ import org.eclipse.jgit.transport.sshd.SshdSessionFactory; import org.eclipse.jgit.transport.sshd.SshdSessionFactoryBuilder; import org.eclipse.jgit.treewalk.TreeWalk; import org.eclipse.jgit.treewalk.filter.PathFilter; -import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; import org.thingsboard.server.common.data.sync.vc.VersionControlAuthMethod; @@ -116,15 +124,18 @@ public class GitRepository { .setRemoveDeletedRefs(true)); } - public void checkout(String branch, boolean createBranch) throws GitAPIException { - execute(git.checkout() - .setCreateBranch(createBranch) - .setName(branch)); + public void deleteLocalBranchIfExists(String branch) throws GitAPIException { + execute(git.branchDelete() + .setBranchNames(branch) + .setForce(true)); } - public void reset() throws GitAPIException { + public void resetAndClean() throws GitAPIException { execute(git.reset() .setMode(ResetCommand.ResetType.HARD)); + execute(git.clean() + .setForce(true) + .setCleanDirectories(true)); } public void merge(String branch) throws IOException, GitAPIException { @@ -137,9 +148,9 @@ public class GitRepository { } - public List listBranches() throws GitAPIException { + public List listRemoteBranches() throws GitAPIException { return execute(git.branchList() - .setListMode(ListBranchCommand.ListMode.ALL)).stream() + .setListMode(ListBranchCommand.ListMode.REMOTE)).stream() .filter(ref -> !ref.getName().equals(Constants.HEAD)) .map(ref -> org.eclipse.jgit.lib.Repository.shortenRefName(ref.getName())) .map(name -> StringUtils.removeStart(name, "origin/")) @@ -209,16 +220,9 @@ public class GitRepository { .setOrphan(true) .setForced(true) .setName(name)); -// Set uncommittedChanges = git.status().call().getUncommittedChanges(); -// if (!uncommittedChanges.isEmpty()) { -// RmCommand rm = git.rm(); -// uncommittedChanges.forEach(rm::addFilepattern); -// execute(rm); -// } -// execute(git.clean()); } - public void add(String filesPattern) throws GitAPIException { // FIXME [viacheslav] + public void add(String filesPattern) throws GitAPIException { execute(git.add().setUpdate(true).addFilepattern(filesPattern)); execute(git.add().addFilepattern(filesPattern)); } @@ -230,13 +234,14 @@ public class GitRepository { public Commit commit(String message) throws GitAPIException { RevCommit revCommit = execute(git.commit() - .setMessage(message)); // TODO [viacheslav]: set configurable author for commit + .setMessage(message)); return toCommit(revCommit); } - public void push() throws GitAPIException { - execute(git.push()); + public void push(String localBranch, String remoteBranch) throws GitAPIException { + execute(git.push() + .setRefSpecs(new RefSpec(localBranch + ":" + remoteBranch))); } diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java index 19754750c2..bbdf976804 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java @@ -44,6 +44,8 @@ public interface GitRepositoryService { VersionCreationResult push(PendingCommit commit); + void cleanUp(PendingCommit commit); + void abort(PendingCommit commit); List listBranches(TenantId tenantId); diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/PendingCommit.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/PendingCommit.java index a30f7514a2..16f3ce5e40 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/PendingCommit.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/PendingCommit.java @@ -27,10 +27,12 @@ public class PendingCommit { private final UUID txId; private final TenantId tenantId; private final VersionCreateRequest request; + private final String workingBranch; public PendingCommit(TenantId tenantId, VersionCreateRequest request) { this.txId = UUID.randomUUID(); this.tenantId = tenantId; this.request = request; + this.workingBranch = txId.toString(); } }