Merge pull request #6585 from ViacheslavKlimov/entities-vc-refactoring

Git api usage improvements

application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java

@@ -24,37 +24,39 @@ import org.thingsboard.common.util.JacksonUtil;
 import org.thingsboard.server.common.data.AdminSettings;
 import org.thingsboard.server.common.data.EntityType;
 import org.thingsboard.server.common.data.ExportableEntity;
-import org.thingsboard.server.common.data.StringUtils;
 import org.thingsboard.server.common.data.exception.ThingsboardErrorCode;
 import org.thingsboard.server.common.data.exception.ThingsboardException;
 import org.thingsboard.server.common.data.id.EntityId;
 import org.thingsboard.server.common.data.id.EntityIdFactory;
 import org.thingsboard.server.common.data.id.TenantId;
-import org.thingsboard.server.common.data.sync.vc.*;
+import org.thingsboard.server.common.data.sync.ThrowingRunnable;
-import org.thingsboard.server.common.data.sync.vc.request.load.EntityTypeVersionLoadConfig;
-import org.thingsboard.server.dao.DaoUtil;
-import org.thingsboard.server.dao.entity.EntityService;
-import org.thingsboard.server.dao.settings.AdminSettingsService;
-import org.thingsboard.server.queue.util.TbCoreComponent;
-import org.thingsboard.server.service.security.model.SecurityUser;
-import org.thingsboard.server.service.security.permission.Operation;
-
-import org.thingsboard.server.service.sync.ie.EntitiesExportImportService;
-import org.thingsboard.server.service.sync.ie.exporting.ExportableEntitiesService;
 import org.thingsboard.server.common.data.sync.ie.EntityExportData;
 import org.thingsboard.server.common.data.sync.ie.EntityExportSettings;
 import org.thingsboard.server.common.data.sync.ie.EntityImportResult;
 import org.thingsboard.server.common.data.sync.ie.EntityImportSettings;
+import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings;
+import org.thingsboard.server.common.data.sync.vc.EntityVersion;
+import org.thingsboard.server.common.data.sync.vc.VersionControlAuthMethod;
+import org.thingsboard.server.common.data.sync.vc.VersionCreationResult;
+import org.thingsboard.server.common.data.sync.vc.VersionLoadResult;
+import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo;
 import org.thingsboard.server.common.data.sync.vc.request.create.ComplexVersionCreateRequest;
 import org.thingsboard.server.common.data.sync.vc.request.create.SingleEntityVersionCreateRequest;
 import org.thingsboard.server.common.data.sync.vc.request.create.SyncStrategy;
 import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateConfig;
 import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest;
+import org.thingsboard.server.common.data.sync.vc.request.load.EntityTypeVersionLoadConfig;
 import org.thingsboard.server.common.data.sync.vc.request.load.EntityTypeVersionLoadRequest;
 import org.thingsboard.server.common.data.sync.vc.request.load.SingleEntityVersionLoadRequest;
 import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadConfig;
 import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadRequest;
-import org.thingsboard.server.common.data.sync.ThrowingRunnable;
+import org.thingsboard.server.dao.DaoUtil;
+import org.thingsboard.server.dao.settings.AdminSettingsService;
+import org.thingsboard.server.queue.util.TbCoreComponent;
+import org.thingsboard.server.service.security.model.SecurityUser;
+import org.thingsboard.server.service.security.permission.Operation;
+import org.thingsboard.server.service.sync.ie.EntitiesExportImportService;
+import org.thingsboard.server.service.sync.ie.exporting.ExportableEntitiesService;
 
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -76,7 +78,6 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
     private final EntitiesExportImportService exportImportService;
     private final ExportableEntitiesService exportableEntitiesService;
     private final AdminSettingsService adminSettingsService;
-    private final EntityService entityService;
     private final TransactionTemplate transactionTemplate;
 
     @Override

application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java

@@ -135,7 +135,13 @@ public class LocalGitVersionControlService implements GitVersionControlService {
             if (old != null) {
                 gitRepositoryService.abort(old);
             }
-            gitRepositoryService.prepareCommit(pendingCommit);
+            try {
+                gitRepositoryService.prepareCommit(pendingCommit);
+            } catch (Exception e) {
+                pendingCommitMap.remove(tenantId);
+                gitRepositoryService.cleanUp(pendingCommit);
+                throw e;
+            }
             return pendingCommit;
         } finally {
             lock.unlock();
@@ -171,7 +177,9 @@ public class LocalGitVersionControlService implements GitVersionControlService {
 
     @Override
     public VersionCreationResult push(PendingCommit commit) {
-        return executeInsideLock(commit, gitRepositoryService::push);
+        VersionCreationResult result = executeInsideLock(commit, gitRepositoryService::push);
+        pendingCommitMap.remove(commit.getTenantId());
+        return result;
     }
 
     @Override
@@ -256,6 +264,10 @@ public class LocalGitVersionControlService implements GitVersionControlService {
         try {
             checkCommit(commit);
             r.accept(commit);
+        } catch (Exception e) {
+            pendingCommitMap.remove(commit.getTenantId());
+            gitRepositoryService.cleanUp(commit);
+            throw e;
         } finally {
             lock.unlock();
         }
@@ -267,6 +279,10 @@ public class LocalGitVersionControlService implements GitVersionControlService {
         try {
             checkCommit(commit);
             return c.apply(commit);
+        } catch (Exception e) {
+            pendingCommitMap.remove(commit.getTenantId());
+            gitRepositoryService.cleanUp(commit);
+            throw e;
         } finally {
             lock.unlock();
         }

application/src/main/resources/thingsboard.yml

@@ -1119,6 +1119,7 @@ vc:
   git:
     service: "${JS_VC_GIT_SERVICE:local}" # local/remote
     repos-poll-interval: "${TB_VC_GIT_REPOS_POLL_INTERVAL_SEC:60}"
+    repositories-folder: "${TB_VC_GIT_REPOSITORIES_FOLDER:${java.io.tmpdir}/repositories}"
 
 management:
   endpoints:

common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntitiesVersionControlSettings.java

@@ -16,6 +16,7 @@
 package org.thingsboard.server.common.data.sync.vc;
 
 import lombok.Data;
+
 @Data
 public class EntitiesVersionControlSettings {
     private String repositoryUri;

common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java

@@ -20,8 +20,6 @@ import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.eclipse.jgit.api.errors.GitAPIException;
-import org.eclipse.jgit.api.errors.JGitInternalException;
-import org.eclipse.jgit.api.errors.RefAlreadyExistsException;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
 import org.springframework.stereotype.Service;
@@ -55,7 +53,7 @@ import java.util.stream.Collectors;
 @Service
 public class DefaultGitRepositoryService implements GitRepositoryService {
 
-    @Value("${vc.git.repos-poll-interval:${java.io.tmpdir}/repositories}")
+    @Value("${vc.git.repositories-folder}")
     private String repositoriesFolder;
 
     @Value("${vc.git.repos-poll-interval:60}")
@@ -92,22 +90,12 @@ public class DefaultGitRepositoryService implements GitRepositoryService {
         String branch = commit.getRequest().getBranch();
         try {
             repository.fetch();
-            if (repository.listBranches().contains(branch)) {
+
-                repository.checkout("origin/" + branch, false);
+            repository.createAndCheckoutOrphanBranch(commit.getWorkingBranch());
-                try {
+            repository.resetAndClean();
-                    repository.checkout(branch, true);
+
-                } catch (RefAlreadyExistsException e) {
+            if (repository.listRemoteBranches().contains(branch)) {
-                    repository.checkout(branch, false);
-                }
                 repository.merge(branch);
-            } else { // TODO [viacheslav]: rollback orphan branch on failure
-                try {
-                    repository.createAndCheckoutOrphanBranch(branch); // FIXME [viacheslav]: Checkout returned unexpected result NO_CHANGE for master branch
-                } catch (JGitInternalException e) {
-                    if (!e.getMessage().contains("NO_CHANGE")) {
-                        throw e;
-                    }
-                }
             }
         } catch (IOException | GitAPIException gitAPIException) {
             //TODO: analyze and return meaningful exceptions that we can show to the client;
@@ -140,32 +128,49 @@ public class DefaultGitRepositoryService implements GitRepositoryService {
             result.setRemoved(status.getRemoved().size());
 
             GitRepository.Commit gitCommit = repository.commit(commit.getRequest().getVersionName());
-            repository.push();
+            repository.push(commit.getWorkingBranch(), commit.getRequest().getBranch());
 
             result.setVersion(toVersion(gitCommit));
             return result;
         } catch (GitAPIException gitAPIException) {
             //TODO: analyze and return meaningful exceptions that we can show to the client;
             throw new RuntimeException(gitAPIException);
+        } finally {
+            cleanUp(commit);
         }
     }
 
+    @SneakyThrows
+    @Override
+    public void cleanUp(PendingCommit commit) {
+        GitRepository repository = checkRepository(commit.getTenantId());
+        try {
+            repository.createAndCheckoutOrphanBranch(EntityId.NULL_UUID.toString());
+        } catch (Exception e) {
+            if (!e.getMessage().contains("NO_CHANGE")) {
+                throw e;
+            }
+        }
+        repository.resetAndClean();
+        repository.deleteLocalBranchIfExists(commit.getWorkingBranch());
+    }
+
     @Override
     public void abort(PendingCommit commit) {
-        //TODO: implement;
+        cleanUp(commit);
     }
 
     @Override
     public String getFileContentAtCommit(TenantId tenantId, String relativePath, String versionId) throws IOException {
-       GitRepository repository = checkRepository(tenantId);
+        GitRepository repository = checkRepository(tenantId);
-       return repository.getFileContentAtCommit(relativePath, versionId);
+        return repository.getFileContentAtCommit(relativePath, versionId);
     }
 
     @Override
     public List<String> listBranches(TenantId tenantId) {
         GitRepository repository = checkRepository(tenantId);
         try {
-            return repository.listBranches();
+            return repository.listRemoteBranches();
         } catch (GitAPIException gitAPIException) {
             //TODO: analyze and return meaningful exceptions that we can show to the client;
             throw new RuntimeException(gitAPIException);

common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java

@@ -18,8 +18,16 @@ package org.thingsboard.server.service.sync.vc;
 import com.google.common.collect.Streams;
 import lombok.Data;
 import lombok.Getter;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.sshd.common.util.security.SecurityUtils;
-import org.eclipse.jgit.api.*;
+import org.eclipse.jgit.api.CloneCommand;
+import org.eclipse.jgit.api.Git;
+import org.eclipse.jgit.api.GitCommand;
+import org.eclipse.jgit.api.ListBranchCommand;
+import org.eclipse.jgit.api.LogCommand;
+import org.eclipse.jgit.api.LsRemoteCommand;
+import org.eclipse.jgit.api.ResetCommand;
+import org.eclipse.jgit.api.TransportCommand;
 import org.eclipse.jgit.api.errors.GitAPIException;
 import org.eclipse.jgit.lib.Constants;
 import org.eclipse.jgit.lib.ObjectId;
@@ -28,6 +36,7 @@ import org.eclipse.jgit.lib.ObjectReader;
 import org.eclipse.jgit.revwalk.RevCommit;
 import org.eclipse.jgit.revwalk.filter.RevFilter;
 import org.eclipse.jgit.transport.CredentialsProvider;
+import org.eclipse.jgit.transport.RefSpec;
 import org.eclipse.jgit.transport.SshTransport;
 import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider;
 import org.eclipse.jgit.transport.sshd.JGitKeyCache;
@@ -36,7 +45,6 @@ import org.eclipse.jgit.transport.sshd.SshdSessionFactory;
 import org.eclipse.jgit.transport.sshd.SshdSessionFactoryBuilder;
 import org.eclipse.jgit.treewalk.TreeWalk;
 import org.eclipse.jgit.treewalk.filter.PathFilter;
-import org.thingsboard.server.common.data.StringUtils;
 import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings;
 import org.thingsboard.server.common.data.sync.vc.VersionControlAuthMethod;
 
@@ -116,15 +124,18 @@ public class GitRepository {
                 .setRemoveDeletedRefs(true));
     }
 
-    public void checkout(String branch, boolean createBranch) throws GitAPIException {
+    public void deleteLocalBranchIfExists(String branch) throws GitAPIException {
-        execute(git.checkout()
+        execute(git.branchDelete()
-                .setCreateBranch(createBranch)
+                .setBranchNames(branch)
-                .setName(branch));
+                .setForce(true));
     }
 
-    public void reset() throws GitAPIException {
+    public void resetAndClean() throws GitAPIException {
         execute(git.reset()
                 .setMode(ResetCommand.ResetType.HARD));
+        execute(git.clean()
+                .setForce(true)
+                .setCleanDirectories(true));
     }
 
     public void merge(String branch) throws IOException, GitAPIException {
@@ -137,9 +148,9 @@ public class GitRepository {
     }
 
 
-    public List<String> listBranches() throws GitAPIException {
+    public List<String> listRemoteBranches() throws GitAPIException {
         return execute(git.branchList()
-                .setListMode(ListBranchCommand.ListMode.ALL)).stream()
+                .setListMode(ListBranchCommand.ListMode.REMOTE)).stream()
                 .filter(ref -> !ref.getName().equals(Constants.HEAD))
                 .map(ref -> org.eclipse.jgit.lib.Repository.shortenRefName(ref.getName()))
                 .map(name -> StringUtils.removeStart(name, "origin/"))
@@ -209,16 +220,9 @@ public class GitRepository {
                 .setOrphan(true)
                 .setForced(true)
                 .setName(name));
-//        Set<String> uncommittedChanges = git.status().call().getUncommittedChanges();
-//        if (!uncommittedChanges.isEmpty()) {
-//            RmCommand rm = git.rm();
-//            uncommittedChanges.forEach(rm::addFilepattern);
-//            execute(rm);
-//        }
-//        execute(git.clean());
     }
 
-    public void add(String filesPattern) throws GitAPIException { // FIXME [viacheslav]
+    public void add(String filesPattern) throws GitAPIException {
         execute(git.add().setUpdate(true).addFilepattern(filesPattern));
         execute(git.add().addFilepattern(filesPattern));
     }
@@ -230,13 +234,14 @@ public class GitRepository {
 
     public Commit commit(String message) throws GitAPIException {
         RevCommit revCommit = execute(git.commit()
-                .setMessage(message)); // TODO [viacheslav]: set configurable author for commit
+                .setMessage(message));
         return toCommit(revCommit);
     }
 
 
-    public void push() throws GitAPIException {
+    public void push(String localBranch, String remoteBranch) throws GitAPIException {
-        execute(git.push());
+        execute(git.push()
+                .setRefSpecs(new RefSpec(localBranch + ":" + remoteBranch)));
     }
 
 

common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java

@@ -44,6 +44,8 @@ public interface GitRepositoryService {
 
     VersionCreationResult push(PendingCommit commit);
 
+    void cleanUp(PendingCommit commit);
+
     void abort(PendingCommit commit);
 
     List<String> listBranches(TenantId tenantId);

common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/PendingCommit.java

@@ -27,10 +27,12 @@ public class PendingCommit {
     private final UUID txId;
     private final TenantId tenantId;
     private final VersionCreateRequest request;
+    private final String workingBranch;
 
     public PendingCommit(TenantId tenantId, VersionCreateRequest request) {
         this.txId = UUID.randomUUID();
         this.tenantId = tenantId;
         this.request = request;
+        this.workingBranch = txId.toString();
     }
 }