Merge pull request #6585 from ViacheslavKlimov/entities-vc-refactoring

Git api usage improvements
2022-05-23 13:25:15 +03:00 · 2022-05-23 13:25:15 +03:00 · 490a0c2ae3
commit 490a0c2ae3
parent 421624f3d4 2a89fc67ae
8 changed files with 93 additions and 60 deletions
--- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java
+++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java
@ -24,37 +24,39 @@ import org.thingsboard.common.util.JacksonUtil;
 import org.thingsboard.server.common.data.AdminSettings;
 import org.thingsboard.server.common.data.EntityType;
 import org.thingsboard.server.common.data.ExportableEntity;
-import org.thingsboard.server.common.data.StringUtils;
 import org.thingsboard.server.common.data.exception.ThingsboardErrorCode;
 import org.thingsboard.server.common.data.exception.ThingsboardException;
 import org.thingsboard.server.common.data.id.EntityId;
 import org.thingsboard.server.common.data.id.EntityIdFactory;
 import org.thingsboard.server.common.data.id.TenantId;
-import org.thingsboard.server.common.data.sync.vc.*;
-import org.thingsboard.server.common.data.sync.vc.request.load.EntityTypeVersionLoadConfig;
-import org.thingsboard.server.dao.DaoUtil;
-import org.thingsboard.server.dao.entity.EntityService;
-import org.thingsboard.server.dao.settings.AdminSettingsService;
-import org.thingsboard.server.queue.util.TbCoreComponent;
-import org.thingsboard.server.service.security.model.SecurityUser;
-import org.thingsboard.server.service.security.permission.Operation;
-
-import org.thingsboard.server.service.sync.ie.EntitiesExportImportService;
-import org.thingsboard.server.service.sync.ie.exporting.ExportableEntitiesService;
+import org.thingsboard.server.common.data.sync.ThrowingRunnable;
 import org.thingsboard.server.common.data.sync.ie.EntityExportData;
 import org.thingsboard.server.common.data.sync.ie.EntityExportSettings;
 import org.thingsboard.server.common.data.sync.ie.EntityImportResult;
 import org.thingsboard.server.common.data.sync.ie.EntityImportSettings;
+import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings;
+import org.thingsboard.server.common.data.sync.vc.EntityVersion;
+import org.thingsboard.server.common.data.sync.vc.VersionControlAuthMethod;
+import org.thingsboard.server.common.data.sync.vc.VersionCreationResult;
+import org.thingsboard.server.common.data.sync.vc.VersionLoadResult;
+import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo;
 import org.thingsboard.server.common.data.sync.vc.request.create.ComplexVersionCreateRequest;
 import org.thingsboard.server.common.data.sync.vc.request.create.SingleEntityVersionCreateRequest;
 import org.thingsboard.server.common.data.sync.vc.request.create.SyncStrategy;
 import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateConfig;
 import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest;
+import org.thingsboard.server.common.data.sync.vc.request.load.EntityTypeVersionLoadConfig;
 import org.thingsboard.server.common.data.sync.vc.request.load.EntityTypeVersionLoadRequest;
 import org.thingsboard.server.common.data.sync.vc.request.load.SingleEntityVersionLoadRequest;
 import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadConfig;
 import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadRequest;
-import org.thingsboard.server.common.data.sync.ThrowingRunnable;
+import org.thingsboard.server.dao.DaoUtil;
+import org.thingsboard.server.dao.settings.AdminSettingsService;
+import org.thingsboard.server.queue.util.TbCoreComponent;
+import org.thingsboard.server.service.security.model.SecurityUser;
+import org.thingsboard.server.service.security.permission.Operation;
+import org.thingsboard.server.service.sync.ie.EntitiesExportImportService;
+import org.thingsboard.server.service.sync.ie.exporting.ExportableEntitiesService;

 import java.util.ArrayList;
 import java.util.HashMap;
@ -76,7 +78,6 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
    private final EntitiesExportImportService exportImportService;
    private final ExportableEntitiesService exportableEntitiesService;
    private final AdminSettingsService adminSettingsService;
-    private final EntityService entityService;
    private final TransactionTemplate transactionTemplate;

    @Override
--- a/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java
+++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java
@ -135,7 +135,13 @@ public class LocalGitVersionControlService implements GitVersionControlService {
            if (old != null) {
                gitRepositoryService.abort(old);
            }
+            try {
                gitRepositoryService.prepareCommit(pendingCommit);
+            } catch (Exception e) {
+                pendingCommitMap.remove(tenantId);
+                gitRepositoryService.cleanUp(pendingCommit);
+                throw e;
+            }
            return pendingCommit;
        } finally {
            lock.unlock();
@ -171,7 +177,9 @@ public class LocalGitVersionControlService implements GitVersionControlService {

    @Override
    public VersionCreationResult push(PendingCommit commit) {
-        return executeInsideLock(commit, gitRepositoryService::push);
+        VersionCreationResult result = executeInsideLock(commit, gitRepositoryService::push);
+        pendingCommitMap.remove(commit.getTenantId());
+        return result;
    }

    @Override
@ -256,6 +264,10 @@ public class LocalGitVersionControlService implements GitVersionControlService {
        try {
            checkCommit(commit);
            r.accept(commit);
+        } catch (Exception e) {
+            pendingCommitMap.remove(commit.getTenantId());
+            gitRepositoryService.cleanUp(commit);
+            throw e;
        } finally {
            lock.unlock();
        }
@ -267,6 +279,10 @@ public class LocalGitVersionControlService implements GitVersionControlService {
        try {
            checkCommit(commit);
            return c.apply(commit);
+        } catch (Exception e) {
+            pendingCommitMap.remove(commit.getTenantId());
+            gitRepositoryService.cleanUp(commit);
+            throw e;
        } finally {
            lock.unlock();
        }
--- a/application/src/main/resources/thingsboard.yml
+++ b/application/src/main/resources/thingsboard.yml
@ -1119,6 +1119,7 @@ vc:
  git:
    service: "${JS_VC_GIT_SERVICE:local}" # local/remote
    repos-poll-interval: "${TB_VC_GIT_REPOS_POLL_INTERVAL_SEC:60}"
+    repositories-folder: "${TB_VC_GIT_REPOSITORIES_FOLDER:${java.io.tmpdir}/repositories}"

 management:
  endpoints:
--- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntitiesVersionControlSettings.java
+++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntitiesVersionControlSettings.java
@ -16,6 +16,7 @@
 package org.thingsboard.server.common.data.sync.vc;

 import lombok.Data;
+
@Data
 public class EntitiesVersionControlSettings {
    private String repositoryUri;
--- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java
+++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java
@ -20,8 +20,6 @@ import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.eclipse.jgit.api.errors.GitAPIException;
-import org.eclipse.jgit.api.errors.JGitInternalException;
-import org.eclipse.jgit.api.errors.RefAlreadyExistsException;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
 import org.springframework.stereotype.Service;
@ -55,7 +53,7 @@ import java.util.stream.Collectors;
@Service
 public class DefaultGitRepositoryService implements GitRepositoryService {

-    @Value("${vc.git.repos-poll-interval:${java.io.tmpdir}/repositories}")
+    @Value("${vc.git.repositories-folder}")
    private String repositoriesFolder;

    @Value("${vc.git.repos-poll-interval:60}")
@ -92,22 +90,12 @@ public class DefaultGitRepositoryService implements GitRepositoryService {
        String branch = commit.getRequest().getBranch();
        try {
            repository.fetch();
-            if (repository.listBranches().contains(branch)) {
-                repository.checkout("origin/" + branch, false);
-                try {
-                    repository.checkout(branch, true);
-                } catch (RefAlreadyExistsException e) {
-                    repository.checkout(branch, false);
-                }
+
+            repository.createAndCheckoutOrphanBranch(commit.getWorkingBranch());
+            repository.resetAndClean();
+
+            if (repository.listRemoteBranches().contains(branch)) {
                repository.merge(branch);
-            } else { // TODO [viacheslav]: rollback orphan branch on failure
-                try {
-                    repository.createAndCheckoutOrphanBranch(branch); // FIXME [viacheslav]: Checkout returned unexpected result NO_CHANGE for master branch
-                } catch (JGitInternalException e) {
-                    if (!e.getMessage().contains("NO_CHANGE")) {
-                        throw e;
-                    }
-                }
            }
        } catch (IOException | GitAPIException gitAPIException) {
            //TODO: analyze and return meaningful exceptions that we can show to the client;
@ -140,19 +128,36 @@ public class DefaultGitRepositoryService implements GitRepositoryService {
            result.setRemoved(status.getRemoved().size());

            GitRepository.Commit gitCommit = repository.commit(commit.getRequest().getVersionName());
-            repository.push();
+            repository.push(commit.getWorkingBranch(), commit.getRequest().getBranch());

            result.setVersion(toVersion(gitCommit));
            return result;
        } catch (GitAPIException gitAPIException) {
            //TODO: analyze and return meaningful exceptions that we can show to the client;
            throw new RuntimeException(gitAPIException);
+        } finally {
+            cleanUp(commit);
        }
    }

+    @SneakyThrows
+    @Override
+    public void cleanUp(PendingCommit commit) {
+        GitRepository repository = checkRepository(commit.getTenantId());
+        try {
+            repository.createAndCheckoutOrphanBranch(EntityId.NULL_UUID.toString());
+        } catch (Exception e) {
+            if (!e.getMessage().contains("NO_CHANGE")) {
+                throw e;
+            }
+        }
+        repository.resetAndClean();
+        repository.deleteLocalBranchIfExists(commit.getWorkingBranch());
+    }
+
    @Override
    public void abort(PendingCommit commit) {
-        //TODO: implement;
+        cleanUp(commit);
    }

    @Override
@ -165,7 +170,7 @@ public class DefaultGitRepositoryService implements GitRepositoryService {
    public List<String> listBranches(TenantId tenantId) {
        GitRepository repository = checkRepository(tenantId);
        try {
-            return repository.listBranches();
+            return repository.listRemoteBranches();
        } catch (GitAPIException gitAPIException) {
            //TODO: analyze and return meaningful exceptions that we can show to the client;
            throw new RuntimeException(gitAPIException);
--- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java
+++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java
@ -18,8 +18,16 @@ package org.thingsboard.server.service.sync.vc;
 import com.google.common.collect.Streams;
 import lombok.Data;
 import lombok.Getter;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.sshd.common.util.security.SecurityUtils;
-import org.eclipse.jgit.api.*;
+import org.eclipse.jgit.api.CloneCommand;
+import org.eclipse.jgit.api.Git;
+import org.eclipse.jgit.api.GitCommand;
+import org.eclipse.jgit.api.ListBranchCommand;
+import org.eclipse.jgit.api.LogCommand;
+import org.eclipse.jgit.api.LsRemoteCommand;
+import org.eclipse.jgit.api.ResetCommand;
+import org.eclipse.jgit.api.TransportCommand;
 import org.eclipse.jgit.api.errors.GitAPIException;
 import org.eclipse.jgit.lib.Constants;
 import org.eclipse.jgit.lib.ObjectId;
@ -28,6 +36,7 @@ import org.eclipse.jgit.lib.ObjectReader;
 import org.eclipse.jgit.revwalk.RevCommit;
 import org.eclipse.jgit.revwalk.filter.RevFilter;
 import org.eclipse.jgit.transport.CredentialsProvider;
+import org.eclipse.jgit.transport.RefSpec;
 import org.eclipse.jgit.transport.SshTransport;
 import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider;
 import org.eclipse.jgit.transport.sshd.JGitKeyCache;
@ -36,7 +45,6 @@ import org.eclipse.jgit.transport.sshd.SshdSessionFactory;
 import org.eclipse.jgit.transport.sshd.SshdSessionFactoryBuilder;
 import org.eclipse.jgit.treewalk.TreeWalk;
 import org.eclipse.jgit.treewalk.filter.PathFilter;
-import org.thingsboard.server.common.data.StringUtils;
 import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings;
 import org.thingsboard.server.common.data.sync.vc.VersionControlAuthMethod;

@ -116,15 +124,18 @@ public class GitRepository {
                .setRemoveDeletedRefs(true));
    }

-    public void checkout(String branch, boolean createBranch) throws GitAPIException {
-        execute(git.checkout()
-                .setCreateBranch(createBranch)
-                .setName(branch));
+    public void deleteLocalBranchIfExists(String branch) throws GitAPIException {
+        execute(git.branchDelete()
+                .setBranchNames(branch)
+                .setForce(true));
    }

-    public void reset() throws GitAPIException {
+    public void resetAndClean() throws GitAPIException {
        execute(git.reset()
                .setMode(ResetCommand.ResetType.HARD));
+        execute(git.clean()
+                .setForce(true)
+                .setCleanDirectories(true));
    }

    public void merge(String branch) throws IOException, GitAPIException {
@ -137,9 +148,9 @@ public class GitRepository {
    }


-    public List<String> listBranches() throws GitAPIException {
+    public List<String> listRemoteBranches() throws GitAPIException {
        return execute(git.branchList()
-                .setListMode(ListBranchCommand.ListMode.ALL)).stream()
+                .setListMode(ListBranchCommand.ListMode.REMOTE)).stream()
                .filter(ref -> !ref.getName().equals(Constants.HEAD))
                .map(ref -> org.eclipse.jgit.lib.Repository.shortenRefName(ref.getName()))
                .map(name -> StringUtils.removeStart(name, "origin/"))
@ -209,16 +220,9 @@ public class GitRepository {
                .setOrphan(true)
                .setForced(true)
                .setName(name));
-//        Set<String> uncommittedChanges = git.status().call().getUncommittedChanges();
-//        if (!uncommittedChanges.isEmpty()) {
-//            RmCommand rm = git.rm();
-//            uncommittedChanges.forEach(rm::addFilepattern);
-//            execute(rm);
-//        }
-//        execute(git.clean());
    }

-    public void add(String filesPattern) throws GitAPIException { // FIXME [viacheslav]
+    public void add(String filesPattern) throws GitAPIException {
        execute(git.add().setUpdate(true).addFilepattern(filesPattern));
        execute(git.add().addFilepattern(filesPattern));
    }
@ -230,13 +234,14 @@ public class GitRepository {

    public Commit commit(String message) throws GitAPIException {
        RevCommit revCommit = execute(git.commit()
-                .setMessage(message)); // TODO [viacheslav]: set configurable author for commit
+                .setMessage(message));
        return toCommit(revCommit);
    }


-    public void push() throws GitAPIException {
-        execute(git.push());
+    public void push(String localBranch, String remoteBranch) throws GitAPIException {
+        execute(git.push()
+                .setRefSpecs(new RefSpec(localBranch + ":" + remoteBranch)));
    }


--- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java
+++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java
@ -44,6 +44,8 @@ public interface GitRepositoryService {

    VersionCreationResult push(PendingCommit commit);

+    void cleanUp(PendingCommit commit);
+
    void abort(PendingCommit commit);

    List<String> listBranches(TenantId tenantId);
--- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/PendingCommit.java
+++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/PendingCommit.java
@ -27,10 +27,12 @@ public class PendingCommit {
    private final UUID txId;
    private final TenantId tenantId;
    private final VersionCreateRequest request;
+    private final String workingBranch;

    public PendingCommit(TenantId tenantId, VersionCreateRequest request) {
        this.txId = UUID.randomUUID();
        this.tenantId = tenantId;
        this.request = request;
+        this.workingBranch = txId.toString();
    }
 }