dendi-wang/thingsboard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #6585 from ViacheslavKlimov/entities-vc-refactoring

Browse Source 
Git api usage improvements
This commit is contained in:
Igor Kulikov 2022-05-23 13:25:15 +03:00 committed by GitHub
commit 490a0c2ae3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 93 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java
View File

@ -24,37 +24,39 @@ import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.common.data.AdminSettings;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.ExportableEntity;
import org.thingsboard.server.common.data.StringUtils;
import org.thingsboard.server.common.data.exception.ThingsboardErrorCode;
import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.EntityIdFactory;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.sync.vc.*;
import org.thingsboard.server.common.data.sync.vc.request.load.EntityTypeVersionLoadConfig;
import org.thingsboard.server.dao.DaoUtil;
import org.thingsboard.server.dao.entity.EntityService;
import org.thingsboard.server.dao.settings.AdminSettingsService;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.security.model.SecurityUser;
import org.thingsboard.server.service.security.permission.Operation;
import org.thingsboard.server.service.sync.ie.EntitiesExportImportService;
import org.thingsboard.server.service.sync.ie.exporting.ExportableEntitiesService;
import org.thingsboard.server.common.data.sync.ThrowingRunnable;
import org.thingsboard.server.common.data.sync.ie.EntityExportData;
import org.thingsboard.server.common.data.sync.ie.EntityExportSettings;
import org.thingsboard.server.common.data.sync.ie.EntityImportResult;
import org.thingsboard.server.common.data.sync.ie.EntityImportSettings;
import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings;
import org.thingsboard.server.common.data.sync.vc.EntityVersion;
import org.thingsboard.server.common.data.sync.vc.VersionControlAuthMethod;
import org.thingsboard.server.common.data.sync.vc.VersionCreationResult;
import org.thingsboard.server.common.data.sync.vc.VersionLoadResult;
import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo;
import org.thingsboard.server.common.data.sync.vc.request.create.ComplexVersionCreateRequest;
import org.thingsboard.server.common.data.sync.vc.request.create.SingleEntityVersionCreateRequest;
import org.thingsboard.server.common.data.sync.vc.request.create.SyncStrategy;
import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateConfig;
import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest;
import org.thingsboard.server.common.data.sync.vc.request.load.EntityTypeVersionLoadConfig;
import org.thingsboard.server.common.data.sync.vc.request.load.EntityTypeVersionLoadRequest;
import org.thingsboard.server.common.data.sync.vc.request.load.SingleEntityVersionLoadRequest;
import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadConfig;
import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadRequest;
import org.thingsboard.server.common.data.sync.ThrowingRunnable;
import org.thingsboard.server.dao.DaoUtil;
import org.thingsboard.server.dao.settings.AdminSettingsService;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.security.model.SecurityUser;
import org.thingsboard.server.service.security.permission.Operation;
import org.thingsboard.server.service.sync.ie.EntitiesExportImportService;
import org.thingsboard.server.service.sync.ie.exporting.ExportableEntitiesService;
import java.util.ArrayList;
import java.util.HashMap;
@ -76,7 +78,6 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
private final EntitiesExportImportService exportImportService;
private final ExportableEntitiesService exportableEntitiesService;
private final AdminSettingsService adminSettingsService;
private final EntityService entityService;
private final TransactionTemplate transactionTemplate;
@Override

20
application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java
View File

@ -135,7 +135,13 @@ public class LocalGitVersionControlService implements GitVersionControlService {
if (old != null) {
gitRepositoryService.abort(old);
}
gitRepositoryService.prepareCommit(pendingCommit);
try {
gitRepositoryService.prepareCommit(pendingCommit);
} catch (Exception e) {
pendingCommitMap.remove(tenantId);
gitRepositoryService.cleanUp(pendingCommit);
throw e;
}
return pendingCommit;
} finally {
lock.unlock();
@ -171,7 +177,9 @@ public class LocalGitVersionControlService implements GitVersionControlService {
@Override
public VersionCreationResult push(PendingCommit commit) {
return executeInsideLock(commit, gitRepositoryService::push);
VersionCreationResult result = executeInsideLock(commit, gitRepositoryService::push);
pendingCommitMap.remove(commit.getTenantId());
return result;
}
@Override
@ -256,6 +264,10 @@ public class LocalGitVersionControlService implements GitVersionControlService {
try {
checkCommit(commit);
r.accept(commit);
} catch (Exception e) {
pendingCommitMap.remove(commit.getTenantId());
gitRepositoryService.cleanUp(commit);
throw e;
} finally {
lock.unlock();
}
@ -267,6 +279,10 @@ public class LocalGitVersionControlService implements GitVersionControlService {
try {
checkCommit(commit);
return c.apply(commit);
} catch (Exception e) {
pendingCommitMap.remove(commit.getTenantId());
gitRepositoryService.cleanUp(commit);
throw e;
} finally {
lock.unlock();
}

1
application/src/main/resources/thingsboard.yml
View File

@ -1119,6 +1119,7 @@ vc:
git:
service: "${JS_VC_GIT_SERVICE:local}" # local/remote
repos-poll-interval: "${TB_VC_GIT_REPOS_POLL_INTERVAL_SEC:60}"
repositories-folder: "${TB_VC_GIT_REPOSITORIES_FOLDER:${java.io.tmpdir}/repositories}"
management:
endpoints:

3
common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntitiesVersionControlSettings.java
View File

@ -16,6 +16,7 @@
package org.thingsboard.server.common.data.sync.vc;
import lombok.Data;
@Data
public class EntitiesVersionControlSettings {
private String repositoryUri;
@ -26,4 +27,4 @@ public class EntitiesVersionControlSettings {
private String privateKey;
private String privateKeyPassword;
private String defaultBranch;
}
}

51
common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java
View File

@ -20,8 +20,6 @@ import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.eclipse.jgit.api.errors.JGitInternalException;
import org.eclipse.jgit.api.errors.RefAlreadyExistsException;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Service;
@ -55,7 +53,7 @@ import java.util.stream.Collectors;
@Service
public class DefaultGitRepositoryService implements GitRepositoryService {
@Value("${vc.git.repos-poll-interval:${java.io.tmpdir}/repositories}")
@Value("${vc.git.repositories-folder}")
private String repositoriesFolder;
@Value("${vc.git.repos-poll-interval:60}")
@ -92,22 +90,12 @@ public class DefaultGitRepositoryService implements GitRepositoryService {
String branch = commit.getRequest().getBranch();
try {
repository.fetch();
if (repository.listBranches().contains(branch)) {
repository.checkout("origin/" + branch, false);
try {
repository.checkout(branch, true);
} catch (RefAlreadyExistsException e) {
repository.checkout(branch, false);
}
repository.createAndCheckoutOrphanBranch(commit.getWorkingBranch());
repository.resetAndClean();
if (repository.listRemoteBranches().contains(branch)) {
repository.merge(branch);
} else { // TODO [viacheslav]: rollback orphan branch on failure
try {
repository.createAndCheckoutOrphanBranch(branch); // FIXME [viacheslav]: Checkout returned unexpected result NO_CHANGE for master branch
} catch (JGitInternalException e) {
if (!e.getMessage().contains("NO_CHANGE")) {
throw e;
}
}
}
} catch (IOException | GitAPIException gitAPIException) {
//TODO: analyze and return meaningful exceptions that we can show to the client;
@ -140,32 +128,49 @@ public class DefaultGitRepositoryService implements GitRepositoryService {
result.setRemoved(status.getRemoved().size());
GitRepository.Commit gitCommit = repository.commit(commit.getRequest().getVersionName());
repository.push();
repository.push(commit.getWorkingBranch(), commit.getRequest().getBranch());
result.setVersion(toVersion(gitCommit));
return result;
} catch (GitAPIException gitAPIException) {
//TODO: analyze and return meaningful exceptions that we can show to the client;
throw new RuntimeException(gitAPIException);
} finally {
cleanUp(commit);
}
}
@SneakyThrows
@Override
public void cleanUp(PendingCommit commit) {
GitRepository repository = checkRepository(commit.getTenantId());
try {
repository.createAndCheckoutOrphanBranch(EntityId.NULL_UUID.toString());
} catch (Exception e) {
if (!e.getMessage().contains("NO_CHANGE")) {
throw e;
}
}
repository.resetAndClean();
repository.deleteLocalBranchIfExists(commit.getWorkingBranch());
}
@Override
public void abort(PendingCommit commit) {
//TODO: implement;
cleanUp(commit);
}
@Override
public String getFileContentAtCommit(TenantId tenantId, String relativePath, String versionId) throws IOException {
GitRepository repository = checkRepository(tenantId);
return repository.getFileContentAtCommit(relativePath, versionId);
GitRepository repository = checkRepository(tenantId);
return repository.getFileContentAtCommit(relativePath, versionId);
}
@Override
public List<String> listBranches(TenantId tenantId) {
GitRepository repository = checkRepository(tenantId);
try {
return repository.listBranches();
return repository.listRemoteBranches();
} catch (GitAPIException gitAPIException) {
//TODO: analyze and return meaningful exceptions that we can show to the client;
throw new RuntimeException(gitAPIException);

45
common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java
View File

@ -18,8 +18,16 @@ package org.thingsboard.server.service.sync.vc;
import com.google.common.collect.Streams;
import lombok.Data;
import lombok.Getter;
import org.apache.commons.lang3.StringUtils;
import org.apache.sshd.common.util.security.SecurityUtils;
import org.eclipse.jgit.api.*;
import org.eclipse.jgit.api.CloneCommand;
import org.eclipse.jgit.api.Git;
import org.eclipse.jgit.api.GitCommand;
import org.eclipse.jgit.api.ListBranchCommand;
import org.eclipse.jgit.api.LogCommand;
import org.eclipse.jgit.api.LsRemoteCommand;
import org.eclipse.jgit.api.ResetCommand;
import org.eclipse.jgit.api.TransportCommand;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.eclipse.jgit.lib.Constants;
import org.eclipse.jgit.lib.ObjectId;
@ -28,6 +36,7 @@ import org.eclipse.jgit.lib.ObjectReader;
import org.eclipse.jgit.revwalk.RevCommit;
import org.eclipse.jgit.revwalk.filter.RevFilter;
import org.eclipse.jgit.transport.CredentialsProvider;
import org.eclipse.jgit.transport.RefSpec;
import org.eclipse.jgit.transport.SshTransport;
import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider;
import org.eclipse.jgit.transport.sshd.JGitKeyCache;
@ -36,7 +45,6 @@ import org.eclipse.jgit.transport.sshd.SshdSessionFactory;
import org.eclipse.jgit.transport.sshd.SshdSessionFactoryBuilder;
import org.eclipse.jgit.treewalk.TreeWalk;
import org.eclipse.jgit.treewalk.filter.PathFilter;
import org.thingsboard.server.common.data.StringUtils;
import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings;
import org.thingsboard.server.common.data.sync.vc.VersionControlAuthMethod;
@ -116,15 +124,18 @@ public class GitRepository {
.setRemoveDeletedRefs(true));
}
public void checkout(String branch, boolean createBranch) throws GitAPIException {
execute(git.checkout()
.setCreateBranch(createBranch)
.setName(branch));
public void deleteLocalBranchIfExists(String branch) throws GitAPIException {
execute(git.branchDelete()
.setBranchNames(branch)
.setForce(true));
}
public void reset() throws GitAPIException {
public void resetAndClean() throws GitAPIException {
execute(git.reset()
.setMode(ResetCommand.ResetType.HARD));
execute(git.clean()
.setForce(true)
.setCleanDirectories(true));
}
public void merge(String branch) throws IOException, GitAPIException {
@ -137,9 +148,9 @@ public class GitRepository {
}
public List<String> listBranches() throws GitAPIException {
public List<String> listRemoteBranches() throws GitAPIException {
return execute(git.branchList()
.setListMode(ListBranchCommand.ListMode.ALL)).stream()
.setListMode(ListBranchCommand.ListMode.REMOTE)).stream()
.filter(ref -> !ref.getName().equals(Constants.HEAD))
.map(ref -> org.eclipse.jgit.lib.Repository.shortenRefName(ref.getName()))
.map(name -> StringUtils.removeStart(name, "origin/"))
@ -209,16 +220,9 @@ public class GitRepository {
.setOrphan(true)
.setForced(true)
.setName(name));
// Set<String> uncommittedChanges = git.status().call().getUncommittedChanges();
// if (!uncommittedChanges.isEmpty()) {
// RmCommand rm = git.rm();
// uncommittedChanges.forEach(rm::addFilepattern);
// execute(rm);
// }
// execute(git.clean());
}
public void add(String filesPattern) throws GitAPIException { // FIXME [viacheslav]
public void add(String filesPattern) throws GitAPIException {
execute(git.add().setUpdate(true).addFilepattern(filesPattern));
execute(git.add().addFilepattern(filesPattern));
}
@ -230,13 +234,14 @@ public class GitRepository {
public Commit commit(String message) throws GitAPIException {
RevCommit revCommit = execute(git.commit()
.setMessage(message)); // TODO [viacheslav]: set configurable author for commit
.setMessage(message));
return toCommit(revCommit);
}
public void push() throws GitAPIException {
execute(git.push());
public void push(String localBranch, String remoteBranch) throws GitAPIException {
execute(git.push()
.setRefSpecs(new RefSpec(localBranch + ":" + remoteBranch)));
}

2
common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java
View File

@ -44,6 +44,8 @@ public interface GitRepositoryService {
VersionCreationResult push(PendingCommit commit);
void cleanUp(PendingCommit commit);
void abort(PendingCommit commit);
List<String> listBranches(TenantId tenantId);

2
common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/PendingCommit.java
View File

@ -27,10 +27,12 @@ public class PendingCommit {
private final UUID txId;
private final TenantId tenantId;
private final VersionCreateRequest request;
private final String workingBranch;
public PendingCommit(TenantId tenantId, VersionCreateRequest request) {
this.txId = UUID.randomUUID();
this.tenantId = tenantId;
this.request = request;
this.workingBranch = txId.toString();
}
}