dendi-wang/thingsboard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Version Control Service refactoring

Browse Source
This commit is contained in:
Andrii Shvaika 2022-05-18 14:46:54 +03:00
parent 7ec45fd5f1
commit c6ebf049a2
12 changed files with 694 additions and 192 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
application/pom.xml
View File

@ -345,10 +345,6 @@
<artifactId>Java-WebSocket</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.eclipse.jgit</groupId>
<artifactId>org.eclipse.jgit</artifactId>
</dependency>
</dependencies>
<build>

2
application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java
View File

@ -172,7 +172,7 @@ public class EntitiesVersionControlController extends BaseController {
@PathVariable EntityType entityType,
@PathVariable String versionId) throws ThingsboardException {
try {
return versionControlService.listEntitiesAtVersion(getTenantId(), entityType, branch, versionId);
return versionControlService.listEntitiesAtVersion(getTenantId(), branch, versionId, entityType);
} catch (Exception e) {
throw handleException(e);
}

209
application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java
View File

@ -75,7 +75,6 @@ import org.thingsboard.server.common.data.sync.vc.request.load.EntityTypeVersion
import org.thingsboard.server.common.data.sync.vc.request.load.SingleEntityVersionLoadRequest;
import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadConfig;
import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadRequest;
import org.thingsboard.server.utils.GitRepository;
import org.thingsboard.server.common.data.sync.ThrowingRunnable;
import java.io.File;
@ -105,85 +104,31 @@ import static org.thingsboard.server.dao.sql.query.EntityKeyMapping.CREATED_TIME
@Slf4j
public class DefaultEntitiesVersionControlService implements EntitiesVersionControlService {
private final GitVersionControlService gitService;
private final EntitiesExportImportService exportImportService;
private final ExportableEntitiesService exportableEntitiesService;
private final AttributesService attributesService;
private final EntityService entityService;
private final TenantDao tenantDao;
private final TransactionTemplate transactionTemplate;
// TODO [viacheslav]: concurrency
private final Map<TenantId, GitRepository> repositories = new ConcurrentHashMap<>();
@Value("${java.io.tmpdir}/repositories")
private String repositoriesFolder;
private static final String SETTINGS_KEY = "vc";
private final ObjectWriter jsonWriter = new ObjectMapper().writer(SerializationFeature.INDENT_OUTPUT);
@AfterStartUp
public void init() {
DaoUtil.processInBatches(tenantDao::findTenantsIds, 100, tenantId -> {
EntitiesVersionControlSettings settings = getSettings(tenantId);
if (settings != null) {
try {
initRepository(tenantId, settings);
} catch (Exception e) {
log.warn("Failed to init repository for tenant {}", tenantId, e);
}
}
});
Executors.newSingleThreadScheduledExecutor().scheduleWithFixedDelay(() -> {
repositories.forEach((tenantId, repository) -> {
try {
repository.fetch();
log.info("Fetching remote repository for tenant {}", tenantId);
} catch (Exception e) {
log.warn("Failed to fetch repository for tenant {}", tenantId, e);
}
});
}, 5, 5, TimeUnit.SECONDS);
}
public static final String SETTINGS_KEY = "vc";
@Override
public VersionCreationResult saveEntitiesVersion(SecurityUser user, VersionCreateRequest request) throws Exception {
GitRepository repository = checkRepository(user.getTenantId());
repository.fetch();
if (repository.listBranches().contains(request.getBranch())) {
repository.checkout("origin/" + request.getBranch(), false);
try {
repository.checkout(request.getBranch(), true);
} catch (RefAlreadyExistsException e) {
repository.checkout(request.getBranch(), false);
}
repository.merge(request.getBranch());
} else { // TODO [viacheslav]: rollback orphan branch on failure
try {
repository.createAndCheckoutOrphanBranch(request.getBranch()); // FIXME [viacheslav]: Checkout returned unexpected result NO_CHANGE for master branch
} catch (JGitInternalException e) {
if (!e.getMessage().contains("NO_CHANGE")) {
throw e;
}
}
}
var commit = gitService.prepareCommit(user.getTenantId(), request);
switch (request.getType()) {
case SINGLE_ENTITY: {
SingleEntityVersionCreateRequest versionCreateRequest = (SingleEntityVersionCreateRequest) request;
saveEntityData(user, repository, versionCreateRequest.getEntityId(), versionCreateRequest.getConfig());
saveEntityData(user, commit, versionCreateRequest.getEntityId(), versionCreateRequest.getConfig());
break;
}
case COMPLEX: {
ComplexVersionCreateRequest versionCreateRequest = (ComplexVersionCreateRequest) request;
versionCreateRequest.getEntityTypes().forEach((entityType, config) -> {
if (ObjectUtils.defaultIfNull(config.getSyncStrategy(), versionCreateRequest.getSyncStrategy()) == SyncStrategy.OVERWRITE) {
try {
FileUtils.deleteDirectory(Path.of(repository.getDirectory(), getRelativePath(entityType, null)).toFile());
} catch (IOException e) {
throw new RuntimeException(e);
}
gitService.deleteAll(commit, entityType);
}
if (config.isAllEntities()) {
@ -203,7 +148,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
}, 100, data -> {
EntityId entityId = data.getEntityId();
try {
saveEntityData(user, repository, entityId, config);
saveEntityData(user, commit, entityId, config);
} catch (Exception e) {
throw new RuntimeException(e);
}
@ -211,7 +156,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
} else {
for (UUID entityId : config.getEntityIds()) {
try {
saveEntityData(user, repository, EntityIdFactory.getByTypeAndUuid(entityType, entityId), config);
saveEntityData(user, commit, EntityIdFactory.getByTypeAndUuid(entityType, entityId), config);
} catch (Exception e) {
throw new RuntimeException(e);
}
@ -223,90 +168,50 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
}
}
repository.add(".");
VersionCreationResult result = new VersionCreationResult();
GitRepository.Status status = repository.status();
result.setAdded(status.getAdded().size());
result.setModified(status.getModified().size());
result.setRemoved(status.getRemoved().size());
GitRepository.Commit commit = repository.commit(request.getVersionName());
repository.push();
result.setVersion(toVersion(commit));
return result;
return gitService.push(commit);
}
private void saveEntityData(SecurityUser user, GitRepository repository, EntityId entityId, VersionCreateConfig config) throws Exception {
private void saveEntityData(SecurityUser user, PendingCommit commit, EntityId entityId, VersionCreateConfig config) throws Exception {
EntityExportData<ExportableEntity<EntityId>> entityData = exportImportService.exportEntity(user, entityId, EntityExportSettings.builder()
.exportRelations(config.isSaveRelations())
.build());
String entityDataJson = jsonWriter.writeValueAsString(entityData);
FileUtils.write(Path.of(repository.getDirectory(), getRelativePath(entityData.getEntityType(),
entityData.getEntity().getId().toString())).toFile(), entityDataJson, StandardCharsets.UTF_8);
gitService.addToCommit(commit, entityData);
}
@Override
public List<EntityVersion> listEntityVersions(TenantId tenantId, String branch, EntityId externalId) throws Exception {
return listVersions(tenantId, branch, getRelativePath(externalId.getEntityType(), externalId.getId().toString()));
return gitService.listVersions(tenantId, branch, externalId);
}
@Override
public List<EntityVersion> listEntityTypeVersions(TenantId tenantId, String branch, EntityType entityType) throws Exception {
return listVersions(tenantId, branch, getRelativePath(entityType, null));
return gitService.listVersions(tenantId, branch, entityType);
}
@Override
public List<EntityVersion> listVersions(TenantId tenantId, String branch) throws Exception {
return listVersions(tenantId, branch, null);
return gitService.listVersions(tenantId, branch);
}
private List<EntityVersion> listVersions(TenantId tenantId, String branch, String path) throws Exception {
GitRepository repository = checkRepository(tenantId);
return repository.listCommits(branch, path, Integer.MAX_VALUE).stream()
.map(this::toVersion)
.collect(Collectors.toList());
}
@Override
public List<VersionedEntityInfo> listEntitiesAtVersion(TenantId tenantId, EntityType entityType, String branch, String versionId) throws Exception {
return listEntitiesAtVersion(tenantId, branch, versionId, getRelativePath(entityType, null));
public List<VersionedEntityInfo> listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType) throws Exception {
return gitService.listEntitiesAtVersion(tenantId, branch, versionId, entityType);
}
@Override
public List<VersionedEntityInfo> listAllEntitiesAtVersion(TenantId tenantId, String branch, String versionId) throws Exception {
return listEntitiesAtVersion(tenantId, branch, versionId, null);
return gitService.listEntitiesAtVersion(tenantId, branch, versionId);
}
private List<VersionedEntityInfo> listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, String path) throws Exception {
GitRepository repository = checkRepository(tenantId);
checkVersion(tenantId, branch, versionId);
return repository.listFilesAtCommit(versionId, path).stream()
.map(filePath -> {
EntityId entityId = fromRelativePath(filePath);
VersionedEntityInfo info = new VersionedEntityInfo();
info.setExternalId(entityId);
return info;
})
.collect(Collectors.toList());
}
@Override
public List<VersionLoadResult> loadEntitiesVersion(SecurityUser user, VersionLoadRequest request) throws Exception {
GitRepository repository = checkRepository(user.getTenantId());
EntityVersion version = checkVersion(user.getTenantId(), request.getBranch(), request.getVersionId());
switch (request.getType()) {
case SINGLE_ENTITY: {
SingleEntityVersionLoadRequest versionLoadRequest = (SingleEntityVersionLoadRequest) request;
EntityImportResult<?> importResult = transactionTemplate.execute(status -> {
try {
EntityImportResult<?> result = loadEntity(user, repository, versionLoadRequest.getExternalEntityId(), version.getId(), versionLoadRequest.getConfig());
EntityImportResult<?> result = loadEntity(user, request, versionLoadRequest.getConfig(), versionLoadRequest.getExternalEntityId());
result.getSaveReferencesCallback().run();
return result;
} catch (Exception e) {
@ -340,11 +245,11 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
Set<EntityId> remoteEntities;
try {
remoteEntities = listEntitiesAtVersion(user.getTenantId(), request.getBranch(), request.getVersionId(), getRelativePath(entityType, null)).stream()
remoteEntities = listEntitiesAtVersion(user.getTenantId(), request.getBranch(), request.getVersionId(), entityType).stream()
.map(VersionedEntityInfo::getExternalId)
.collect(Collectors.toSet());
for (EntityId externalEntityId : remoteEntities) {
EntityImportResult<?> importResult = loadEntity(user, repository, externalEntityId, version.getId(), config);
EntityImportResult<?> importResult = loadEntity(user, request, config, externalEntityId);
if (importResult.getOldEntity() == null) created.incrementAndGet();
else updated.incrementAndGet();
@ -402,10 +307,8 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
}
}
private EntityImportResult<?> loadEntity(SecurityUser user, GitRepository repository, EntityId externalId, String versionId, VersionLoadConfig config) throws Exception {
String entityDataJson = repository.getFileContentAtCommit(getRelativePath(externalId.getEntityType(), externalId.getId().toString()), versionId);
EntityExportData entityData = JacksonUtil.fromString(entityDataJson, EntityExportData.class);
private EntityImportResult<?> loadEntity(SecurityUser user, VersionLoadRequest request, VersionLoadConfig config, EntityId entityId) throws Exception {
EntityExportData entityData = gitService.getEntity(user.getTenantId(), request.getVersionId(), entityId);
return exportImportService.importEntity(user, entityData, EntityImportSettings.builder()
.updateRelations(config.isLoadRelations())
.findExistingByName(config.isFindExistingEntityByName())
@ -415,43 +318,9 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
@Override
public List<String> listBranches(TenantId tenantId) throws Exception {
GitRepository repository = checkRepository(tenantId);
return repository.listBranches();
return gitService.listBranches(tenantId);
}
private EntityVersion checkVersion(TenantId tenantId, String branch, String versionId) throws Exception {
return listVersions(tenantId, branch, null).stream()
.filter(version -> version.getId().equals(versionId))
.findFirst().orElseThrow(() -> new IllegalArgumentException("Version not found"));
}
private GitRepository checkRepository(TenantId tenantId) {
return Optional.ofNullable(repositories.get(tenantId))
.orElseThrow(() -> new IllegalStateException("Repository is not initialized"));
}
private void initRepository(TenantId tenantId, EntitiesVersionControlSettings settings) throws Exception {
Path repositoryDirectory = Path.of(repositoriesFolder, tenantId.getId().toString());
GitRepository repository;
if (Files.exists(repositoryDirectory)) {
FileUtils.forceDelete(repositoryDirectory.toFile());
}
Files.createDirectories(repositoryDirectory);
repository = GitRepository.clone(settings.getRepositoryUri(), settings.getUsername(), settings.getPassword(), repositoryDirectory.toFile());
repositories.put(tenantId, repository);
}
private void clearRepository(TenantId tenantId) throws IOException {
GitRepository repository = repositories.get(tenantId);
if (repository != null) {
FileUtils.deleteDirectory(new File(repository.getDirectory()));
repositories.remove(tenantId);
}
}
@SneakyThrows
@Override
public void saveSettings(TenantId tenantId, EntitiesVersionControlSettings settings) {
@ -459,41 +328,11 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
new BaseAttributeKvEntry(System.currentTimeMillis(), new JsonDataEntry(SETTINGS_KEY, JacksonUtil.toString(settings)))
)).get();
initRepository(tenantId, settings);
gitService.initRepository(tenantId, settings);
}
@SneakyThrows
@Override
public EntitiesVersionControlSettings getSettings(TenantId tenantId) {
return attributesService.find(tenantId, tenantId, DataConstants.SERVER_SCOPE, SETTINGS_KEY).get()
.flatMap(KvEntry::getJsonValue)
.map(json -> {
try {
return JacksonUtil.fromString(json, EntitiesVersionControlSettings.class);
} catch (IllegalArgumentException e) {
return null;
}
})
.orElse(null);
return gitService.getSettings(tenantId);
}
private EntityVersion toVersion(GitRepository.Commit commit) {
return new EntityVersion(commit.getId(), commit.getMessage());
}
private String getRelativePath(EntityType entityType, String entityId) {
String path = entityType.name().toLowerCase();
if (entityId != null) {
path += "/" + entityId + ".json";
}
return path;
}
private EntityId fromRelativePath(String path) {
EntityType entityType = EntityType.valueOf(StringUtils.substringBefore(path, "/").toUpperCase());
String entityId = StringUtils.substringBetween(path, "/", ".json");
return EntityIdFactory.getByTypeAndUuid(entityType, entityId);
}
}

2
application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java
View File

@ -41,7 +41,7 @@ public interface EntitiesVersionControlService {
List<EntityVersion> listVersions(TenantId tenantId, String branch) throws Exception;
List<VersionedEntityInfo> listEntitiesAtVersion(TenantId tenantId, EntityType entityType, String branch, String versionId) throws Exception;
List<VersionedEntityInfo> listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType) throws Exception;
List<VersionedEntityInfo> listAllEntitiesAtVersion(TenantId tenantId, String branch, String versionId) throws Exception;

258
application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java Normal file
View File

@ -0,0 +1,258 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.sync.vc;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectWriter;
import com.fasterxml.jackson.databind.SerializationFeature;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Service;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.common.data.DataConstants;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.ExportableEntity;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.kv.KvEntry;
import org.thingsboard.server.common.data.sync.ie.EntityExportData;
import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings;
import org.thingsboard.server.common.data.sync.vc.EntityVersion;
import org.thingsboard.server.common.data.sync.vc.VersionCreationResult;
import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo;
import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest;
import org.thingsboard.server.dao.DaoUtil;
import org.thingsboard.server.dao.attributes.AttributesService;
import org.thingsboard.server.dao.tenant.TenantDao;
import org.thingsboard.server.queue.util.AfterStartUp;
import java.io.IOException;
import java.util.ConcurrentModificationException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import java.util.function.Consumer;
import java.util.function.Function;
@Slf4j
@RequiredArgsConstructor
@Service
@ConditionalOnProperty(prefix = "vc", value = "git.service", havingValue = "local", matchIfMissing = true)
public class LocalGitVersionControlService implements GitVersionControlService {
private final ObjectWriter jsonWriter = new ObjectMapper().writer(SerializationFeature.INDENT_OUTPUT);
private final GitRepositoryService gitRepositoryService;
private final TenantDao tenantDao;
private final AttributesService attributesService;
private final ConcurrentMap<TenantId, Lock> tenantRepoLocks = new ConcurrentHashMap<>();
private final Map<TenantId, PendingCommit> pendingCommitMap = new HashMap<>();
@AfterStartUp
public void init() {
DaoUtil.processInBatches(tenantDao::findTenantsIds, 100, tenantId -> {
EntitiesVersionControlSettings settings = getSettings(tenantId);
if (settings != null) {
try {
gitRepositoryService.initRepository(tenantId, settings);
} catch (Exception e) {
log.warn("Failed to init repository for tenant {}", tenantId, e);
}
}
});
}
@Override
@SneakyThrows
public EntitiesVersionControlSettings getSettings(TenantId tenantId) {
return attributesService.find(tenantId, tenantId, DataConstants.SERVER_SCOPE, DefaultEntitiesVersionControlService.SETTINGS_KEY).get()
.flatMap(KvEntry::getJsonValue)
.map(json -> {
try {
return JacksonUtil.fromString(json, EntitiesVersionControlSettings.class);
} catch (IllegalArgumentException e) {
return null;
}
})
.orElse(null);
}
@Override
public void initRepository(TenantId tenantId, EntitiesVersionControlSettings settings) {
var lock = getRepoLock(tenantId);
lock.lock();
try {
gitRepositoryService.initRepository(tenantId, settings);
} catch (Exception e) {
//TODO: analyze and return meaningful exceptions that we can show to the client;
throw new RuntimeException(e);
} finally {
lock.unlock();
}
}
@Override
public PendingCommit prepareCommit(TenantId tenantId, VersionCreateRequest request) {
var lock = getRepoLock(tenantId);
lock.lock();
try {
var pendingCommit = new PendingCommit(tenantId, request);
PendingCommit old = pendingCommitMap.put(tenantId, pendingCommit);
if (old != null) {
gitRepositoryService.abort(old);
}
gitRepositoryService.prepareCommit(pendingCommit);
return pendingCommit;
} finally {
lock.unlock();
}
}
@Override
public void deleteAll(PendingCommit commit, EntityType entityType) {
doInsideLock(commit, c -> {
try {
gitRepositoryService.deleteFolderContent(commit, getRelativePath(entityType, null));
} catch (IOException e) {
//TODO: analyze and return meaningful exceptions that we can show to the client;
throw new RuntimeException(e);
}
});
}
@Override
public void addToCommit(PendingCommit commit, EntityExportData<ExportableEntity<EntityId>> entityData) {
doInsideLock(commit, c -> {
String entityDataJson;
try {
entityDataJson = jsonWriter.writeValueAsString(entityData);
gitRepositoryService.add(c, getRelativePath(entityData.getEntityType(),
entityData.getEntity().getId().toString()), entityDataJson);
} catch (IOException e) {
//TODO: analyze and return meaningful exceptions that we can show to the client;
throw new RuntimeException(e);
}
});
}
@Override
public VersionCreationResult push(PendingCommit commit) {
return executeInsideLock(commit, gitRepositoryService::push);
}
@Override
public List<EntityVersion> listVersions(TenantId tenantId, String branch) {
return listVersions(tenantId, branch, (String) null);
}
@Override
public List<EntityVersion> listVersions(TenantId tenantId, String branch, EntityType entityType) {
return listVersions(tenantId, branch, getRelativePath(entityType, null));
}
@Override
public List<EntityVersion> listVersions(TenantId tenantId, String branch, EntityId entityId) {
return listVersions(tenantId, branch, getRelativePath(entityId.getEntityType(), entityId.getId().toString()));
}
@Override
public List<VersionedEntityInfo> listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType) {
try {
return gitRepositoryService.listEntitiesAtVersion(tenantId, branch, versionId, entityType != null ? getRelativePath(entityType, null) : null);
} catch (Exception e) {
//TODO: analyze and return meaningful exceptions that we can show to the client;
throw new RuntimeException(e);
}
}
@Override
public List<VersionedEntityInfo> listEntitiesAtVersion(TenantId tenantId, String branch, String versionId) {
return listEntitiesAtVersion(tenantId, branch, versionId, null);
}
@Override
public List<String> listBranches(TenantId tenantId) {
return gitRepositoryService.listBranches(tenantId);
}
@Override
public EntityExportData<?> getEntity(TenantId tenantId, String versionId, EntityId entityId) {
try {
String entityDataJson = gitRepositoryService.getFileContentAtCommit(tenantId,
getRelativePath(entityId.getEntityType(), entityId.getId().toString()), versionId);
return JacksonUtil.fromString(entityDataJson, EntityExportData.class);
} catch (Exception e) {
//TODO: analyze and return meaningful exceptions that we can show to the client;
throw new RuntimeException(e);
}
}
private List<EntityVersion> listVersions(TenantId tenantId, String branch, String path) {
try {
return gitRepositoryService.listVersions(tenantId, branch, path);
} catch (Exception e) {
//TODO: analyze and return meaningful exceptions that we can show to the client;
throw new RuntimeException(e);
}
}
private void doInsideLock(PendingCommit commit, Consumer<PendingCommit> r) {
var lock = getRepoLock(commit.getTenantId());
lock.lock();
try {
checkCommit(commit);
r.accept(commit);
} finally {
lock.unlock();
}
}
private <T> T executeInsideLock(PendingCommit commit, Function<PendingCommit, T> c) {
var lock = getRepoLock(commit.getTenantId());
lock.lock();
try {
checkCommit(commit);
return c.apply(commit);
} finally {
lock.unlock();
}
}
private void checkCommit(PendingCommit commit) {
PendingCommit existing = pendingCommitMap.get(commit.getTenantId());
if (existing == null || !existing.getTxId().equals(commit.getTxId())) {
throw new ConcurrentModificationException();
}
}
private String getRelativePath(EntityType entityType, String entityId) {
String path = entityType.name().toLowerCase();
if (entityId != null) {
path += "/" + entityId + ".json";
}
return path;
}
private Lock getRepoLock(TenantId tenantId) {
return tenantRepoLocks.computeIfAbsent(tenantId, t -> new ReentrantLock());
}
}

5
application/src/main/resources/thingsboard.yml
View File

@ -1115,6 +1115,11 @@ metrics:
# Metrics percentiles returned by actuator for timer metrics. List of double values (divided by ,).
percentiles: "${METRICS_TIMER_PERCENTILES:0.5}"
vc:
git:
service: "${JS_VC_GIT_SERVICE:local}" # local/remote
repos-poll-interval: "${TB_VC_GIT_REPOS_POLL_INTERVAL_SEC:60}"
management:
endpoints:
web:

21
common/version-control/pom.xml
View File

@ -40,6 +40,23 @@
<groupId>org.springframework</groupId>
<artifactId>spring-core</artifactId>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-context-support</artifactId>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-context</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>javax.annotation</groupId>
<artifactId>javax.annotation-api</artifactId>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
@ -65,6 +82,10 @@
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
</dependency>
<dependency>
<groupId>org.eclipse.jgit</groupId>
<artifactId>org.eclipse.jgit</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>

239
common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java Normal file
View File

@ -0,0 +1,239 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.sync.vc;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.eclipse.jgit.api.errors.JGitInternalException;
import org.eclipse.jgit.api.errors.RefAlreadyExistsException;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Service;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.EntityIdFactory;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings;
import org.thingsboard.server.common.data.sync.vc.EntityVersion;
import org.thingsboard.server.common.data.sync.vc.VersionCreationResult;
import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
@Slf4j
@ConditionalOnProperty(prefix = "vc", value = "git.service", havingValue = "local", matchIfMissing = true)
@Service
public class DefaultGitRepositoryService implements GitRepositoryService {
@Value("${vc.git.repos-poll-interval:${java.io.tmpdir}/repositories}")
private String repositoriesFolder;
@Value("${vc.git.repos-poll-interval:60}")
private long reposPollInterval;
private ScheduledExecutorService scheduler;
private final Map<TenantId, GitRepository> repositories = new ConcurrentHashMap<>();
@PostConstruct
public void init() {
scheduler = Executors.newSingleThreadScheduledExecutor();
scheduler.scheduleWithFixedDelay(() -> {
repositories.forEach((tenantId, repository) -> {
try {
repository.fetch();
log.info("Fetching remote repository for tenant {}", tenantId);
} catch (Exception e) {
log.warn("Failed to fetch repository for tenant {}", tenantId, e);
}
});
}, reposPollInterval, reposPollInterval, TimeUnit.SECONDS);
}
@PreDestroy
public void stop() {
if (scheduler != null) {
scheduler.shutdownNow();
}
}
@Override
public void prepareCommit(PendingCommit commit) {
GitRepository repository = checkRepository(commit.getTenantId());
String branch = commit.getRequest().getBranch();
try {
repository.fetch();
if (repository.listBranches().contains(branch)) {
repository.checkout("origin/" + branch, false);
try {
repository.checkout(branch, true);
} catch (RefAlreadyExistsException e) {
repository.checkout(branch, false);
}
repository.merge(branch);
} else { // TODO [viacheslav]: rollback orphan branch on failure
try {
repository.createAndCheckoutOrphanBranch(branch); // FIXME [viacheslav]: Checkout returned unexpected result NO_CHANGE for master branch
} catch (JGitInternalException e) {
if (!e.getMessage().contains("NO_CHANGE")) {
throw e;
}
}
}
} catch (IOException | GitAPIException gitAPIException) {
//TODO: analyze and return meaningful exceptions that we can show to the client;
throw new RuntimeException(gitAPIException);
}
}
@Override
public void deleteFolderContent(PendingCommit commit, String relativePath) throws IOException {
GitRepository repository = checkRepository(commit.getTenantId());
FileUtils.deleteDirectory(Path.of(repository.getDirectory(), relativePath).toFile());
}
@Override
public void add(PendingCommit commit, String relativePath, String entityDataJson) throws IOException {
GitRepository repository = checkRepository(commit.getTenantId());
FileUtils.write(Path.of(repository.getDirectory(), relativePath).toFile(), entityDataJson, StandardCharsets.UTF_8);
}
@Override
public VersionCreationResult push(PendingCommit commit) {
GitRepository repository = checkRepository(commit.getTenantId());
try {
repository.add(".");
VersionCreationResult result = new VersionCreationResult();
GitRepository.Status status = repository.status();
result.setAdded(status.getAdded().size());
result.setModified(status.getModified().size());
result.setRemoved(status.getRemoved().size());
GitRepository.Commit gitCommit = repository.commit(commit.getRequest().getVersionName());
repository.push();
result.setVersion(toVersion(gitCommit));
return result;
} catch (GitAPIException gitAPIException) {
//TODO: analyze and return meaningful exceptions that we can show to the client;
throw new RuntimeException(gitAPIException);
}
}
@Override
public void abort(PendingCommit commit) {
//TODO: implement;
}
@Override
public String getFileContentAtCommit(TenantId tenantId, String relativePath, String versionId) throws IOException {
GitRepository repository = checkRepository(tenantId);
return repository.getFileContentAtCommit(relativePath, versionId);
}
@Override
public List<String> listBranches(TenantId tenantId) {
GitRepository repository = checkRepository(tenantId);
try {
return repository.listBranches();
} catch (GitAPIException gitAPIException) {
//TODO: analyze and return meaningful exceptions that we can show to the client;
throw new RuntimeException(gitAPIException);
}
}
private EntityVersion checkVersion(TenantId tenantId, String branch, String versionId) throws Exception {
return listVersions(tenantId, branch, null).stream()
.filter(version -> version.getId().equals(versionId))
.findFirst().orElseThrow(() -> new IllegalArgumentException("Version not found"));
}
private GitRepository checkRepository(TenantId tenantId) {
return Optional.ofNullable(repositories.get(tenantId))
.orElseThrow(() -> new IllegalStateException("Repository is not initialized"));
}
@Override
public List<EntityVersion> listVersions(TenantId tenantId, String branch, String path) throws Exception {
GitRepository repository = checkRepository(tenantId);
return repository.listCommits(branch, path, Integer.MAX_VALUE).stream()
.map(this::toVersion)
.collect(Collectors.toList());
}
@Override
public List<VersionedEntityInfo> listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, String path) throws Exception {
GitRepository repository = checkRepository(tenantId);
checkVersion(tenantId, branch, versionId);
return repository.listFilesAtCommit(versionId, path).stream()
.map(filePath -> {
EntityId entityId = fromRelativePath(filePath);
VersionedEntityInfo info = new VersionedEntityInfo();
info.setExternalId(entityId);
return info;
})
.collect(Collectors.toList());
}
@Override
public void initRepository(TenantId tenantId, EntitiesVersionControlSettings settings) throws Exception {
Path repositoryDirectory = Path.of(repositoriesFolder, tenantId.getId().toString());
GitRepository repository;
if (Files.exists(repositoryDirectory)) {
FileUtils.forceDelete(repositoryDirectory.toFile());
}
Files.createDirectories(repositoryDirectory);
repository = GitRepository.clone(settings.getRepositoryUri(), settings.getUsername(), settings.getPassword(), repositoryDirectory.toFile());
repositories.put(tenantId, repository);
}
private void clearRepository(TenantId tenantId) throws IOException {
GitRepository repository = repositories.get(tenantId);
if (repository != null) {
FileUtils.deleteDirectory(new File(repository.getDirectory()));
repositories.remove(tenantId);
}
}
private EntityVersion toVersion(GitRepository.Commit commit) {
return new EntityVersion(commit.getId(), commit.getMessage());
}
private EntityId fromRelativePath(String path) {
EntityType entityType = EntityType.valueOf(StringUtils.substringBefore(path, "/").toUpperCase());
String entityId = StringUtils.substringBetween(path, "/", ".json");
return EntityIdFactory.getByTypeAndUuid(entityType, entityId);
}
}

2
application/src/main/java/org/thingsboard/server/utils/GitRepository.java → common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java
View File

@ -13,7 +13,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.utils;
package org.thingsboard.server.service.sync.vc;
import com.google.common.collect.Streams;
import lombok.Data;

48
common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java Normal file
View File

@ -0,0 +1,48 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.sync.vc;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings;
import org.thingsboard.server.common.data.sync.vc.EntityVersion;
import org.thingsboard.server.common.data.sync.vc.VersionCreationResult;
import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo;
import java.io.IOException;
import java.util.List;
public interface GitRepositoryService {
void prepareCommit(PendingCommit pendingCommit);
List<EntityVersion> listVersions(TenantId tenantId, String branch, String path) throws Exception;
List<VersionedEntityInfo> listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, String path) throws Exception;
void initRepository(TenantId tenantId, EntitiesVersionControlSettings settings) throws Exception;
void add(PendingCommit commit, String relativePath, String entityDataJson) throws IOException;
void deleteFolderContent(PendingCommit commit, String relativePath) throws IOException;
VersionCreationResult push(PendingCommit commit);
void abort(PendingCommit commit);
List<String> listBranches(TenantId tenantId);
String getFileContentAtCommit(TenantId tenantId, String relativePath, String versionId) throws IOException;
}

60
common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java Normal file
View File

@ -0,0 +1,60 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.sync.vc;
import lombok.SneakyThrows;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.ExportableEntity;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.sync.ie.EntityExportData;
import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings;
import org.thingsboard.server.common.data.sync.vc.EntityVersion;
import org.thingsboard.server.common.data.sync.vc.VersionCreationResult;
import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo;
import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest;
import java.util.List;
public interface GitVersionControlService {
@SneakyThrows
EntitiesVersionControlSettings getSettings(TenantId tenantId);
void initRepository(TenantId tenantId, EntitiesVersionControlSettings settings);
PendingCommit prepareCommit(TenantId tenantId, VersionCreateRequest request);
void addToCommit(PendingCommit commit, EntityExportData<ExportableEntity<EntityId>> entityData);
void deleteAll(PendingCommit pendingCommit, EntityType entityType);
VersionCreationResult push(PendingCommit commit);
List<EntityVersion> listVersions(TenantId tenantId, String branch);
List<EntityVersion> listVersions(TenantId tenantId, String branch, EntityType entityType);
List<EntityVersion> listVersions(TenantId tenantId, String branch, EntityId entityId);
List<VersionedEntityInfo> listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType);
List<VersionedEntityInfo> listEntitiesAtVersion(TenantId tenantId, String branch, String versionId);
List<String> listBranches(TenantId tenantId);
EntityExportData<?> getEntity(TenantId tenantId, String versionId, EntityId entityId);
}

36
common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/PendingCommit.java Normal file
View File

@ -0,0 +1,36 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.sync.vc;
import lombok.Data;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest;
import java.util.UUID;
@Data
public class PendingCommit {
private final UUID txId;
private final TenantId tenantId;
private final VersionCreateRequest request;
public PendingCommit(TenantId tenantId, VersionCreateRequest request) {
this.txId = UUID.randomUUID();
this.tenantId = tenantId;
this.request = request;
}
}