Version Control Service refactoring

application/pom.xml

@@ -345,10 +345,6 @@
             <artifactId>Java-WebSocket</artifactId>
             <scope>test</scope>
         </dependency>
-        <dependency>
-            <groupId>org.eclipse.jgit</groupId>
-            <artifactId>org.eclipse.jgit</artifactId>
-        </dependency>
     </dependencies>
 
     <build>

application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java

@@ -172,7 +172,7 @@ public class EntitiesVersionControlController extends BaseController {
                                                            @PathVariable EntityType entityType,
                                                            @PathVariable String versionId) throws ThingsboardException {
         try {
-            return versionControlService.listEntitiesAtVersion(getTenantId(), entityType, branch, versionId);
+            return versionControlService.listEntitiesAtVersion(getTenantId(), branch, versionId, entityType);
         } catch (Exception e) {
             throw handleException(e);
         }

application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java

@@ -75,7 +75,6 @@ import org.thingsboard.server.common.data.sync.vc.request.load.EntityTypeVersion
 import org.thingsboard.server.common.data.sync.vc.request.load.SingleEntityVersionLoadRequest;
 import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadConfig;
 import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadRequest;
-import org.thingsboard.server.utils.GitRepository;
 import org.thingsboard.server.common.data.sync.ThrowingRunnable;
 
 import java.io.File;
@@ -105,85 +104,31 @@ import static org.thingsboard.server.dao.sql.query.EntityKeyMapping.CREATED_TIME
 @Slf4j
 public class DefaultEntitiesVersionControlService implements EntitiesVersionControlService {
 
+    private final GitVersionControlService gitService;
     private final EntitiesExportImportService exportImportService;
     private final ExportableEntitiesService exportableEntitiesService;
     private final AttributesService attributesService;
     private final EntityService entityService;
-    private final TenantDao tenantDao;
     private final TransactionTemplate transactionTemplate;
 
-    // TODO [viacheslav]: concurrency
+    public static final String SETTINGS_KEY = "vc";
-    private final Map<TenantId, GitRepository> repositories = new ConcurrentHashMap<>();
-    @Value("${java.io.tmpdir}/repositories")
-    private String repositoriesFolder;
-
-    private static final String SETTINGS_KEY = "vc";
-    private final ObjectWriter jsonWriter = new ObjectMapper().writer(SerializationFeature.INDENT_OUTPUT);
-
-
-    @AfterStartUp
-    public void init() {
-        DaoUtil.processInBatches(tenantDao::findTenantsIds, 100, tenantId -> {
-            EntitiesVersionControlSettings settings = getSettings(tenantId);
-            if (settings != null) {
-                try {
-                    initRepository(tenantId, settings);
-                } catch (Exception e) {
-                    log.warn("Failed to init repository for tenant {}", tenantId, e);
-                }
-            }
-        });
-        Executors.newSingleThreadScheduledExecutor().scheduleWithFixedDelay(() -> {
-            repositories.forEach((tenantId, repository) -> {
-                try {
-                    repository.fetch();
-                    log.info("Fetching remote repository for tenant {}", tenantId);
-                } catch (Exception e) {
-                    log.warn("Failed to fetch repository for tenant {}", tenantId, e);
-                }
-            });
-        }, 5, 5, TimeUnit.SECONDS);
-    }
-
 
     @Override
     public VersionCreationResult saveEntitiesVersion(SecurityUser user, VersionCreateRequest request) throws Exception {
-        GitRepository repository = checkRepository(user.getTenantId());
 
-        repository.fetch();
+        var commit = gitService.prepareCommit(user.getTenantId(), request);
-        if (repository.listBranches().contains(request.getBranch())) {
-            repository.checkout("origin/" + request.getBranch(), false);
-            try {
-                repository.checkout(request.getBranch(), true);
-            } catch (RefAlreadyExistsException e) {
-                repository.checkout(request.getBranch(), false);
-            }
-            repository.merge(request.getBranch());
-        } else { // TODO [viacheslav]: rollback orphan branch on failure
-            try {
-                repository.createAndCheckoutOrphanBranch(request.getBranch()); // FIXME [viacheslav]: Checkout returned unexpected result NO_CHANGE for master branch
-            } catch (JGitInternalException e) {
-                if (!e.getMessage().contains("NO_CHANGE")) {
-                    throw e;
-                }
-            }
-        }
 
         switch (request.getType()) {
             case SINGLE_ENTITY: {
                 SingleEntityVersionCreateRequest versionCreateRequest = (SingleEntityVersionCreateRequest) request;
-                saveEntityData(user, repository, versionCreateRequest.getEntityId(), versionCreateRequest.getConfig());
+                saveEntityData(user, commit, versionCreateRequest.getEntityId(), versionCreateRequest.getConfig());
                 break;
             }
             case COMPLEX: {
                 ComplexVersionCreateRequest versionCreateRequest = (ComplexVersionCreateRequest) request;
                 versionCreateRequest.getEntityTypes().forEach((entityType, config) -> {
                     if (ObjectUtils.defaultIfNull(config.getSyncStrategy(), versionCreateRequest.getSyncStrategy()) == SyncStrategy.OVERWRITE) {
-                        try {
+                        gitService.deleteAll(commit, entityType);
-                            FileUtils.deleteDirectory(Path.of(repository.getDirectory(), getRelativePath(entityType, null)).toFile());
-                        } catch (IOException e) {
-                            throw new RuntimeException(e);
-                        }
                     }
 
                     if (config.isAllEntities()) {
@@ -203,7 +148,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
                         }, 100, data -> {
                             EntityId entityId = data.getEntityId();
                             try {
-                                saveEntityData(user, repository, entityId, config);
+                                saveEntityData(user, commit, entityId, config);
                             } catch (Exception e) {
                                 throw new RuntimeException(e);
                             }
@@ -211,7 +156,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
                     } else {
                         for (UUID entityId : config.getEntityIds()) {
                             try {
-                                saveEntityData(user, repository, EntityIdFactory.getByTypeAndUuid(entityType, entityId), config);
+                                saveEntityData(user, commit, EntityIdFactory.getByTypeAndUuid(entityType, entityId), config);
                             } catch (Exception e) {
                                 throw new RuntimeException(e);
                             }
@@ -223,90 +168,50 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
             }
         }
 
-        repository.add(".");
+        return gitService.push(commit);
-
-        VersionCreationResult result = new VersionCreationResult();
-        GitRepository.Status status = repository.status();
-        result.setAdded(status.getAdded().size());
-        result.setModified(status.getModified().size());
-        result.setRemoved(status.getRemoved().size());
-
-        GitRepository.Commit commit = repository.commit(request.getVersionName());
-        repository.push();
-
-        result.setVersion(toVersion(commit));
-        return result;
     }
 
-    private void saveEntityData(SecurityUser user, GitRepository repository, EntityId entityId, VersionCreateConfig config) throws Exception {
+    private void saveEntityData(SecurityUser user, PendingCommit commit, EntityId entityId, VersionCreateConfig config) throws Exception {
         EntityExportData<ExportableEntity<EntityId>> entityData = exportImportService.exportEntity(user, entityId, EntityExportSettings.builder()
                 .exportRelations(config.isSaveRelations())
                 .build());
-        String entityDataJson = jsonWriter.writeValueAsString(entityData);
+        gitService.addToCommit(commit, entityData);
-        FileUtils.write(Path.of(repository.getDirectory(), getRelativePath(entityData.getEntityType(),
-                entityData.getEntity().getId().toString())).toFile(), entityDataJson, StandardCharsets.UTF_8);
     }
 
 
     @Override
     public List<EntityVersion> listEntityVersions(TenantId tenantId, String branch, EntityId externalId) throws Exception {
-        return listVersions(tenantId, branch, getRelativePath(externalId.getEntityType(), externalId.getId().toString()));
+        return gitService.listVersions(tenantId, branch, externalId);
     }
 
     @Override
     public List<EntityVersion> listEntityTypeVersions(TenantId tenantId, String branch, EntityType entityType) throws Exception {
-        return listVersions(tenantId, branch, getRelativePath(entityType, null));
+        return gitService.listVersions(tenantId, branch, entityType);
     }
 
     @Override
     public List<EntityVersion> listVersions(TenantId tenantId, String branch) throws Exception {
-        return listVersions(tenantId, branch, null);
+        return gitService.listVersions(tenantId, branch);
     }
 
-    private List<EntityVersion> listVersions(TenantId tenantId, String branch, String path) throws Exception {
-        GitRepository repository = checkRepository(tenantId);
-        return repository.listCommits(branch, path, Integer.MAX_VALUE).stream()
-                .map(this::toVersion)
-                .collect(Collectors.toList());
-    }
-
-
     @Override
-    public List<VersionedEntityInfo> listEntitiesAtVersion(TenantId tenantId, EntityType entityType, String branch, String versionId) throws Exception {
+    public List<VersionedEntityInfo> listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType) throws Exception {
-        return listEntitiesAtVersion(tenantId, branch, versionId, getRelativePath(entityType, null));
+        return gitService.listEntitiesAtVersion(tenantId, branch, versionId, entityType);
     }
 
     @Override
     public List<VersionedEntityInfo> listAllEntitiesAtVersion(TenantId tenantId, String branch, String versionId) throws Exception {
-        return listEntitiesAtVersion(tenantId, branch, versionId, null);
+        return gitService.listEntitiesAtVersion(tenantId, branch, versionId);
     }
 
-    private List<VersionedEntityInfo> listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, String path) throws Exception {
-        GitRepository repository = checkRepository(tenantId);
-        checkVersion(tenantId, branch, versionId);
-        return repository.listFilesAtCommit(versionId, path).stream()
-                .map(filePath -> {
-                    EntityId entityId = fromRelativePath(filePath);
-                    VersionedEntityInfo info = new VersionedEntityInfo();
-                    info.setExternalId(entityId);
-                    return info;
-                })
-                .collect(Collectors.toList());
-    }
-
-
     @Override
     public List<VersionLoadResult> loadEntitiesVersion(SecurityUser user, VersionLoadRequest request) throws Exception {
-        GitRepository repository = checkRepository(user.getTenantId());
-
-        EntityVersion version = checkVersion(user.getTenantId(), request.getBranch(), request.getVersionId());
-
         switch (request.getType()) {
             case SINGLE_ENTITY: {
                 SingleEntityVersionLoadRequest versionLoadRequest = (SingleEntityVersionLoadRequest) request;
                 EntityImportResult<?> importResult = transactionTemplate.execute(status -> {
                     try {
-                        EntityImportResult<?> result = loadEntity(user, repository, versionLoadRequest.getExternalEntityId(), version.getId(), versionLoadRequest.getConfig());
+                        EntityImportResult<?> result = loadEntity(user, request, versionLoadRequest.getConfig(), versionLoadRequest.getExternalEntityId());
                         result.getSaveReferencesCallback().run();
                         return result;
                     } catch (Exception e) {
@@ -340,11 +245,11 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
 
                         Set<EntityId> remoteEntities;
                         try {
-                            remoteEntities = listEntitiesAtVersion(user.getTenantId(), request.getBranch(), request.getVersionId(), getRelativePath(entityType, null)).stream()
+                            remoteEntities = listEntitiesAtVersion(user.getTenantId(), request.getBranch(), request.getVersionId(), entityType).stream()
                                     .map(VersionedEntityInfo::getExternalId)
                                     .collect(Collectors.toSet());
                             for (EntityId externalEntityId : remoteEntities) {
-                                EntityImportResult<?> importResult = loadEntity(user, repository, externalEntityId, version.getId(), config);
+                                EntityImportResult<?> importResult = loadEntity(user, request, config, externalEntityId);
 
                                 if (importResult.getOldEntity() == null) created.incrementAndGet();
                                 else updated.incrementAndGet();
@@ -402,10 +307,8 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
         }
     }
 
-    private EntityImportResult<?> loadEntity(SecurityUser user, GitRepository repository, EntityId externalId, String versionId, VersionLoadConfig config) throws Exception {
+    private EntityImportResult<?> loadEntity(SecurityUser user, VersionLoadRequest request, VersionLoadConfig config, EntityId entityId) throws Exception {
-        String entityDataJson = repository.getFileContentAtCommit(getRelativePath(externalId.getEntityType(), externalId.getId().toString()), versionId);
+        EntityExportData entityData = gitService.getEntity(user.getTenantId(), request.getVersionId(), entityId);
-        EntityExportData entityData = JacksonUtil.fromString(entityDataJson, EntityExportData.class);
-
         return exportImportService.importEntity(user, entityData, EntityImportSettings.builder()
                 .updateRelations(config.isLoadRelations())
                 .findExistingByName(config.isFindExistingEntityByName())
@@ -415,43 +318,9 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
 
     @Override
     public List<String> listBranches(TenantId tenantId) throws Exception {
-        GitRepository repository = checkRepository(tenantId);
+        return gitService.listBranches(tenantId);
-        return repository.listBranches();
     }
 
-
-    private EntityVersion checkVersion(TenantId tenantId, String branch, String versionId) throws Exception {
-        return listVersions(tenantId, branch, null).stream()
-                .filter(version -> version.getId().equals(versionId))
-                .findFirst().orElseThrow(() -> new IllegalArgumentException("Version not found"));
-    }
-
-    private GitRepository checkRepository(TenantId tenantId) {
-        return Optional.ofNullable(repositories.get(tenantId))
-                .orElseThrow(() -> new IllegalStateException("Repository is not initialized"));
-    }
-
-    private void initRepository(TenantId tenantId, EntitiesVersionControlSettings settings) throws Exception {
-        Path repositoryDirectory = Path.of(repositoriesFolder, tenantId.getId().toString());
-        GitRepository repository;
-        if (Files.exists(repositoryDirectory)) {
-            FileUtils.forceDelete(repositoryDirectory.toFile());
-        }
-
-        Files.createDirectories(repositoryDirectory);
-        repository = GitRepository.clone(settings.getRepositoryUri(), settings.getUsername(), settings.getPassword(), repositoryDirectory.toFile());
-        repositories.put(tenantId, repository);
-    }
-
-    private void clearRepository(TenantId tenantId) throws IOException {
-        GitRepository repository = repositories.get(tenantId);
-        if (repository != null) {
-            FileUtils.deleteDirectory(new File(repository.getDirectory()));
-            repositories.remove(tenantId);
-        }
-    }
-
-
     @SneakyThrows
     @Override
     public void saveSettings(TenantId tenantId, EntitiesVersionControlSettings settings) {
@@ -459,41 +328,11 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
                 new BaseAttributeKvEntry(System.currentTimeMillis(), new JsonDataEntry(SETTINGS_KEY, JacksonUtil.toString(settings)))
         )).get();
 
-        initRepository(tenantId, settings);
+        gitService.initRepository(tenantId, settings);
     }
 
-    @SneakyThrows
     @Override
     public EntitiesVersionControlSettings getSettings(TenantId tenantId) {
-        return attributesService.find(tenantId, tenantId, DataConstants.SERVER_SCOPE, SETTINGS_KEY).get()
+        return gitService.getSettings(tenantId);
-                .flatMap(KvEntry::getJsonValue)
-                .map(json -> {
-                    try {
-                        return JacksonUtil.fromString(json, EntitiesVersionControlSettings.class);
-                    } catch (IllegalArgumentException e) {
-                        return null;
-                    }
-                })
-                .orElse(null);
     }
-
-
-    private EntityVersion toVersion(GitRepository.Commit commit) {
-        return new EntityVersion(commit.getId(), commit.getMessage());
-    }
-
-    private String getRelativePath(EntityType entityType, String entityId) {
-        String path = entityType.name().toLowerCase();
-        if (entityId != null) {
-            path += "/" + entityId + ".json";
-        }
-        return path;
-    }
-
-    private EntityId fromRelativePath(String path) {
-        EntityType entityType = EntityType.valueOf(StringUtils.substringBefore(path, "/").toUpperCase());
-        String entityId = StringUtils.substringBetween(path, "/", ".json");
-        return EntityIdFactory.getByTypeAndUuid(entityType, entityId);
-    }
-
 }

application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java

@@ -41,7 +41,7 @@ public interface EntitiesVersionControlService {
     List<EntityVersion> listVersions(TenantId tenantId, String branch) throws Exception;
 
 
-    List<VersionedEntityInfo> listEntitiesAtVersion(TenantId tenantId, EntityType entityType, String branch, String versionId) throws Exception;
+    List<VersionedEntityInfo> listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType) throws Exception;
 
     List<VersionedEntityInfo> listAllEntitiesAtVersion(TenantId tenantId, String branch, String versionId) throws Exception;
 

application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java

@@ -0,0 +1,258 @@
+/**
+ * Copyright © 2016-2022 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.service.sync.vc;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectWriter;
+import com.fasterxml.jackson.databind.SerializationFeature;
+import lombok.RequiredArgsConstructor;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.stereotype.Service;
+import org.thingsboard.common.util.JacksonUtil;
+import org.thingsboard.server.common.data.DataConstants;
+import org.thingsboard.server.common.data.EntityType;
+import org.thingsboard.server.common.data.ExportableEntity;
+import org.thingsboard.server.common.data.id.EntityId;
+import org.thingsboard.server.common.data.id.TenantId;
+import org.thingsboard.server.common.data.kv.KvEntry;
+import org.thingsboard.server.common.data.sync.ie.EntityExportData;
+import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings;
+import org.thingsboard.server.common.data.sync.vc.EntityVersion;
+import org.thingsboard.server.common.data.sync.vc.VersionCreationResult;
+import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo;
+import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest;
+import org.thingsboard.server.dao.DaoUtil;
+import org.thingsboard.server.dao.attributes.AttributesService;
+import org.thingsboard.server.dao.tenant.TenantDao;
+import org.thingsboard.server.queue.util.AfterStartUp;
+
+import java.io.IOException;
+import java.util.ConcurrentModificationException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.function.Consumer;
+import java.util.function.Function;
+
+@Slf4j
+@RequiredArgsConstructor
+@Service
+@ConditionalOnProperty(prefix = "vc", value = "git.service", havingValue = "local", matchIfMissing = true)
+public class LocalGitVersionControlService implements GitVersionControlService {
+
+    private final ObjectWriter jsonWriter = new ObjectMapper().writer(SerializationFeature.INDENT_OUTPUT);
+    private final GitRepositoryService gitRepositoryService;
+    private final TenantDao tenantDao;
+    private final AttributesService attributesService;
+    private final ConcurrentMap<TenantId, Lock> tenantRepoLocks = new ConcurrentHashMap<>();
+    private final Map<TenantId, PendingCommit> pendingCommitMap = new HashMap<>();
+
+    @AfterStartUp
+    public void init() {
+        DaoUtil.processInBatches(tenantDao::findTenantsIds, 100, tenantId -> {
+            EntitiesVersionControlSettings settings = getSettings(tenantId);
+            if (settings != null) {
+                try {
+                    gitRepositoryService.initRepository(tenantId, settings);
+                } catch (Exception e) {
+                    log.warn("Failed to init repository for tenant {}", tenantId, e);
+                }
+            }
+        });
+    }
+
+    @Override
+    @SneakyThrows
+    public EntitiesVersionControlSettings getSettings(TenantId tenantId) {
+        return attributesService.find(tenantId, tenantId, DataConstants.SERVER_SCOPE, DefaultEntitiesVersionControlService.SETTINGS_KEY).get()
+                .flatMap(KvEntry::getJsonValue)
+                .map(json -> {
+                    try {
+                        return JacksonUtil.fromString(json, EntitiesVersionControlSettings.class);
+                    } catch (IllegalArgumentException e) {
+                        return null;
+                    }
+                })
+                .orElse(null);
+    }
+
+    @Override
+    public void initRepository(TenantId tenantId, EntitiesVersionControlSettings settings) {
+        var lock = getRepoLock(tenantId);
+        lock.lock();
+        try {
+            gitRepositoryService.initRepository(tenantId, settings);
+        } catch (Exception e) {
+            //TODO: analyze and return meaningful exceptions that we can show to the client;
+            throw new RuntimeException(e);
+        } finally {
+            lock.unlock();
+        }
+    }
+
+    @Override
+    public PendingCommit prepareCommit(TenantId tenantId, VersionCreateRequest request) {
+        var lock = getRepoLock(tenantId);
+        lock.lock();
+        try {
+            var pendingCommit = new PendingCommit(tenantId, request);
+            PendingCommit old = pendingCommitMap.put(tenantId, pendingCommit);
+            if (old != null) {
+                gitRepositoryService.abort(old);
+            }
+            gitRepositoryService.prepareCommit(pendingCommit);
+            return pendingCommit;
+        } finally {
+            lock.unlock();
+        }
+    }
+
+    @Override
+    public void deleteAll(PendingCommit commit, EntityType entityType) {
+        doInsideLock(commit, c -> {
+            try {
+                gitRepositoryService.deleteFolderContent(commit, getRelativePath(entityType, null));
+            } catch (IOException e) {
+                //TODO: analyze and return meaningful exceptions that we can show to the client;
+                throw new RuntimeException(e);
+            }
+        });
+    }
+
+    @Override
+    public void addToCommit(PendingCommit commit, EntityExportData<ExportableEntity<EntityId>> entityData) {
+        doInsideLock(commit, c -> {
+            String entityDataJson;
+            try {
+                entityDataJson = jsonWriter.writeValueAsString(entityData);
+                gitRepositoryService.add(c, getRelativePath(entityData.getEntityType(),
+                        entityData.getEntity().getId().toString()), entityDataJson);
+            } catch (IOException e) {
+                //TODO: analyze and return meaningful exceptions that we can show to the client;
+                throw new RuntimeException(e);
+            }
+        });
+    }
+
+    @Override
+    public VersionCreationResult push(PendingCommit commit) {
+        return executeInsideLock(commit, gitRepositoryService::push);
+    }
+
+    @Override
+    public List<EntityVersion> listVersions(TenantId tenantId, String branch) {
+        return listVersions(tenantId, branch, (String) null);
+    }
+
+    @Override
+    public List<EntityVersion> listVersions(TenantId tenantId, String branch, EntityType entityType) {
+        return listVersions(tenantId, branch, getRelativePath(entityType, null));
+    }
+
+    @Override
+    public List<EntityVersion> listVersions(TenantId tenantId, String branch, EntityId entityId) {
+        return listVersions(tenantId, branch, getRelativePath(entityId.getEntityType(), entityId.getId().toString()));
+    }
+
+    @Override
+    public List<VersionedEntityInfo> listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType) {
+        try {
+            return gitRepositoryService.listEntitiesAtVersion(tenantId, branch, versionId, entityType != null ? getRelativePath(entityType, null) : null);
+        } catch (Exception e) {
+            //TODO: analyze and return meaningful exceptions that we can show to the client;
+            throw new RuntimeException(e);
+        }
+    }
+
+    @Override
+    public List<VersionedEntityInfo> listEntitiesAtVersion(TenantId tenantId, String branch, String versionId) {
+        return listEntitiesAtVersion(tenantId, branch, versionId, null);
+    }
+
+    @Override
+    public List<String> listBranches(TenantId tenantId) {
+        return gitRepositoryService.listBranches(tenantId);
+    }
+
+    @Override
+    public EntityExportData<?> getEntity(TenantId tenantId, String versionId, EntityId entityId) {
+        try {
+            String entityDataJson = gitRepositoryService.getFileContentAtCommit(tenantId,
+                    getRelativePath(entityId.getEntityType(), entityId.getId().toString()), versionId);
+            return JacksonUtil.fromString(entityDataJson, EntityExportData.class);
+        } catch (Exception e) {
+            //TODO: analyze and return meaningful exceptions that we can show to the client;
+            throw new RuntimeException(e);
+        }
+    }
+
+    private List<EntityVersion> listVersions(TenantId tenantId, String branch, String path) {
+        try {
+            return gitRepositoryService.listVersions(tenantId, branch, path);
+        } catch (Exception e) {
+            //TODO: analyze and return meaningful exceptions that we can show to the client;
+            throw new RuntimeException(e);
+        }
+    }
+
+    private void doInsideLock(PendingCommit commit, Consumer<PendingCommit> r) {
+        var lock = getRepoLock(commit.getTenantId());
+        lock.lock();
+        try {
+            checkCommit(commit);
+            r.accept(commit);
+        } finally {
+            lock.unlock();
+        }
+    }
+
+    private <T> T executeInsideLock(PendingCommit commit, Function<PendingCommit, T> c) {
+        var lock = getRepoLock(commit.getTenantId());
+        lock.lock();
+        try {
+            checkCommit(commit);
+            return c.apply(commit);
+        } finally {
+            lock.unlock();
+        }
+    }
+
+    private void checkCommit(PendingCommit commit) {
+        PendingCommit existing = pendingCommitMap.get(commit.getTenantId());
+        if (existing == null || !existing.getTxId().equals(commit.getTxId())) {
+            throw new ConcurrentModificationException();
+        }
+    }
+
+    private String getRelativePath(EntityType entityType, String entityId) {
+        String path = entityType.name().toLowerCase();
+        if (entityId != null) {
+            path += "/" + entityId + ".json";
+        }
+        return path;
+    }
+
+    private Lock getRepoLock(TenantId tenantId) {
+        return tenantRepoLocks.computeIfAbsent(tenantId, t -> new ReentrantLock());
+    }
+
+}

application/src/main/resources/thingsboard.yml

@@ -1115,6 +1115,11 @@ metrics:
     # Metrics percentiles returned by actuator for timer metrics. List of double values (divided by ,).
     percentiles: "${METRICS_TIMER_PERCENTILES:0.5}"
 
+vc:
+  git:
+    service: "${JS_VC_GIT_SERVICE:local}" # local/remote
+    repos-poll-interval: "${TB_VC_GIT_REPOS_POLL_INTERVAL_SEC:60}"
+
 management:
   endpoints:
     web:

common/version-control/pom.xml

@@ -40,6 +40,23 @@
             <groupId>org.springframework</groupId>
             <artifactId>spring-core</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.springframework</groupId>
+            <artifactId>spring-context-support</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework</groupId>
+            <artifactId>spring-context</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-web</artifactId>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>javax.annotation</groupId>
+            <artifactId>javax.annotation-api</artifactId>
+        </dependency>
         <dependency>
             <groupId>com.google.guava</groupId>
             <artifactId>guava</artifactId>
@@ -65,6 +82,10 @@
             <groupId>ch.qos.logback</groupId>
             <artifactId>logback-classic</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.eclipse.jgit</groupId>
+            <artifactId>org.eclipse.jgit</artifactId>
+        </dependency>
         <dependency>
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-starter-test</artifactId>

common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java

@@ -0,0 +1,239 @@
+/**
+ * Copyright © 2016-2022 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.service.sync.vc;
+
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.eclipse.jgit.api.errors.GitAPIException;
+import org.eclipse.jgit.api.errors.JGitInternalException;
+import org.eclipse.jgit.api.errors.RefAlreadyExistsException;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.stereotype.Service;
+import org.thingsboard.server.common.data.EntityType;
+import org.thingsboard.server.common.data.id.EntityId;
+import org.thingsboard.server.common.data.id.EntityIdFactory;
+import org.thingsboard.server.common.data.id.TenantId;
+import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings;
+import org.thingsboard.server.common.data.sync.vc.EntityVersion;
+import org.thingsboard.server.common.data.sync.vc.VersionCreationResult;
+import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo;
+
+import javax.annotation.PostConstruct;
+import javax.annotation.PreDestroy;
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
+@Slf4j
+@ConditionalOnProperty(prefix = "vc", value = "git.service", havingValue = "local", matchIfMissing = true)
+@Service
+public class DefaultGitRepositoryService implements GitRepositoryService {
+
+    @Value("${vc.git.repos-poll-interval:${java.io.tmpdir}/repositories}")
+    private String repositoriesFolder;
+
+    @Value("${vc.git.repos-poll-interval:60}")
+    private long reposPollInterval;
+
+    private ScheduledExecutorService scheduler;
+    private final Map<TenantId, GitRepository> repositories = new ConcurrentHashMap<>();
+
+    @PostConstruct
+    public void init() {
+        scheduler = Executors.newSingleThreadScheduledExecutor();
+        scheduler.scheduleWithFixedDelay(() -> {
+            repositories.forEach((tenantId, repository) -> {
+                try {
+                    repository.fetch();
+                    log.info("Fetching remote repository for tenant {}", tenantId);
+                } catch (Exception e) {
+                    log.warn("Failed to fetch repository for tenant {}", tenantId, e);
+                }
+            });
+        }, reposPollInterval, reposPollInterval, TimeUnit.SECONDS);
+    }
+
+    @PreDestroy
+    public void stop() {
+        if (scheduler != null) {
+            scheduler.shutdownNow();
+        }
+    }
+
+    @Override
+    public void prepareCommit(PendingCommit commit) {
+        GitRepository repository = checkRepository(commit.getTenantId());
+        String branch = commit.getRequest().getBranch();
+        try {
+            repository.fetch();
+            if (repository.listBranches().contains(branch)) {
+                repository.checkout("origin/" + branch, false);
+                try {
+                    repository.checkout(branch, true);
+                } catch (RefAlreadyExistsException e) {
+                    repository.checkout(branch, false);
+                }
+                repository.merge(branch);
+            } else { // TODO [viacheslav]: rollback orphan branch on failure
+                try {
+                    repository.createAndCheckoutOrphanBranch(branch); // FIXME [viacheslav]: Checkout returned unexpected result NO_CHANGE for master branch
+                } catch (JGitInternalException e) {
+                    if (!e.getMessage().contains("NO_CHANGE")) {
+                        throw e;
+                    }
+                }
+            }
+        } catch (IOException | GitAPIException gitAPIException) {
+            //TODO: analyze and return meaningful exceptions that we can show to the client;
+            throw new RuntimeException(gitAPIException);
+        }
+    }
+
+    @Override
+    public void deleteFolderContent(PendingCommit commit, String relativePath) throws IOException {
+        GitRepository repository = checkRepository(commit.getTenantId());
+        FileUtils.deleteDirectory(Path.of(repository.getDirectory(), relativePath).toFile());
+    }
+
+    @Override
+    public void add(PendingCommit commit, String relativePath, String entityDataJson) throws IOException {
+        GitRepository repository = checkRepository(commit.getTenantId());
+        FileUtils.write(Path.of(repository.getDirectory(), relativePath).toFile(), entityDataJson, StandardCharsets.UTF_8);
+    }
+
+    @Override
+    public VersionCreationResult push(PendingCommit commit) {
+        GitRepository repository = checkRepository(commit.getTenantId());
+        try {
+            repository.add(".");
+
+            VersionCreationResult result = new VersionCreationResult();
+            GitRepository.Status status = repository.status();
+            result.setAdded(status.getAdded().size());
+            result.setModified(status.getModified().size());
+            result.setRemoved(status.getRemoved().size());
+
+            GitRepository.Commit gitCommit = repository.commit(commit.getRequest().getVersionName());
+            repository.push();
+
+            result.setVersion(toVersion(gitCommit));
+            return result;
+        } catch (GitAPIException gitAPIException) {
+            //TODO: analyze and return meaningful exceptions that we can show to the client;
+            throw new RuntimeException(gitAPIException);
+        }
+    }
+
+    @Override
+    public void abort(PendingCommit commit) {
+        //TODO: implement;
+    }
+
+    @Override
+    public String getFileContentAtCommit(TenantId tenantId, String relativePath, String versionId) throws IOException {
+       GitRepository repository = checkRepository(tenantId);
+       return repository.getFileContentAtCommit(relativePath, versionId);
+    }
+
+    @Override
+    public List<String> listBranches(TenantId tenantId) {
+        GitRepository repository = checkRepository(tenantId);
+        try {
+            return repository.listBranches();
+        } catch (GitAPIException gitAPIException) {
+            //TODO: analyze and return meaningful exceptions that we can show to the client;
+            throw new RuntimeException(gitAPIException);
+        }
+    }
+
+    private EntityVersion checkVersion(TenantId tenantId, String branch, String versionId) throws Exception {
+        return listVersions(tenantId, branch, null).stream()
+                .filter(version -> version.getId().equals(versionId))
+                .findFirst().orElseThrow(() -> new IllegalArgumentException("Version not found"));
+    }
+
+    private GitRepository checkRepository(TenantId tenantId) {
+        return Optional.ofNullable(repositories.get(tenantId))
+                .orElseThrow(() -> new IllegalStateException("Repository is not initialized"));
+    }
+
+    @Override
+    public List<EntityVersion> listVersions(TenantId tenantId, String branch, String path) throws Exception {
+        GitRepository repository = checkRepository(tenantId);
+        return repository.listCommits(branch, path, Integer.MAX_VALUE).stream()
+                .map(this::toVersion)
+                .collect(Collectors.toList());
+    }
+
+    @Override
+    public List<VersionedEntityInfo> listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, String path) throws Exception {
+        GitRepository repository = checkRepository(tenantId);
+        checkVersion(tenantId, branch, versionId);
+        return repository.listFilesAtCommit(versionId, path).stream()
+                .map(filePath -> {
+                    EntityId entityId = fromRelativePath(filePath);
+                    VersionedEntityInfo info = new VersionedEntityInfo();
+                    info.setExternalId(entityId);
+                    return info;
+                })
+                .collect(Collectors.toList());
+    }
+
+    @Override
+    public void initRepository(TenantId tenantId, EntitiesVersionControlSettings settings) throws Exception {
+        Path repositoryDirectory = Path.of(repositoriesFolder, tenantId.getId().toString());
+        GitRepository repository;
+        if (Files.exists(repositoryDirectory)) {
+            FileUtils.forceDelete(repositoryDirectory.toFile());
+        }
+
+        Files.createDirectories(repositoryDirectory);
+        repository = GitRepository.clone(settings.getRepositoryUri(), settings.getUsername(), settings.getPassword(), repositoryDirectory.toFile());
+        repositories.put(tenantId, repository);
+    }
+
+    private void clearRepository(TenantId tenantId) throws IOException {
+        GitRepository repository = repositories.get(tenantId);
+        if (repository != null) {
+            FileUtils.deleteDirectory(new File(repository.getDirectory()));
+            repositories.remove(tenantId);
+        }
+    }
+
+
+    private EntityVersion toVersion(GitRepository.Commit commit) {
+        return new EntityVersion(commit.getId(), commit.getMessage());
+    }
+
+    private EntityId fromRelativePath(String path) {
+        EntityType entityType = EntityType.valueOf(StringUtils.substringBefore(path, "/").toUpperCase());
+        String entityId = StringUtils.substringBetween(path, "/", ".json");
+        return EntityIdFactory.getByTypeAndUuid(entityType, entityId);
+    }
+}

common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java

@@ -13,7 +13,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.thingsboard.server.utils;
+package org.thingsboard.server.service.sync.vc;
 
 import com.google.common.collect.Streams;
 import lombok.Data;

common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java

@@ -0,0 +1,48 @@
+/**
+ * Copyright © 2016-2022 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.service.sync.vc;
+
+import org.thingsboard.server.common.data.id.TenantId;
+import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings;
+import org.thingsboard.server.common.data.sync.vc.EntityVersion;
+import org.thingsboard.server.common.data.sync.vc.VersionCreationResult;
+import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo;
+
+import java.io.IOException;
+import java.util.List;
+
+public interface GitRepositoryService {
+
+    void prepareCommit(PendingCommit pendingCommit);
+
+    List<EntityVersion> listVersions(TenantId tenantId, String branch, String path) throws Exception;
+
+    List<VersionedEntityInfo> listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, String path) throws Exception;
+
+    void initRepository(TenantId tenantId, EntitiesVersionControlSettings settings) throws Exception;
+
+    void add(PendingCommit commit, String relativePath, String entityDataJson) throws IOException;
+
+    void deleteFolderContent(PendingCommit commit, String relativePath) throws IOException;
+
+    VersionCreationResult push(PendingCommit commit);
+
+    void abort(PendingCommit commit);
+
+    List<String> listBranches(TenantId tenantId);
+
+    String getFileContentAtCommit(TenantId tenantId, String relativePath, String versionId) throws IOException;
+}

common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java

@@ -0,0 +1,60 @@
+/**
+ * Copyright © 2016-2022 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.service.sync.vc;
+
+import lombok.SneakyThrows;
+import org.thingsboard.server.common.data.EntityType;
+import org.thingsboard.server.common.data.ExportableEntity;
+import org.thingsboard.server.common.data.id.EntityId;
+import org.thingsboard.server.common.data.id.TenantId;
+import org.thingsboard.server.common.data.sync.ie.EntityExportData;
+import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings;
+import org.thingsboard.server.common.data.sync.vc.EntityVersion;
+import org.thingsboard.server.common.data.sync.vc.VersionCreationResult;
+import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo;
+import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest;
+
+import java.util.List;
+
+public interface GitVersionControlService {
+
+    @SneakyThrows
+    EntitiesVersionControlSettings getSettings(TenantId tenantId);
+
+    void initRepository(TenantId tenantId, EntitiesVersionControlSettings settings);
+
+    PendingCommit prepareCommit(TenantId tenantId, VersionCreateRequest request);
+
+    void addToCommit(PendingCommit commit, EntityExportData<ExportableEntity<EntityId>> entityData);
+
+    void deleteAll(PendingCommit pendingCommit, EntityType entityType);
+
+    VersionCreationResult push(PendingCommit commit);
+
+    List<EntityVersion> listVersions(TenantId tenantId, String branch);
+
+    List<EntityVersion> listVersions(TenantId tenantId, String branch, EntityType entityType);
+
+    List<EntityVersion> listVersions(TenantId tenantId, String branch, EntityId entityId);
+
+    List<VersionedEntityInfo> listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType);
+
+    List<VersionedEntityInfo> listEntitiesAtVersion(TenantId tenantId, String branch, String versionId);
+
+    List<String> listBranches(TenantId tenantId);
+
+    EntityExportData<?> getEntity(TenantId tenantId, String versionId, EntityId entityId);
+}

common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/PendingCommit.java

@@ -0,0 +1,36 @@
+/**
+ * Copyright © 2016-2022 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.service.sync.vc;
+
+import lombok.Data;
+import org.thingsboard.server.common.data.id.TenantId;
+import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest;
+
+import java.util.UUID;
+
+@Data
+public class PendingCommit {
+
+    private final UUID txId;
+    private final TenantId tenantId;
+    private final VersionCreateRequest request;
+
+    public PendingCommit(TenantId tenantId, VersionCreateRequest request) {
+        this.txId = UUID.randomUUID();
+        this.tenantId = tenantId;
+        this.request = request;
+    }
+}