API for listing entities at version, loading all entities at version, listing all available versions

application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java

@@ -29,15 +29,18 @@ import org.thingsboard.server.common.data.ExportableEntity;
 import org.thingsboard.server.common.data.exception.ThingsboardException;
 import org.thingsboard.server.common.data.id.EntityId;
 import org.thingsboard.server.common.data.id.EntityIdFactory;
+import org.thingsboard.server.service.security.model.SecurityUser;
 import org.thingsboard.server.service.sync.exporting.data.EntityExportData;
 import org.thingsboard.server.service.sync.importing.EntityImportResult;
 import org.thingsboard.server.service.sync.vcs.DefaultEntitiesVersionControlService;
 import org.thingsboard.server.service.sync.vcs.data.EntitiesVersionControlSettings;
 import org.thingsboard.server.service.sync.vcs.data.EntityVersion;
 
+import java.util.Arrays;
 import java.util.List;
 import java.util.Set;
 import java.util.UUID;
+import java.util.stream.Collectors;
 
 @RestController
 @RequestMapping("/api/entities/vc")
@@ -51,13 +54,27 @@ public class EntitiesVersionControlController extends BaseController {
     @PostMapping("/version/{entityType}/{entityId}")
     @PreAuthorize("hasAuthority('TENANT_ADMIN')")
     public EntityVersion saveEntityVersion(@PathVariable EntityType entityType,
-                                           @PathVariable("entityId") UUID entityUuid,
+                                           @PathVariable("entityId") UUID id,
                                            @RequestParam String branch,
                                            @RequestBody String versionName) throws Exception {
-        EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityUuid);
+        EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, id);
         return versionControlService.saveEntityVersion(getTenantId(), entityId, branch, versionName);
     }
 
+    @PostMapping("/version/{entityType}")
+    @PreAuthorize("hasAuthority('TENANT_ADMIN')")
+    public EntityVersion saveEntitiesVersion(@PathVariable EntityType entityType,
+                                             @RequestParam UUID[] ids,
+                                             @RequestParam String branch,
+                                             @RequestBody String versionName) throws Exception {
+        List<EntityId> entitiesIds = Arrays.stream(ids)
+                .map(id -> EntityIdFactory.getByTypeAndUuid(entityType, id))
+                .collect(Collectors.toList());
+        return versionControlService.saveEntitiesVersion(getTenantId(), entitiesIds, branch, versionName);
+    }
+
+
+
     @GetMapping("/version/{entityType}/{entityId}")
     @PreAuthorize("hasAuthority('TENANT_ADMIN')")
     public List<EntityVersion> listEntityVersions(@PathVariable EntityType entityType,
@@ -67,29 +84,68 @@ public class EntitiesVersionControlController extends BaseController {
         return versionControlService.listEntityVersions(getTenantId(), entityId, branch, Integer.MAX_VALUE);
     }
 
+    @GetMapping("/version/{entityType}")
+    @PreAuthorize("hasAuthority('TENANT_ADMIN')")
+    public List<EntityVersion> listEntityTypeVersions(@PathVariable EntityType entityType,
+                                                      @RequestParam String branch) throws Exception {
+        return versionControlService.listEntityTypeVersions(getTenantId(), entityType, branch, Integer.MAX_VALUE);
+    }
+
+    @GetMapping("/version")
+    @PreAuthorize("hasAuthority('TENANT_ADMIN')")
+    public List<EntityVersion> listVersions(@RequestParam String branch) throws Exception {
+        return versionControlService.listVersions(getTenantId(), branch, Integer.MAX_VALUE);
+    }
+
+
+
+    @GetMapping("/files/version/{versionId}")
+    @PreAuthorize("hasAuthority('TENANT_ADMIN')")
+    public List<String> listFilesAtVersion(@RequestParam String branch,
+                                           @PathVariable String versionId) throws Exception {
+        return versionControlService.listFilesAtVersion(getTenantId(), branch, versionId);
+    }
+
 
 
     @GetMapping("/entity/{entityType}/{entityId}/{versionId}")
     @PreAuthorize("hasAuthority('TENANT_ADMIN')")
     public EntityExportData<ExportableEntity<EntityId>> getEntityAtVersion(@PathVariable EntityType entityType,
                                                                            @PathVariable("entityId") UUID entityUuid,
+                                                                           @RequestParam String branch,
                                                                            @PathVariable String versionId) throws Exception {
         EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityUuid);
-        return versionControlService.getEntityAtVersion(getTenantId(), entityId, versionId);
+        return versionControlService.getEntityAtVersion(getTenantId(), entityId, branch, versionId);
     }
 
     @PostMapping("/entity/{entityType}/{entityId}/{versionId}")
     @PreAuthorize("hasAuthority('TENANT_ADMIN')")
     public EntityImportResult<ExportableEntity<EntityId>> loadEntityVersion(@PathVariable EntityType entityType,
                                                                             @PathVariable("entityId") UUID entityUuid,
+                                                                            @RequestParam String branch,
                                                                             @PathVariable String versionId) throws Exception {
         EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityUuid);
-        return versionControlService.loadEntityVersion(getTenantId(), entityId, versionId);
+        EntityImportResult<ExportableEntity<EntityId>> result = versionControlService.loadEntityVersion(getTenantId(), entityId, branch, versionId);
+        onEntityUpdatedOrCreated(getCurrentUser(), result.getSavedEntity(), result.getOldEntity(), result.getOldEntity() == null);
+        return result;
+    }
+
+    @PostMapping("/entity/{versionId}")
+    @PreAuthorize("hasAuthority('TENANT_ADMIN')")
+    public List<EntityImportResult<ExportableEntity<EntityId>>> loadAllAtVersion(@RequestParam String branch,
+                                                                                 @PathVariable String versionId) throws Exception {
+        SecurityUser user = getCurrentUser();
+        List<EntityImportResult<ExportableEntity<EntityId>>> resultList = versionControlService.loadAllAtVersion(user.getTenantId(), branch, versionId);
+        resultList.forEach(result -> {
+            onEntityUpdatedOrCreated(user, result.getSavedEntity(), result.getOldEntity(), result.getOldEntity() == null);
+        });
+        return resultList;
     }
 
 
 
     @GetMapping("/branches")
+    @PreAuthorize("hasAuthority('TENANT_ADMIN')")
     public Set<String> getAllowedBranches() throws ThingsboardException {
         return versionControlService.getAllowedBranches(getTenantId());
     }

application/src/main/java/org/thingsboard/server/service/sync/vcs/DefaultEntitiesVersionControlService.java

@@ -22,6 +22,7 @@ import com.fasterxml.jackson.databind.SerializationFeature;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.io.FileUtils;
+import org.eclipse.jgit.api.errors.GitAPIException;
 import org.springframework.scheduling.annotation.Scheduled;
 import org.springframework.stereotype.Service;
 import org.thingsboard.common.util.JacksonUtil;
@@ -46,6 +47,8 @@ import org.thingsboard.server.utils.git.Repository;
 import org.thingsboard.server.utils.git.data.Commit;
 
 import java.io.File;
+import java.io.IOException;
+import java.io.UncheckedIOException;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -54,7 +57,9 @@ import java.util.List;
 import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReadWriteLock;
 import java.util.concurrent.locks.ReentrantLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
 import java.util.stream.Collectors;
 
 @Service
@@ -72,8 +77,8 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
     private static final String SETTINGS_KEY = "vc";
 
     private Repository repository;
-    private final ReentrantLock fetchLock = new ReentrantLock();
+    private final Lock fetchLock = new ReentrantLock();
-    private final Lock writeLock = new ReentrantLock();
+    private final ReadWriteLock repositoryLock = new ReentrantReadWriteLock();
 
     @AfterStartUp
     public void init() throws Exception {
@@ -89,17 +94,9 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
 
 
     @Scheduled(initialDelay = 10 * 1000, fixedDelay = 10 * 1000)
-    public void fetch() throws Exception {
+    private void fetch() throws Exception {
         if (repository == null) return;
-
+        tryFetch();
-        if (fetchLock.tryLock()) {
-            try {
-                log.info("Fetching remote repository");
-                repository.fetch();
-            } finally {
-                fetchLock.unlock();
-            }
-        }
     }
 
 
@@ -118,20 +115,12 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
                 .exportOutboundRelations(false)
                 .build();
         List<EntityExportData<ExportableEntity<EntityId>>> entityDataList = entitiesIds.stream()
-                .map(entityId -> {
+                .map(entityId -> exportImportService.exportEntity(tenantId, entityId, exportSettings))
-                    return exportImportService.exportEntity(tenantId, entityId, exportSettings);
-                })
                 .collect(Collectors.toList());
 
-        if (fetchLock.tryLock()) {
+        tryFetch();
-            try {
-                repository.fetch();
-            } finally {
-                fetchLock.unlock();
-            }
-        }
 
-        writeLock.lock();
+        repositoryLock.writeLock().lock();
         try {
             if (repository.listBranches().contains(branch)) {
                 repository.checkout(branch);
@@ -148,9 +137,9 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
 
             Commit commit = repository.commit(versionName, ".", "Tenant " + tenantId);
             repository.push();
-            return new EntityVersion(commit.getId(), commit.getMessage(), commit.getAuthorName());
+            return toVersion(commit);
         } finally {
-            writeLock.unlock();
+            repositoryLock.writeLock().unlock();
         }
     }
 
@@ -158,38 +147,72 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
 
     @Override
     public List<EntityVersion> listEntityVersions(TenantId tenantId, EntityId entityId, String branch, int limit) throws Exception {
-        checkRepository();
+        return listVersions(tenantId, branch, getRelativePathForEntity(entityId), limit);
-        checkBranch(tenantId, branch);
-
-        return repository.listCommits(branch, getRelativePathForEntity(entityId), limit).stream()
-                .map(commit -> new EntityVersion(commit.getId(), commit.getMessage(), commit.getAuthorName()))
-                .collect(Collectors.toList());
     }
 
     @Override
     public List<EntityVersion> listEntityTypeVersions(TenantId tenantId, EntityType entityType, String branch, int limit) throws Exception {
-        checkRepository();
+        return listVersions(tenantId, getRelativePathForEntityType(entityType), limit);
-        checkBranch(tenantId, branch);
+    }
 
-        return repository.listCommits(branch, getRelativePathForEntityType(entityType), limit).stream()
+    @Override
-                .map(commit -> new EntityVersion(commit.getId(), commit.getMessage(), commit.getAuthorName()))
+    public List<EntityVersion> listVersions(TenantId tenantId, String branch, int limit) throws Exception {
-                .collect(Collectors.toList());
+        return listVersions(tenantId, branch, null, limit);
+    }
+
+    private List<EntityVersion> listVersions(TenantId tenantId, String branch, String path, int limit) throws Exception {
+        repositoryLock.readLock().lock();
+        try {
+            checkRepository();
+            checkBranch(tenantId, branch);
+
+            return repository.listCommits(branch, path, limit).stream()
+                    .map(this::toVersion)
+                    .collect(Collectors.toList());
+
+        } finally {
+            repositoryLock.readLock().unlock();
+        }
     }
 
 
 
     @Override
-    public <E extends ExportableEntity<I>, I extends EntityId> EntityExportData<E> getEntityAtVersion(TenantId tenantId, I entityId, String versionId) throws Exception {
+    public List<String> listFilesAtVersion(TenantId tenantId, String branch, String versionId) throws Exception {
-        checkRepository();
+        repositoryLock.readLock().lock();
-        // FIXME [viacheslav]: validate access
+        try {
-
+            if (listVersions(tenantId, branch, Integer.MAX_VALUE).stream()
-        String entityDataJson = repository.getFileContentAtCommit(getRelativePathForEntity(entityId), versionId);
+                    .noneMatch(version -> version.getId().equals(versionId))) {
-        return JacksonUtil.fromString(entityDataJson, new TypeReference<EntityExportData<E>>() {});
+                throw new IllegalArgumentException("Unknown version");
+            }
+            return repository.listFilesAtCommit(versionId);
+        } finally {
+            repositoryLock.readLock().unlock();
+        }
     }
 
+
+
     @Override
-    public <E extends ExportableEntity<I>, I extends EntityId> EntityImportResult<E> loadEntityVersion(TenantId tenantId, I entityId, String versionId) throws Exception {
+    public <E extends ExportableEntity<I>, I extends EntityId> EntityExportData<E> getEntityAtVersion(TenantId tenantId, I entityId, String branch, String versionId) throws Exception {
-        EntityExportData<E> entityData = getEntityAtVersion(tenantId, entityId, versionId);
+        repositoryLock.readLock().lock();
+        try {
+            if (listEntityVersions(tenantId, entityId, branch, Integer.MAX_VALUE).stream()
+                    .noneMatch(version -> version.getId().equals(versionId))) {
+                throw new IllegalArgumentException("Unknown version");
+            }
+
+            String entityDataJson = repository.getFileContentAtCommit(getRelativePathForEntity(entityId), versionId);
+            return parseEntityData(entityDataJson);
+        } finally {
+            repositoryLock.readLock().unlock();
+        }
+    }
+
+
+    @Override
+    public <E extends ExportableEntity<I>, I extends EntityId> EntityImportResult<E> loadEntityVersion(TenantId tenantId, I entityId, String branch, String versionId) throws Exception {
+        EntityExportData<E> entityData = getEntityAtVersion(tenantId, entityId, branch, versionId);
         return exportImportService.importEntity(tenantId, entityData, EntityImportSettings.builder()
                 .importInboundRelations(false)
                 .importOutboundRelations(false)
@@ -197,6 +220,47 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
                 .build());
     }
 
+    @Override
+    public List<EntityImportResult<ExportableEntity<EntityId>>> loadAllAtVersion(TenantId tenantId, String branch, String versionId) throws Exception {
+        repositoryLock.readLock().lock();
+        try {
+            List<EntityExportData<ExportableEntity<EntityId>>> entityDataList = listFilesAtVersion(tenantId, branch, versionId).stream()
+                    .map(entityDataFilePath -> {
+                        String entityDataJson;
+                        try {
+                            entityDataJson = repository.getFileContentAtCommit(entityDataFilePath, versionId);
+                        } catch (IOException e) {
+                            throw new UncheckedIOException(e);
+                        }
+                        return parseEntityData(entityDataJson);
+                    })
+                    .collect(Collectors.toList());
+
+            return exportImportService.importEntities(tenantId, entityDataList, EntityImportSettings.builder()
+                    .importInboundRelations(false)
+                    .importOutboundRelations(false)
+                    .updateReferencesToOtherEntities(true)
+                    .build());
+        } finally {
+            repositoryLock.readLock().unlock();
+        }
+    }
+
+    private void tryFetch() throws GitAPIException {
+        repositoryLock.readLock().lock();
+        try {
+            if (fetchLock.tryLock()) {
+                try {
+                    log.info("Fetching remote repository");
+                    repository.fetch();
+                } finally {
+                    fetchLock.unlock();
+                }
+            }
+        } finally {
+            repositoryLock.readLock().unlock();
+        }
+    }
 
 
     private String getRelativePathForEntity(EntityId entityId) {
@@ -210,6 +274,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
 
 
     private void checkBranch(TenantId tenantId, String branch) {
+        // TODO [viacheslav]: all branches are available by default?
         if (!getAllowedBranches(tenantId).contains(branch)) {
             throw new IllegalArgumentException("Tenant does not have access to this branch");
         }
@@ -222,9 +287,24 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
                 .orElse(Collections.emptySet());
     }
 
+    private EntityVersion toVersion(Commit commit) {
+        return new EntityVersion(commit.getId(), commit.getMessage(), commit.getAuthorName());
+    }
+
+    private <E extends ExportableEntity<I>, I extends EntityId> EntityExportData<E> parseEntityData(String entityDataJson) {
+        return JacksonUtil.fromString(entityDataJson, new TypeReference<EntityExportData<E>>() {});
+    }
+
+
+
     @Override
     public void saveSettings(EntitiesVersionControlSettings settings) throws Exception {
-        this.repository = initRepository(settings.getGitSettings());
+        repositoryLock.writeLock().lock();
+        try {
+            this.repository = initRepository(settings.getGitSettings());
+        } finally {
+            repositoryLock.writeLock().unlock();
+        }
 
         AdminSettings adminSettings = Optional.ofNullable(adminSettingsService.findAdminSettingsByKey(TenantId.SYS_TENANT_ID, SETTINGS_KEY))
                 .orElseGet(() -> {
@@ -244,7 +324,6 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
     }
 
 
-
     private void checkRepository() {
         if (repository == null) {
             throw new IllegalStateException("Repository is not initialized");
@@ -263,12 +342,17 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
     }
 
     public void resetRepository() throws Exception {
-        if (this.repository != null) {
+        repositoryLock.writeLock().lock();
-            FileUtils.deleteDirectory(new File(repository.getDirectory()));
+        try {
-            this.repository = null;
+            if (this.repository != null) {
+                FileUtils.deleteDirectory(new File(repository.getDirectory()));
+                this.repository = null;
+            }
+            EntitiesVersionControlSettings settings = getSettings();
+            this.repository = initRepository(settings.getGitSettings());
+        } finally {
+            repositoryLock.writeLock().unlock();
         }
-        EntitiesVersionControlSettings settings = getSettings();
-        this.repository = initRepository(settings.getGitSettings());
     }
 
 }

application/src/main/java/org/thingsboard/server/service/sync/vcs/EntitiesVersionControlService.java

@@ -37,10 +37,17 @@ public interface EntitiesVersionControlService {
 
     List<EntityVersion> listEntityTypeVersions(TenantId tenantId, EntityType entityType, String branch, int limit) throws Exception;
 
+    List<EntityVersion> listVersions(TenantId tenantId, String branch, int limit) throws Exception;
 
-    <E extends ExportableEntity<I>, I extends EntityId> EntityExportData<E> getEntityAtVersion(TenantId tenantId, I entityId, String versionId) throws Exception;
 
-    <E extends ExportableEntity<I>, I extends EntityId> EntityImportResult<E> loadEntityVersion(TenantId tenantId, I entityId, String versionId) throws Exception;
+    List<String> listFilesAtVersion(TenantId tenantId, String branch, String versionId) throws Exception;
+
+
+    <E extends ExportableEntity<I>, I extends EntityId> EntityExportData<E> getEntityAtVersion(TenantId tenantId, I entityId, String branch, String versionId) throws Exception;
+
+    <E extends ExportableEntity<I>, I extends EntityId> EntityImportResult<E> loadEntityVersion(TenantId tenantId, I entityId, String branch, String versionId) throws Exception;
+
+    List<EntityImportResult<ExportableEntity<EntityId>>> loadAllAtVersion(TenantId tenantId, String branch, String versionId) throws Exception;
 
 
     void saveSettings(EntitiesVersionControlSettings settings) throws Exception;

application/src/main/java/org/thingsboard/server/utils/git/Repository.java

@@ -113,13 +113,13 @@ public class Repository {
     }
 
 
-    public List<String> listFilesAtCommit(Commit commit) throws IOException {
+    public List<String> listFilesAtCommit(String commitId) throws IOException {
-        return listFilesAtCommit(commit, null);
+        return listFilesAtCommit(commitId, null);
     }
 
-    public List<String> listFilesAtCommit(Commit commit, String path) throws IOException {
+    public List<String> listFilesAtCommit(String commitId, String path) throws IOException {
         List<String> files = new ArrayList<>();
-        RevCommit revCommit = resolveCommit(commit.getId());
+        RevCommit revCommit = resolveCommit(commitId);
         try (TreeWalk treeWalk = new TreeWalk(git.getRepository())) {
             treeWalk.reset(revCommit.getTree().getId());
             if (StringUtils.isNotEmpty(path)) {