API for listing entities at version, loading all entities at version, listing all available versions
This commit is contained in:
Viacheslav Klimov
parent
f82be0153b
commit
b3dfed5bad
@ -29,15 +29,18 @@ import org.thingsboard.server.common.data.ExportableEntity;
|
||||
import org.thingsboard.server.common.data.exception.ThingsboardException;
|
||||
import org.thingsboard.server.common.data.id.EntityId;
|
||||
import org.thingsboard.server.common.data.id.EntityIdFactory;
|
||||
import org.thingsboard.server.service.security.model.SecurityUser;
|
||||
import org.thingsboard.server.service.sync.exporting.data.EntityExportData;
|
||||
import org.thingsboard.server.service.sync.importing.EntityImportResult;
|
||||
import org.thingsboard.server.service.sync.vcs.DefaultEntitiesVersionControlService;
|
||||
import org.thingsboard.server.service.sync.vcs.data.EntitiesVersionControlSettings;
|
||||
import org.thingsboard.server.service.sync.vcs.data.EntityVersion;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.UUID;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@RestController
|
||||
@RequestMapping("/api/entities/vc")
|
||||
@ -51,13 +54,27 @@ public class EntitiesVersionControlController extends BaseController {
|
||||
@PostMapping("/version/{entityType}/{entityId}")
|
||||
@PreAuthorize("hasAuthority('TENANT_ADMIN')")
|
||||
public EntityVersion saveEntityVersion(@PathVariable EntityType entityType,
|
||||
@PathVariable("entityId") UUID entityUuid,
|
||||
@PathVariable("entityId") UUID id,
|
||||
@RequestParam String branch,
|
||||
@RequestBody String versionName) throws Exception {
|
||||
EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityUuid);
|
||||
EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, id);
|
||||
return versionControlService.saveEntityVersion(getTenantId(), entityId, branch, versionName);
|
||||
}
|
||||
|
||||
@PostMapping("/version/{entityType}")
|
||||
@PreAuthorize("hasAuthority('TENANT_ADMIN')")
|
||||
public EntityVersion saveEntitiesVersion(@PathVariable EntityType entityType,
|
||||
@RequestParam UUID[] ids,
|
||||
@RequestParam String branch,
|
||||
@RequestBody String versionName) throws Exception {
|
||||
List<EntityId> entitiesIds = Arrays.stream(ids)
|
||||
.map(id -> EntityIdFactory.getByTypeAndUuid(entityType, id))
|
||||
.collect(Collectors.toList());
|
||||
return versionControlService.saveEntitiesVersion(getTenantId(), entitiesIds, branch, versionName);
|
||||
}
|
||||
|
||||
|
||||
|
||||
@GetMapping("/version/{entityType}/{entityId}")
|
||||
@PreAuthorize("hasAuthority('TENANT_ADMIN')")
|
||||
public List<EntityVersion> listEntityVersions(@PathVariable EntityType entityType,
|
||||
@ -67,29 +84,68 @@ public class EntitiesVersionControlController extends BaseController {
|
||||
return versionControlService.listEntityVersions(getTenantId(), entityId, branch, Integer.MAX_VALUE);
|
||||
}
|
||||
|
||||
@GetMapping("/version/{entityType}")
|
||||
@PreAuthorize("hasAuthority('TENANT_ADMIN')")
|
||||
public List<EntityVersion> listEntityTypeVersions(@PathVariable EntityType entityType,
|
||||
@RequestParam String branch) throws Exception {
|
||||
return versionControlService.listEntityTypeVersions(getTenantId(), entityType, branch, Integer.MAX_VALUE);
|
||||
}
|
||||
|
||||
@GetMapping("/version")
|
||||
@PreAuthorize("hasAuthority('TENANT_ADMIN')")
|
||||
public List<EntityVersion> listVersions(@RequestParam String branch) throws Exception {
|
||||
return versionControlService.listVersions(getTenantId(), branch, Integer.MAX_VALUE);
|
||||
}
|
||||
|
||||
|
||||
|
||||
@GetMapping("/files/version/{versionId}")
|
||||
@PreAuthorize("hasAuthority('TENANT_ADMIN')")
|
||||
public List<String> listFilesAtVersion(@RequestParam String branch,
|
||||
@PathVariable String versionId) throws Exception {
|
||||
return versionControlService.listFilesAtVersion(getTenantId(), branch, versionId);
|
||||
}
|
||||
|
||||
|
||||
|
||||
@GetMapping("/entity/{entityType}/{entityId}/{versionId}")
|
||||
@PreAuthorize("hasAuthority('TENANT_ADMIN')")
|
||||
public EntityExportData<ExportableEntity<EntityId>> getEntityAtVersion(@PathVariable EntityType entityType,
|
||||
@PathVariable("entityId") UUID entityUuid,
|
||||
@RequestParam String branch,
|
||||
@PathVariable String versionId) throws Exception {
|
||||
EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityUuid);
|
||||
return versionControlService.getEntityAtVersion(getTenantId(), entityId, versionId);
|
||||
return versionControlService.getEntityAtVersion(getTenantId(), entityId, branch, versionId);
|
||||
}
|
||||
|
||||
@PostMapping("/entity/{entityType}/{entityId}/{versionId}")
|
||||
@PreAuthorize("hasAuthority('TENANT_ADMIN')")
|
||||
public EntityImportResult<ExportableEntity<EntityId>> loadEntityVersion(@PathVariable EntityType entityType,
|
||||
@PathVariable("entityId") UUID entityUuid,
|
||||
@RequestParam String branch,
|
||||
@PathVariable String versionId) throws Exception {
|
||||
EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityUuid);
|
||||
return versionControlService.loadEntityVersion(getTenantId(), entityId, versionId);
|
||||
EntityImportResult<ExportableEntity<EntityId>> result = versionControlService.loadEntityVersion(getTenantId(), entityId, branch, versionId);
|
||||
onEntityUpdatedOrCreated(getCurrentUser(), result.getSavedEntity(), result.getOldEntity(), result.getOldEntity() == null);
|
||||
return result;
|
||||
}
|
||||
|
||||
@PostMapping("/entity/{versionId}")
|
||||
@PreAuthorize("hasAuthority('TENANT_ADMIN')")
|
||||
public List<EntityImportResult<ExportableEntity<EntityId>>> loadAllAtVersion(@RequestParam String branch,
|
||||
@PathVariable String versionId) throws Exception {
|
||||
SecurityUser user = getCurrentUser();
|
||||
List<EntityImportResult<ExportableEntity<EntityId>>> resultList = versionControlService.loadAllAtVersion(user.getTenantId(), branch, versionId);
|
||||
resultList.forEach(result -> {
|
||||
onEntityUpdatedOrCreated(user, result.getSavedEntity(), result.getOldEntity(), result.getOldEntity() == null);
|
||||
});
|
||||
return resultList;
|
||||
}
|
||||
|
||||
|
||||
|
||||
@GetMapping("/branches")
|
||||
@PreAuthorize("hasAuthority('TENANT_ADMIN')")
|
||||
public Set<String> getAllowedBranches() throws ThingsboardException {
|
||||
return versionControlService.getAllowedBranches(getTenantId());
|
||||
}
|
||||
|
||||
@ -22,6 +22,7 @@ import com.fasterxml.jackson.databind.SerializationFeature;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.eclipse.jgit.api.errors.GitAPIException;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.thingsboard.common.util.JacksonUtil;
|
||||
@ -46,6 +47,8 @@ import org.thingsboard.server.utils.git.Repository;
|
||||
import org.thingsboard.server.utils.git.data.Commit;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.UncheckedIOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
@ -54,7 +57,9 @@ import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.locks.Lock;
|
||||
import java.util.concurrent.locks.ReadWriteLock;
|
||||
import java.util.concurrent.locks.ReentrantLock;
|
||||
import java.util.concurrent.locks.ReentrantReadWriteLock;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Service
|
||||
@ -72,8 +77,8 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
|
||||
private static final String SETTINGS_KEY = "vc";
|
||||
|
||||
private Repository repository;
|
||||
private final ReentrantLock fetchLock = new ReentrantLock();
|
||||
private final Lock writeLock = new ReentrantLock();
|
||||
private final Lock fetchLock = new ReentrantLock();
|
||||
private final ReadWriteLock repositoryLock = new ReentrantReadWriteLock();
|
||||
|
||||
@AfterStartUp
|
||||
public void init() throws Exception {
|
||||
@ -89,17 +94,9 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
|
||||
|
||||
|
||||
@Scheduled(initialDelay = 10 * 1000, fixedDelay = 10 * 1000)
|
||||
public void fetch() throws Exception {
|
||||
private void fetch() throws Exception {
|
||||
if (repository == null) return;
|
||||
|
||||
if (fetchLock.tryLock()) {
|
||||
try {
|
||||
log.info("Fetching remote repository");
|
||||
repository.fetch();
|
||||
} finally {
|
||||
fetchLock.unlock();
|
||||
}
|
||||
}
|
||||
tryFetch();
|
||||
}
|
||||
|
||||
|
||||
@ -118,20 +115,12 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
|
||||
.exportOutboundRelations(false)
|
||||
.build();
|
||||
List<EntityExportData<ExportableEntity<EntityId>>> entityDataList = entitiesIds.stream()
|
||||
.map(entityId -> {
|
||||
return exportImportService.exportEntity(tenantId, entityId, exportSettings);
|
||||
})
|
||||
.map(entityId -> exportImportService.exportEntity(tenantId, entityId, exportSettings))
|
||||
.collect(Collectors.toList());
|
||||
|
||||
if (fetchLock.tryLock()) {
|
||||
try {
|
||||
repository.fetch();
|
||||
} finally {
|
||||
fetchLock.unlock();
|
||||
}
|
||||
}
|
||||
tryFetch();
|
||||
|
||||
writeLock.lock();
|
||||
repositoryLock.writeLock().lock();
|
||||
try {
|
||||
if (repository.listBranches().contains(branch)) {
|
||||
repository.checkout(branch);
|
||||
@ -148,9 +137,9 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
|
||||
|
||||
Commit commit = repository.commit(versionName, ".", "Tenant " + tenantId);
|
||||
repository.push();
|
||||
return new EntityVersion(commit.getId(), commit.getMessage(), commit.getAuthorName());
|
||||
return toVersion(commit);
|
||||
} finally {
|
||||
writeLock.unlock();
|
||||
repositoryLock.writeLock().unlock();
|
||||
}
|
||||
}
|
||||
|
||||
@ -158,38 +147,72 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
|
||||
|
||||
@Override
|
||||
public List<EntityVersion> listEntityVersions(TenantId tenantId, EntityId entityId, String branch, int limit) throws Exception {
|
||||
checkRepository();
|
||||
checkBranch(tenantId, branch);
|
||||
|
||||
return repository.listCommits(branch, getRelativePathForEntity(entityId), limit).stream()
|
||||
.map(commit -> new EntityVersion(commit.getId(), commit.getMessage(), commit.getAuthorName()))
|
||||
.collect(Collectors.toList());
|
||||
return listVersions(tenantId, branch, getRelativePathForEntity(entityId), limit);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<EntityVersion> listEntityTypeVersions(TenantId tenantId, EntityType entityType, String branch, int limit) throws Exception {
|
||||
checkRepository();
|
||||
checkBranch(tenantId, branch);
|
||||
return listVersions(tenantId, getRelativePathForEntityType(entityType), limit);
|
||||
}
|
||||
|
||||
return repository.listCommits(branch, getRelativePathForEntityType(entityType), limit).stream()
|
||||
.map(commit -> new EntityVersion(commit.getId(), commit.getMessage(), commit.getAuthorName()))
|
||||
.collect(Collectors.toList());
|
||||
@Override
|
||||
public List<EntityVersion> listVersions(TenantId tenantId, String branch, int limit) throws Exception {
|
||||
return listVersions(tenantId, branch, null, limit);
|
||||
}
|
||||
|
||||
private List<EntityVersion> listVersions(TenantId tenantId, String branch, String path, int limit) throws Exception {
|
||||
repositoryLock.readLock().lock();
|
||||
try {
|
||||
checkRepository();
|
||||
checkBranch(tenantId, branch);
|
||||
|
||||
return repository.listCommits(branch, path, limit).stream()
|
||||
.map(this::toVersion)
|
||||
.collect(Collectors.toList());
|
||||
|
||||
} finally {
|
||||
repositoryLock.readLock().unlock();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Override
|
||||
public <E extends ExportableEntity<I>, I extends EntityId> EntityExportData<E> getEntityAtVersion(TenantId tenantId, I entityId, String versionId) throws Exception {
|
||||
checkRepository();
|
||||
// FIXME [viacheslav]: validate access
|
||||
|
||||
String entityDataJson = repository.getFileContentAtCommit(getRelativePathForEntity(entityId), versionId);
|
||||
return JacksonUtil.fromString(entityDataJson, new TypeReference<EntityExportData<E>>() {});
|
||||
public List<String> listFilesAtVersion(TenantId tenantId, String branch, String versionId) throws Exception {
|
||||
repositoryLock.readLock().lock();
|
||||
try {
|
||||
if (listVersions(tenantId, branch, Integer.MAX_VALUE).stream()
|
||||
.noneMatch(version -> version.getId().equals(versionId))) {
|
||||
throw new IllegalArgumentException("Unknown version");
|
||||
}
|
||||
return repository.listFilesAtCommit(versionId);
|
||||
} finally {
|
||||
repositoryLock.readLock().unlock();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Override
|
||||
public <E extends ExportableEntity<I>, I extends EntityId> EntityImportResult<E> loadEntityVersion(TenantId tenantId, I entityId, String versionId) throws Exception {
|
||||
EntityExportData<E> entityData = getEntityAtVersion(tenantId, entityId, versionId);
|
||||
public <E extends ExportableEntity<I>, I extends EntityId> EntityExportData<E> getEntityAtVersion(TenantId tenantId, I entityId, String branch, String versionId) throws Exception {
|
||||
repositoryLock.readLock().lock();
|
||||
try {
|
||||
if (listEntityVersions(tenantId, entityId, branch, Integer.MAX_VALUE).stream()
|
||||
.noneMatch(version -> version.getId().equals(versionId))) {
|
||||
throw new IllegalArgumentException("Unknown version");
|
||||
}
|
||||
|
||||
String entityDataJson = repository.getFileContentAtCommit(getRelativePathForEntity(entityId), versionId);
|
||||
return parseEntityData(entityDataJson);
|
||||
} finally {
|
||||
repositoryLock.readLock().unlock();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public <E extends ExportableEntity<I>, I extends EntityId> EntityImportResult<E> loadEntityVersion(TenantId tenantId, I entityId, String branch, String versionId) throws Exception {
|
||||
EntityExportData<E> entityData = getEntityAtVersion(tenantId, entityId, branch, versionId);
|
||||
return exportImportService.importEntity(tenantId, entityData, EntityImportSettings.builder()
|
||||
.importInboundRelations(false)
|
||||
.importOutboundRelations(false)
|
||||
@ -197,6 +220,47 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
|
||||
.build());
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<EntityImportResult<ExportableEntity<EntityId>>> loadAllAtVersion(TenantId tenantId, String branch, String versionId) throws Exception {
|
||||
repositoryLock.readLock().lock();
|
||||
try {
|
||||
List<EntityExportData<ExportableEntity<EntityId>>> entityDataList = listFilesAtVersion(tenantId, branch, versionId).stream()
|
||||
.map(entityDataFilePath -> {
|
||||
String entityDataJson;
|
||||
try {
|
||||
entityDataJson = repository.getFileContentAtCommit(entityDataFilePath, versionId);
|
||||
} catch (IOException e) {
|
||||
throw new UncheckedIOException(e);
|
||||
}
|
||||
return parseEntityData(entityDataJson);
|
||||
})
|
||||
.collect(Collectors.toList());
|
||||
|
||||
return exportImportService.importEntities(tenantId, entityDataList, EntityImportSettings.builder()
|
||||
.importInboundRelations(false)
|
||||
.importOutboundRelations(false)
|
||||
.updateReferencesToOtherEntities(true)
|
||||
.build());
|
||||
} finally {
|
||||
repositoryLock.readLock().unlock();
|
||||
}
|
||||
}
|
||||
|
||||
private void tryFetch() throws GitAPIException {
|
||||
repositoryLock.readLock().lock();
|
||||
try {
|
||||
if (fetchLock.tryLock()) {
|
||||
try {
|
||||
log.info("Fetching remote repository");
|
||||
repository.fetch();
|
||||
} finally {
|
||||
fetchLock.unlock();
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
repositoryLock.readLock().unlock();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private String getRelativePathForEntity(EntityId entityId) {
|
||||
@ -210,6 +274,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
|
||||
|
||||
|
||||
private void checkBranch(TenantId tenantId, String branch) {
|
||||
// TODO [viacheslav]: all branches are available by default?
|
||||
if (!getAllowedBranches(tenantId).contains(branch)) {
|
||||
throw new IllegalArgumentException("Tenant does not have access to this branch");
|
||||
}
|
||||
@ -222,9 +287,24 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
|
||||
.orElse(Collections.emptySet());
|
||||
}
|
||||
|
||||
private EntityVersion toVersion(Commit commit) {
|
||||
return new EntityVersion(commit.getId(), commit.getMessage(), commit.getAuthorName());
|
||||
}
|
||||
|
||||
private <E extends ExportableEntity<I>, I extends EntityId> EntityExportData<E> parseEntityData(String entityDataJson) {
|
||||
return JacksonUtil.fromString(entityDataJson, new TypeReference<EntityExportData<E>>() {});
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Override
|
||||
public void saveSettings(EntitiesVersionControlSettings settings) throws Exception {
|
||||
this.repository = initRepository(settings.getGitSettings());
|
||||
repositoryLock.writeLock().lock();
|
||||
try {
|
||||
this.repository = initRepository(settings.getGitSettings());
|
||||
} finally {
|
||||
repositoryLock.writeLock().unlock();
|
||||
}
|
||||
|
||||
AdminSettings adminSettings = Optional.ofNullable(adminSettingsService.findAdminSettingsByKey(TenantId.SYS_TENANT_ID, SETTINGS_KEY))
|
||||
.orElseGet(() -> {
|
||||
@ -244,7 +324,6 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
|
||||
}
|
||||
|
||||
|
||||
|
||||
private void checkRepository() {
|
||||
if (repository == null) {
|
||||
throw new IllegalStateException("Repository is not initialized");
|
||||
@ -263,12 +342,17 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
|
||||
}
|
||||
|
||||
public void resetRepository() throws Exception {
|
||||
if (this.repository != null) {
|
||||
FileUtils.deleteDirectory(new File(repository.getDirectory()));
|
||||
this.repository = null;
|
||||
repositoryLock.writeLock().lock();
|
||||
try {
|
||||
if (this.repository != null) {
|
||||
FileUtils.deleteDirectory(new File(repository.getDirectory()));
|
||||
this.repository = null;
|
||||
}
|
||||
EntitiesVersionControlSettings settings = getSettings();
|
||||
this.repository = initRepository(settings.getGitSettings());
|
||||
} finally {
|
||||
repositoryLock.writeLock().unlock();
|
||||
}
|
||||
EntitiesVersionControlSettings settings = getSettings();
|
||||
this.repository = initRepository(settings.getGitSettings());
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -37,10 +37,17 @@ public interface EntitiesVersionControlService {
|
||||
|
||||
List<EntityVersion> listEntityTypeVersions(TenantId tenantId, EntityType entityType, String branch, int limit) throws Exception;
|
||||
|
||||
List<EntityVersion> listVersions(TenantId tenantId, String branch, int limit) throws Exception;
|
||||
|
||||
<E extends ExportableEntity<I>, I extends EntityId> EntityExportData<E> getEntityAtVersion(TenantId tenantId, I entityId, String versionId) throws Exception;
|
||||
|
||||
<E extends ExportableEntity<I>, I extends EntityId> EntityImportResult<E> loadEntityVersion(TenantId tenantId, I entityId, String versionId) throws Exception;
|
||||
List<String> listFilesAtVersion(TenantId tenantId, String branch, String versionId) throws Exception;
|
||||
|
||||
|
||||
<E extends ExportableEntity<I>, I extends EntityId> EntityExportData<E> getEntityAtVersion(TenantId tenantId, I entityId, String branch, String versionId) throws Exception;
|
||||
|
||||
<E extends ExportableEntity<I>, I extends EntityId> EntityImportResult<E> loadEntityVersion(TenantId tenantId, I entityId, String branch, String versionId) throws Exception;
|
||||
|
||||
List<EntityImportResult<ExportableEntity<EntityId>>> loadAllAtVersion(TenantId tenantId, String branch, String versionId) throws Exception;
|
||||
|
||||
|
||||
void saveSettings(EntitiesVersionControlSettings settings) throws Exception;
|
||||
|
||||
@ -113,13 +113,13 @@ public class Repository {
|
||||
}
|
||||
|
||||
|
||||
public List<String> listFilesAtCommit(Commit commit) throws IOException {
|
||||
return listFilesAtCommit(commit, null);
|
||||
public List<String> listFilesAtCommit(String commitId) throws IOException {
|
||||
return listFilesAtCommit(commitId, null);
|
||||
}
|
||||
|
||||
public List<String> listFilesAtCommit(Commit commit, String path) throws IOException {
|
||||
public List<String> listFilesAtCommit(String commitId, String path) throws IOException {
|
||||
List<String> files = new ArrayList<>();
|
||||
RevCommit revCommit = resolveCommit(commit.getId());
|
||||
RevCommit revCommit = resolveCommit(commitId);
|
||||
try (TreeWalk treeWalk = new TreeWalk(git.getRepository())) {
|
||||
treeWalk.reset(revCommit.getTree().getId());
|
||||
if (StringUtils.isNotEmpty(path)) {
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user