dendi-wang/thingsboard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #6559 from ViacheslavKlimov/feature/entities-vc

Browse Source 
Changes to version create request; fixes for git api usage
This commit is contained in:
Andrew Shvayka 2022-05-18 09:17:54 +03:00 committed by GitHub
commit 846d9bba75
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 85 additions and 128 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java
View File

@ -15,7 +15,6 @@
*/ */
package org.thingsboard.server.controller; package org.thingsboard.server.controller;
import io.swagger.annotations.ApiModelProperty;
import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiOperation;
import lombok.Data; import lombok.Data;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
@ -72,43 +71,27 @@ public class EntitiesVersionControlController extends BaseController {
" \"saveRelations\": true\n" + " \"saveRelations\": true\n" +
" }\n" + " }\n" +
"}\n```" + NEW_LINE + "}\n```" + NEW_LINE +
"ENTITY_LIST:" + NEW_LINE + "COMPLEX:" + NEW_LINE +
"```\n{\n" + "```\n{\n" +
" \"type\": \"ENTITY_LIST\",\n" + " \"type\": \"COMPLEX\",\n" +
"\n" + "\n" +
" \"versionName\": \"Version 1.0\",\n" + " \"versionName\": \"Devices and profiles: release 2\",\n" +
" \"branch\": \"dev\",\n" + " \"branch\": \"master\",\n" +
"\n" +
" \"entitiesIds\": [\n" +
" {\n" +
" \"entityType\": \"DEVICE\",\n" +
" \"id\": \"b79448e0-d4f4-11ec-847b-0f432358ab48\"\n" +
" },\n" +
" {\n" +
" \"entityType\": \"DEVICE_PROFILE\",\n" +
" \"id\": \"b7944123-d4f4-11ec-847b-0f432358ab48\"\n" +
" }\n" +
" ],\n" +
" \"config\": {\n" +
" \"saveRelations\": true,\n" +
" \"syncStrategy\": \"MERGE\"\n" +
" }\n" +
"}\n```" + NEW_LINE +
"ENTITY_TYPE:" + NEW_LINE +
"```\n{\n" +
" \"type\": \"ENTITY_TYPE\",\n" +
"\n" +
" \"versionName\": \"Version 1.0\",\n" +
" \"branch\": \"dev\",\n" +
"\n" + "\n" +
" \"syncStrategy\": \"OVERWRITE\",\n" +
" \"entityTypes\": {\n" + " \"entityTypes\": {\n" +
" \"DEVICE\": {\n" + " \"DEVICE\": {\n" +
" \"saveRelations\": true,\n" + " \"syncStrategy\": null,\n" +
" \"syncStrategy\": \"MERGE\"\n" + " \"allEntities\": true,\n" +
" \"saveRelations\": true\n" +
" },\n" + " },\n" +
" \"DEVICE_PROFILE\": {\n" + " \"DEVICE_PROFILE\": {\n" +
" \"saveRelations\": true,\n" + " \"syncStrategy\": \"MERGE\",\n" +
" \"syncStrategy\": \"OVERWRITE\"\n" + " \"allEntities\": false,\n" +
" \"entityIds\": [\n" +
" \"b79448e0-d4f4-11ec-847b-0f432358ab48\"\n" +
" ],\n" +
" \"saveRelations\": true\n" +
" }\n" + " }\n" +
" }\n" + " }\n" +
"}\n```") "}\n```")

94
application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java
View File

@ -22,7 +22,10 @@ import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows; import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FileUtils; import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.eclipse.jgit.api.errors.JGitInternalException;
import org.eclipse.jgit.api.errors.RefAlreadyExistsException;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.transaction.support.TransactionTemplate; import org.springframework.transaction.support.TransactionTemplate;
@ -63,7 +66,6 @@ import org.thingsboard.server.service.sync.vc.data.EntityVersion;
import org.thingsboard.server.service.sync.vc.data.VersionCreationResult; import org.thingsboard.server.service.sync.vc.data.VersionCreationResult;
import org.thingsboard.server.service.sync.vc.data.VersionLoadResult; import org.thingsboard.server.service.sync.vc.data.VersionLoadResult;
import org.thingsboard.server.service.sync.vc.data.VersionedEntityInfo; import org.thingsboard.server.service.sync.vc.data.VersionedEntityInfo;
import org.thingsboard.server.service.sync.vc.data.request.create.EntityListVersionCreateRequest;
import org.thingsboard.server.service.sync.vc.data.request.create.ComplexVersionCreateRequest; import org.thingsboard.server.service.sync.vc.data.request.create.ComplexVersionCreateRequest;
import org.thingsboard.server.service.sync.vc.data.request.create.SingleEntityVersionCreateRequest; import org.thingsboard.server.service.sync.vc.data.request.create.SingleEntityVersionCreateRequest;
import org.thingsboard.server.service.sync.vc.data.request.create.SyncStrategy; import org.thingsboard.server.service.sync.vc.data.request.create.SyncStrategy;
@ -88,6 +90,7 @@ import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Optional; import java.util.Optional;
import java.util.Set; import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Executors; import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
@ -149,10 +152,21 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
repository.fetch(); repository.fetch();
if (repository.listBranches().contains(request.getBranch())) { if (repository.listBranches().contains(request.getBranch())) {
repository.checkout(request.getBranch()); repository.checkout("origin/" + request.getBranch(), false);
try {
repository.checkout(request.getBranch(), true);
} catch (RefAlreadyExistsException e) {
repository.checkout(request.getBranch(), false);
}
repository.merge(request.getBranch()); repository.merge(request.getBranch());
} else { // TODO [viacheslav]: rollback orphan branch on failure } else { // TODO [viacheslav]: rollback orphan branch on failure
repository.createAndCheckoutOrphanBranch(request.getBranch()); // FIXME [viacheslav]: Checkout returned unexpected result NO_CHANGE for master branch try {
repository.createAndCheckoutOrphanBranch(request.getBranch()); // FIXME [viacheslav]: Checkout returned unexpected result NO_CHANGE for master branch
} catch (JGitInternalException e) {
if (!e.getMessage().contains("NO_CHANGE")) {
throw e;
}
}
} }
switch (request.getType()) { switch (request.getType()) {
@ -161,28 +175,10 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
saveEntityData(user, repository, versionCreateRequest.getEntityId(), versionCreateRequest.getConfig()); saveEntityData(user, repository, versionCreateRequest.getEntityId(), versionCreateRequest.getConfig());
break; break;
} }
case ENTITY_LIST: {
EntityListVersionCreateRequest versionCreateRequest = (EntityListVersionCreateRequest) request;
if (versionCreateRequest.getConfig().getSyncStrategy() == SyncStrategy.OVERWRITE) {
versionCreateRequest.getEntitiesIds().stream()
.map(EntityId::getEntityType).distinct()
.forEach(entityType -> {
try {
FileUtils.deleteDirectory(Path.of(repository.getDirectory(), getRelativePath(entityType, null)).toFile());
} catch (IOException e) {
throw new RuntimeException(e);
}
});
}
for (EntityId entityId : versionCreateRequest.getEntitiesIds()) {
saveEntityData(user, repository, entityId, versionCreateRequest.getConfig());
}
break;
}
case COMPLEX: { case COMPLEX: {
ComplexVersionCreateRequest versionCreateRequest = (ComplexVersionCreateRequest) request; ComplexVersionCreateRequest versionCreateRequest = (ComplexVersionCreateRequest) request;
versionCreateRequest.getEntityTypes().forEach((entityType, config) -> { versionCreateRequest.getEntityTypes().forEach((entityType, config) -> {
if (config.getSyncStrategy() == SyncStrategy.OVERWRITE) { if (ObjectUtils.defaultIfNull(config.getSyncStrategy(), versionCreateRequest.getSyncStrategy()) == SyncStrategy.OVERWRITE) {
try { try {
FileUtils.deleteDirectory(Path.of(repository.getDirectory(), getRelativePath(entityType, null)).toFile()); FileUtils.deleteDirectory(Path.of(repository.getDirectory(), getRelativePath(entityType, null)).toFile());
} catch (IOException e) { } catch (IOException e) {
@ -190,27 +186,38 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
} }
} }
EntityTypeFilter entityTypeFilter = new EntityTypeFilter(); if (config.isAllEntities()) {
entityTypeFilter.setEntityType(entityType); EntityTypeFilter entityTypeFilter = new EntityTypeFilter();
EntityDataPageLink entityDataPageLink = new EntityDataPageLink(); entityTypeFilter.setEntityType(entityType);
entityDataPageLink.setPage(-1); EntityDataPageLink entityDataPageLink = new EntityDataPageLink();
entityDataPageLink.setPageSize(-1); entityDataPageLink.setPage(-1);
EntityKey sortProperty = new EntityKey(EntityKeyType.ENTITY_FIELD, CREATED_TIME); entityDataPageLink.setPageSize(-1);
entityDataPageLink.setSortOrder(new EntityDataSortOrder(sortProperty, EntityDataSortOrder.Direction.DESC)); EntityKey sortProperty = new EntityKey(EntityKeyType.ENTITY_FIELD, CREATED_TIME);
EntityDataQuery query = new EntityDataQuery(entityTypeFilter, entityDataPageLink, List.of(sortProperty), Collections.emptyList(), Collections.emptyList()); entityDataPageLink.setSortOrder(new EntityDataSortOrder(sortProperty, EntityDataSortOrder.Direction.DESC));
EntityDataQuery query = new EntityDataQuery(entityTypeFilter, entityDataPageLink, List.of(sortProperty), Collections.emptyList(), Collections.emptyList());
DaoUtil.processInBatches(pageLink -> { DaoUtil.processInBatches(pageLink -> {
entityDataPageLink.setPage(pageLink.getPage()); entityDataPageLink.setPage(pageLink.getPage());
entityDataPageLink.setPageSize(pageLink.getPageSize()); entityDataPageLink.setPageSize(pageLink.getPageSize());
return entityService.findEntityDataByQuery(user.getTenantId(), new CustomerId(EntityId.NULL_UUID), query); return entityService.findEntityDataByQuery(user.getTenantId(), new CustomerId(EntityId.NULL_UUID), query);
}, 100, data -> { }, 100, data -> {
EntityId entityId = data.getEntityId(); EntityId entityId = data.getEntityId();
try { try {
saveEntityData(user, repository, entityId, config); saveEntityData(user, repository, entityId, config);
} catch (Exception e) { } catch (Exception e) {
throw new RuntimeException(e); throw new RuntimeException(e);
}
});
} else {
for (UUID entityId : config.getEntityIds()) {
try {
saveEntityData(user, repository, EntityIdFactory.getByTypeAndUuid(entityType, entityId), config);
} catch (Exception e) {
throw new RuntimeException(e);
}
} }
}); }
}); });
break; break;
} }
@ -427,7 +434,9 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
private void initRepository(TenantId tenantId, EntitiesVersionControlSettings settings) throws Exception { private void initRepository(TenantId tenantId, EntitiesVersionControlSettings settings) throws Exception {
Path repositoryDirectory = Path.of(repositoriesFolder, tenantId.getId().toString()); Path repositoryDirectory = Path.of(repositoriesFolder, tenantId.getId().toString());
GitRepository repository; GitRepository repository;
FileUtils.forceDelete(repositoryDirectory.toFile()); if (Files.exists(repositoryDirectory)) {
FileUtils.forceDelete(repositoryDirectory.toFile());
}
Files.createDirectories(repositoryDirectory); Files.createDirectories(repositoryDirectory);
repository = GitRepository.clone(settings.getRepositoryUri(), settings.getUsername(), settings.getPassword(), repositoryDirectory.toFile()); repository = GitRepository.clone(settings.getRepositoryUri(), settings.getUsername(), settings.getPassword(), repositoryDirectory.toFile());
@ -450,7 +459,6 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
new BaseAttributeKvEntry(System.currentTimeMillis(), new JsonDataEntry(SETTINGS_KEY, JacksonUtil.toString(settings))) new BaseAttributeKvEntry(System.currentTimeMillis(), new JsonDataEntry(SETTINGS_KEY, JacksonUtil.toString(settings)))
)).get(); )).get();
clearRepository(tenantId);
initRepository(tenantId, settings); initRepository(tenantId, settings);
} }

36
application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityListVersionCreateRequest.java
View File

@ -1,36 +0,0 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.sync.vc.data.request.create;
import lombok.Data;
import lombok.EqualsAndHashCode;
import org.thingsboard.server.common.data.id.EntityId;
import java.util.List;
@Data
@EqualsAndHashCode(callSuper = true)
public class EntityListVersionCreateRequest extends VersionCreateRequest {
private List<EntityId> entitiesIds;
private EntityTypeVersionCreateConfig config;
@Override
public VersionCreateRequestType getType() {
return VersionCreateRequestType.ENTITY_LIST;
}
}

4
application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityTypeVersionCreateConfig.java
View File

@ -17,9 +17,9 @@ package org.thingsboard.server.service.sync.vc.data.request.create;
import lombok.Data; import lombok.Data;
import lombok.EqualsAndHashCode; import lombok.EqualsAndHashCode;
import org.thingsboard.server.common.data.id.EntityId;
import java.util.List; import java.util.List;
import java.util.UUID;
@Data @Data
@EqualsAndHashCode(callSuper = true) @EqualsAndHashCode(callSuper = true)
@ -27,7 +27,7 @@ public class EntityTypeVersionCreateConfig extends VersionCreateConfig {
//optional //optional
private SyncStrategy syncStrategy; private SyncStrategy syncStrategy;
private List<EntityId> entityIds; private List<UUID> entityIds;
private boolean allEntities; private boolean allEntities;
} }

3
application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequest.java
View File

@ -23,8 +23,7 @@ import lombok.Data;
@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type")
@JsonSubTypes({ @JsonSubTypes({
@Type(name = "SINGLE_ENTITY", value = SingleEntityVersionCreateRequest.class), @Type(name = "SINGLE_ENTITY", value = SingleEntityVersionCreateRequest.class),
@Type(name = "ENTITY_LIST", value = EntityListVersionCreateRequest.class), @Type(name = "COMPLEX", value = ComplexVersionCreateRequest.class)
@Type(name = "ENTITY_TYPE", value = ComplexVersionCreateRequest.class)
}) })
@Data @Data
public abstract class VersionCreateRequest { public abstract class VersionCreateRequest {

1
application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequestType.java
View File

@ -17,6 +17,5 @@ package org.thingsboard.server.service.sync.vc.data.request.create;
public enum VersionCreateRequestType { public enum VersionCreateRequestType {
SINGLE_ENTITY, SINGLE_ENTITY,
ENTITY_LIST,
COMPLEX COMPLEX
} }

30
application/src/main/java/org/thingsboard/server/utils/GitRepository.java
View File

@ -23,8 +23,7 @@ import org.eclipse.jgit.api.Git;
import org.eclipse.jgit.api.GitCommand; import org.eclipse.jgit.api.GitCommand;
import org.eclipse.jgit.api.ListBranchCommand; import org.eclipse.jgit.api.ListBranchCommand;
import org.eclipse.jgit.api.LogCommand; import org.eclipse.jgit.api.LogCommand;
import org.eclipse.jgit.api.RmCommand; import org.eclipse.jgit.api.ResetCommand;
import org.eclipse.jgit.api.Status;
import org.eclipse.jgit.api.TransportCommand; import org.eclipse.jgit.api.TransportCommand;
import org.eclipse.jgit.api.errors.GitAPIException; import org.eclipse.jgit.api.errors.GitAPIException;
import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.Constants;
@ -44,8 +43,6 @@ import java.nio.charset.StandardCharsets;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.Set; import java.util.Set;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.stream.Collectors; import java.util.stream.Collectors;
public class GitRepository { public class GitRepository {
@ -84,9 +81,15 @@ public class GitRepository {
.setRemoveDeletedRefs(true)); .setRemoveDeletedRefs(true));
} }
public void checkout(String branch) throws GitAPIException { public void checkout(String branch, boolean createBranch) throws GitAPIException {
execute(git.checkout() execute(git.checkout()
.setName("origin/" + branch)); .setCreateBranch(createBranch)
.setName(branch));
}
public void reset() throws GitAPIException {
execute(git.reset()
.setMode(ResetCommand.ResetType.HARD));
} }
public void merge(String branch) throws IOException, GitAPIException { public void merge(String branch) throws IOException, GitAPIException {
@ -170,14 +173,15 @@ public class GitRepository {
public void createAndCheckoutOrphanBranch(String name) throws GitAPIException { public void createAndCheckoutOrphanBranch(String name) throws GitAPIException {
execute(git.checkout() execute(git.checkout()
.setOrphan(true) .setOrphan(true)
.setForced(true)
.setName(name)); .setName(name));
Set<String> uncommittedChanges = git.status().call().getUncommittedChanges(); // Set<String> uncommittedChanges = git.status().call().getUncommittedChanges();
if (!uncommittedChanges.isEmpty()) { // if (!uncommittedChanges.isEmpty()) {
RmCommand rm = git.rm(); // RmCommand rm = git.rm();
uncommittedChanges.forEach(rm::addFilepattern); // uncommittedChanges.forEach(rm::addFilepattern);
execute(rm); // execute(rm);
} // }
execute(git.clean()); // execute(git.clean());
} }
public void add(String filesPattern) throws GitAPIException { // FIXME [viacheslav] public void add(String filesPattern) throws GitAPIException { // FIXME [viacheslav]