dendi-wang/thingsboard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #6559 from ViacheslavKlimov/feature/entities-vc

Browse Source 
Changes to version create request; fixes for git api usage
This commit is contained in:
Andrew Shvayka 2022-05-18 09:17:54 +03:00 committed by GitHub
commit 846d9bba75
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 85 additions and 128 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java
View File

@ -15,7 +15,6 @@
*/
package org.thingsboard.server.controller;
import io.swagger.annotations.ApiModelProperty;
import io.swagger.annotations.ApiOperation;
import lombok.Data;
import lombok.RequiredArgsConstructor;
@ -72,43 +71,27 @@ public class EntitiesVersionControlController extends BaseController {
" \"saveRelations\": true\n" +
" }\n" +
"}\n```" + NEW_LINE +
"ENTITY_LIST:" + NEW_LINE +
"COMPLEX:" + NEW_LINE +
"```\n{\n" +
" \"type\": \"ENTITY_LIST\",\n" +
" \"type\": \"COMPLEX\",\n" +
"\n" +
" \"versionName\": \"Version 1.0\",\n" +
" \"branch\": \"dev\",\n" +
"\n" +
" \"entitiesIds\": [\n" +
" {\n" +
" \"entityType\": \"DEVICE\",\n" +
" \"id\": \"b79448e0-d4f4-11ec-847b-0f432358ab48\"\n" +
" },\n" +
" {\n" +
" \"entityType\": \"DEVICE_PROFILE\",\n" +
" \"id\": \"b7944123-d4f4-11ec-847b-0f432358ab48\"\n" +
" }\n" +
" ],\n" +
" \"config\": {\n" +
" \"saveRelations\": true,\n" +
" \"syncStrategy\": \"MERGE\"\n" +
" }\n" +
"}\n```" + NEW_LINE +
"ENTITY_TYPE:" + NEW_LINE +
"```\n{\n" +
" \"type\": \"ENTITY_TYPE\",\n" +
"\n" +
" \"versionName\": \"Version 1.0\",\n" +
" \"branch\": \"dev\",\n" +
" \"versionName\": \"Devices and profiles: release 2\",\n" +
" \"branch\": \"master\",\n" +
"\n" +
" \"syncStrategy\": \"OVERWRITE\",\n" +
" \"entityTypes\": {\n" +
" \"DEVICE\": {\n" +
" \"saveRelations\": true,\n" +
" \"syncStrategy\": \"MERGE\"\n" +
" \"syncStrategy\": null,\n" +
" \"allEntities\": true,\n" +
" \"saveRelations\": true\n" +
" },\n" +
" \"DEVICE_PROFILE\": {\n" +
" \"saveRelations\": true,\n" +
" \"syncStrategy\": \"OVERWRITE\"\n" +
" \"syncStrategy\": \"MERGE\",\n" +
" \"allEntities\": false,\n" +
" \"entityIds\": [\n" +
" \"b79448e0-d4f4-11ec-847b-0f432358ab48\"\n" +
" ],\n" +
" \"saveRelations\": true\n" +
" }\n" +
" }\n" +
"}\n```")

94
application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java
View File

@ -22,7 +22,10 @@ import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
import org.eclipse.jgit.api.errors.JGitInternalException;
import org.eclipse.jgit.api.errors.RefAlreadyExistsException;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.springframework.transaction.support.TransactionTemplate;
@ -63,7 +66,6 @@ import org.thingsboard.server.service.sync.vc.data.EntityVersion;
import org.thingsboard.server.service.sync.vc.data.VersionCreationResult;
import org.thingsboard.server.service.sync.vc.data.VersionLoadResult;
import org.thingsboard.server.service.sync.vc.data.VersionedEntityInfo;
import org.thingsboard.server.service.sync.vc.data.request.create.EntityListVersionCreateRequest;
import org.thingsboard.server.service.sync.vc.data.request.create.ComplexVersionCreateRequest;
import org.thingsboard.server.service.sync.vc.data.request.create.SingleEntityVersionCreateRequest;
import org.thingsboard.server.service.sync.vc.data.request.create.SyncStrategy;
@ -88,6 +90,7 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
@ -149,10 +152,21 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
repository.fetch();
if (repository.listBranches().contains(request.getBranch())) {
repository.checkout(request.getBranch());
repository.checkout("origin/" + request.getBranch(), false);
try {
repository.checkout(request.getBranch(), true);
} catch (RefAlreadyExistsException e) {
repository.checkout(request.getBranch(), false);
}
repository.merge(request.getBranch());
} else { // TODO [viacheslav]: rollback orphan branch on failure
repository.createAndCheckoutOrphanBranch(request.getBranch()); // FIXME [viacheslav]: Checkout returned unexpected result NO_CHANGE for master branch
try {
repository.createAndCheckoutOrphanBranch(request.getBranch()); // FIXME [viacheslav]: Checkout returned unexpected result NO_CHANGE for master branch
} catch (JGitInternalException e) {
if (!e.getMessage().contains("NO_CHANGE")) {
throw e;
}
}
}
switch (request.getType()) {
@ -161,28 +175,10 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
saveEntityData(user, repository, versionCreateRequest.getEntityId(), versionCreateRequest.getConfig());
break;
}
case ENTITY_LIST: {
EntityListVersionCreateRequest versionCreateRequest = (EntityListVersionCreateRequest) request;
if (versionCreateRequest.getConfig().getSyncStrategy() == SyncStrategy.OVERWRITE) {
versionCreateRequest.getEntitiesIds().stream()
.map(EntityId::getEntityType).distinct()
.forEach(entityType -> {
try {
FileUtils.deleteDirectory(Path.of(repository.getDirectory(), getRelativePath(entityType, null)).toFile());
} catch (IOException e) {
throw new RuntimeException(e);
}
});
}
for (EntityId entityId : versionCreateRequest.getEntitiesIds()) {
saveEntityData(user, repository, entityId, versionCreateRequest.getConfig());
}
break;
}
case COMPLEX: {
ComplexVersionCreateRequest versionCreateRequest = (ComplexVersionCreateRequest) request;
versionCreateRequest.getEntityTypes().forEach((entityType, config) -> {
if (config.getSyncStrategy() == SyncStrategy.OVERWRITE) {
if (ObjectUtils.defaultIfNull(config.getSyncStrategy(), versionCreateRequest.getSyncStrategy()) == SyncStrategy.OVERWRITE) {
try {
FileUtils.deleteDirectory(Path.of(repository.getDirectory(), getRelativePath(entityType, null)).toFile());
} catch (IOException e) {
@ -190,27 +186,38 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
}
}
EntityTypeFilter entityTypeFilter = new EntityTypeFilter();
entityTypeFilter.setEntityType(entityType);
EntityDataPageLink entityDataPageLink = new EntityDataPageLink();
entityDataPageLink.setPage(-1);
entityDataPageLink.setPageSize(-1);
EntityKey sortProperty = new EntityKey(EntityKeyType.ENTITY_FIELD, CREATED_TIME);
entityDataPageLink.setSortOrder(new EntityDataSortOrder(sortProperty, EntityDataSortOrder.Direction.DESC));
EntityDataQuery query = new EntityDataQuery(entityTypeFilter, entityDataPageLink, List.of(sortProperty), Collections.emptyList(), Collections.emptyList());
if (config.isAllEntities()) {
EntityTypeFilter entityTypeFilter = new EntityTypeFilter();
entityTypeFilter.setEntityType(entityType);
EntityDataPageLink entityDataPageLink = new EntityDataPageLink();
entityDataPageLink.setPage(-1);
entityDataPageLink.setPageSize(-1);
EntityKey sortProperty = new EntityKey(EntityKeyType.ENTITY_FIELD, CREATED_TIME);
entityDataPageLink.setSortOrder(new EntityDataSortOrder(sortProperty, EntityDataSortOrder.Direction.DESC));
EntityDataQuery query = new EntityDataQuery(entityTypeFilter, entityDataPageLink, List.of(sortProperty), Collections.emptyList(), Collections.emptyList());
DaoUtil.processInBatches(pageLink -> {
entityDataPageLink.setPage(pageLink.getPage());
entityDataPageLink.setPageSize(pageLink.getPageSize());
return entityService.findEntityDataByQuery(user.getTenantId(), new CustomerId(EntityId.NULL_UUID), query);
}, 100, data -> {
EntityId entityId = data.getEntityId();
try {
saveEntityData(user, repository, entityId, config);
} catch (Exception e) {
throw new RuntimeException(e);
DaoUtil.processInBatches(pageLink -> {
entityDataPageLink.setPage(pageLink.getPage());
entityDataPageLink.setPageSize(pageLink.getPageSize());
return entityService.findEntityDataByQuery(user.getTenantId(), new CustomerId(EntityId.NULL_UUID), query);
}, 100, data -> {
EntityId entityId = data.getEntityId();
try {
saveEntityData(user, repository, entityId, config);
} catch (Exception e) {
throw new RuntimeException(e);
}
});
} else {
for (UUID entityId : config.getEntityIds()) {
try {
saveEntityData(user, repository, EntityIdFactory.getByTypeAndUuid(entityType, entityId), config);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
});
}
});
break;
}
@ -427,7 +434,9 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
private void initRepository(TenantId tenantId, EntitiesVersionControlSettings settings) throws Exception {
Path repositoryDirectory = Path.of(repositoriesFolder, tenantId.getId().toString());
GitRepository repository;
FileUtils.forceDelete(repositoryDirectory.toFile());
if (Files.exists(repositoryDirectory)) {
FileUtils.forceDelete(repositoryDirectory.toFile());
}
Files.createDirectories(repositoryDirectory);
repository = GitRepository.clone(settings.getRepositoryUri(), settings.getUsername(), settings.getPassword(), repositoryDirectory.toFile());
@ -450,7 +459,6 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
new BaseAttributeKvEntry(System.currentTimeMillis(), new JsonDataEntry(SETTINGS_KEY, JacksonUtil.toString(settings)))
)).get();
clearRepository(tenantId);
initRepository(tenantId, settings);
}

36
application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityListVersionCreateRequest.java
View File

@ -1,36 +0,0 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.sync.vc.data.request.create;
import lombok.Data;
import lombok.EqualsAndHashCode;
import org.thingsboard.server.common.data.id.EntityId;
import java.util.List;
@Data
@EqualsAndHashCode(callSuper = true)
public class EntityListVersionCreateRequest extends VersionCreateRequest {
private List<EntityId> entitiesIds;
private EntityTypeVersionCreateConfig config;
@Override
public VersionCreateRequestType getType() {
return VersionCreateRequestType.ENTITY_LIST;
}
}

4
application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityTypeVersionCreateConfig.java
View File

@ -17,9 +17,9 @@ package org.thingsboard.server.service.sync.vc.data.request.create;
import lombok.Data;
import lombok.EqualsAndHashCode;
import org.thingsboard.server.common.data.id.EntityId;
import java.util.List;
import java.util.UUID;
@Data
@EqualsAndHashCode(callSuper = true)
@ -27,7 +27,7 @@ public class EntityTypeVersionCreateConfig extends VersionCreateConfig {
//optional
private SyncStrategy syncStrategy;
private List<EntityId> entityIds;
private List<UUID> entityIds;
private boolean allEntities;
}

3
application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequest.java
View File

@ -23,8 +23,7 @@ import lombok.Data;
@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type")
@JsonSubTypes({
@Type(name = "SINGLE_ENTITY", value = SingleEntityVersionCreateRequest.class),
@Type(name = "ENTITY_LIST", value = EntityListVersionCreateRequest.class),
@Type(name = "ENTITY_TYPE", value = ComplexVersionCreateRequest.class)
@Type(name = "COMPLEX", value = ComplexVersionCreateRequest.class)
})
@Data
public abstract class VersionCreateRequest {

1
application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequestType.java
View File

@ -17,6 +17,5 @@ package org.thingsboard.server.service.sync.vc.data.request.create;
public enum VersionCreateRequestType {
SINGLE_ENTITY,
ENTITY_LIST,
COMPLEX
}

30
application/src/main/java/org/thingsboard/server/utils/GitRepository.java
View File

@ -23,8 +23,7 @@ import org.eclipse.jgit.api.Git;
import org.eclipse.jgit.api.GitCommand;
import org.eclipse.jgit.api.ListBranchCommand;
import org.eclipse.jgit.api.LogCommand;
import org.eclipse.jgit.api.RmCommand;
import org.eclipse.jgit.api.Status;
import org.eclipse.jgit.api.ResetCommand;
import org.eclipse.jgit.api.TransportCommand;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.eclipse.jgit.lib.Constants;
@ -44,8 +43,6 @@ import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.stream.Collectors;
public class GitRepository {
@ -84,9 +81,15 @@ public class GitRepository {
.setRemoveDeletedRefs(true));
}
public void checkout(String branch) throws GitAPIException {
public void checkout(String branch, boolean createBranch) throws GitAPIException {
execute(git.checkout()
.setName("origin/" + branch));
.setCreateBranch(createBranch)
.setName(branch));
}
public void reset() throws GitAPIException {
execute(git.reset()
.setMode(ResetCommand.ResetType.HARD));
}
public void merge(String branch) throws IOException, GitAPIException {
@ -170,14 +173,15 @@ public class GitRepository {
public void createAndCheckoutOrphanBranch(String name) throws GitAPIException {
execute(git.checkout()
.setOrphan(true)
.setForced(true)
.setName(name));
Set<String> uncommittedChanges = git.status().call().getUncommittedChanges();
if (!uncommittedChanges.isEmpty()) {
RmCommand rm = git.rm();
uncommittedChanges.forEach(rm::addFilepattern);
execute(rm);
}
execute(git.clean());
// Set<String> uncommittedChanges = git.status().call().getUncommittedChanges();
// if (!uncommittedChanges.isEmpty()) {
// RmCommand rm = git.rm();
// uncommittedChanges.forEach(rm::addFilepattern);
// execute(rm);
// }
// execute(git.clean());
}
public void add(String filesPattern) throws GitAPIException { // FIXME [viacheslav]