dendi-wang/thingsboard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add Load Entities Progress

Browse Source
This commit is contained in:
Andrii Shvaika 2022-06-20 18:05:55 +03:00
parent b08b39fb08
commit 2a447ec345
5 changed files with 34 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java
View File

@ -126,8 +126,7 @@ public class EntitiesVersionControlController extends BaseController {
@ApiOperation(value = "", notes = "")
@GetMapping(value = "/version/{requestId}/status")
public VersionCreationResult getVersionCreateRequestStatus(@ApiParam(value = VC_REQUEST_ID_PARAM_DESCRIPTION, required = true)
@PathVariable UUID requestId
) throws Exception {
@PathVariable UUID requestId) throws Exception {
accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ);
return versionControlService.getVersionCreateStatus(getCurrentUser(), requestId);
}
@ -290,6 +289,14 @@ public class EntitiesVersionControlController extends BaseController {
return versionControlService.loadEntitiesVersion(user, request);
}
@ApiOperation(value = "", notes = "")
@GetMapping(value = "/entity/{requestId}/status")
public VersionLoadResult getVersionLoadRequestStatus(@ApiParam(value = VC_REQUEST_ID_PARAM_DESCRIPTION, required = true)
@PathVariable UUID requestId) throws Exception {
accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ);
return versionControlService.getVersionLoadStatus(getCurrentUser(), requestId);
}
@ApiOperation(value = "", notes = "" +
"```\n[\n" +

33
application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java
View File

@ -259,7 +259,8 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
SingleEntityVersionLoadRequest versionLoadRequest = (SingleEntityVersionLoadRequest) request;
VersionLoadConfig config = versionLoadRequest.getConfig();
ListenableFuture<EntityExportData> future = gitServiceQueue.getEntity(user.getTenantId(), request.getVersionId(), versionLoadRequest.getExternalEntityId());
DonAsynchron.withCallback(future, entityData -> doInTemplate(ctx, request, c -> loadSingleEntity(c, config, entityData)),
DonAsynchron.withCallback(future,
entityData -> doInTemplate(ctx, request, c -> loadSingleEntity(c, config, entityData)),
e -> processLoadError(ctx, e), executor);
break;
}
@ -278,19 +279,16 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
private <R> VersionLoadResult doInTemplate(EntitiesImportCtx ctx, VersionLoadRequest request, Function<EntitiesImportCtx, VersionLoadResult> function) {
try {
VersionLoadResult result = transactionTemplate.execute(status -> function.apply(ctx));
try {
for (ThrowingRunnable throwingRunnable : ctx.getEventCallbacks()) {
throwingRunnable.run();
}
} catch (ThingsboardException e) {
throw new RuntimeException(e);
for (ThrowingRunnable throwingRunnable : ctx.getEventCallbacks()) {
throwingRunnable.run();
}
return result;
result.setDone(true);
return cachePut(ctx.getRequestId(), result);
} catch (LoadEntityException e) {
return onError(e.getData(), e.getCause());
return cachePut(ctx.getRequestId(), onError(e.getData(), e.getCause()));
} catch (Exception e) {
log.info("[{}] Failed to process request [{}] due to: ", ctx.getTenantId(), request, e);
throw e;
return cachePut(ctx.getRequestId(), VersionLoadResult.error(EntityLoadError.runtimeError(e.getMessage())));
}
}
@ -329,19 +327,21 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
sw.startNew("Entities " + entityType.name());
ctx.setSettings(getEntityImportSettings(request, entityType));
importEntities(ctx, entityType);
persistToCache(ctx);
}
sw.startNew("Reimport");
reimport(ctx);
persistToCache(ctx);
sw.startNew("Remove Others");
request.getEntityTypes().keySet().stream()
.filter(entityType -> request.getEntityTypes().get(entityType).isRemoveOtherEntities())
.sorted(exportImportService.getEntityTypeComparatorForImport().reversed())
.forEach(entityType -> removeOtherEntities(ctx, entityType));
persistToCache(ctx);
sw.startNew("References and Relations");
exportImportService.saveReferencesAndRelations(ctx);
sw.stop();
@ -430,7 +430,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
}
private VersionLoadResult onError(EntityExportData<?> entityData, Throwable e) {
return analyze(e, entityData).orElseThrow(() -> new RuntimeException(e));
return analyze(e, entityData).orElse(VersionLoadResult.error(EntityLoadError.runtimeError(e.getMessage())));
}
private Optional<VersionLoadResult> analyze(Throwable e, EntityExportData<?> entityData) {
@ -607,7 +607,14 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
taskCache.put(requestId, VersionControlTaskCacheEntry.newForExport(result));
}
private void cachePut(UUID requestId, VersionLoadResult result) {
private VersionLoadResult cachePut(UUID requestId, VersionLoadResult result) {
log.debug("[{}] Cache put: {}", requestId, result);
taskCache.put(requestId, VersionControlTaskCacheEntry.newForImport(result));
return result;
}
private void persistToCache(EntitiesImportCtx ctx) {
cachePut(ctx.getRequestId(), VersionLoadResult.success(new ArrayList<>(ctx.getResults().values())));
}
}

2
application/src/main/resources/thingsboard.yml
View File

@ -422,7 +422,7 @@ cache:
timeToLiveInMinutes: "${CACHE_SPECS_TWO_FA_VERIFICATION_CODES_TTL:60}"
maxSize: "${CACHE_SPECS_TWO_FA_VERIFICATION_CODES_MAX_SIZE:100000}"
versionControlTask:
timeToLiveInMinutes: "${CACHE_SPECS_VERSION_CONTROL_TASK_TTL:60}"
timeToLiveInMinutes: "${CACHE_SPECS_VERSION_CONTROL_TASK_TTL:5}"
maxSize: "${CACHE_SPECS_VERSION_CONTROL_TASK_MAX_SIZE:100000}"
redis:

3
application/src/test/java/org/thingsboard/server/service/sync/ie/BaseExportImportServiceTest.java
View File

@ -82,6 +82,7 @@ import org.thingsboard.server.service.sync.vc.data.SimpleEntitiesExportCtx;
import java.nio.ByteBuffer;
import java.util.Arrays;
import java.util.Collections;
import java.util.UUID;
import static org.assertj.core.api.Assertions.assertThat;
@ -433,7 +434,7 @@ public abstract class BaseExportImportServiceTest extends AbstractControllerTest
}
protected <E extends ExportableEntity<I>, I extends EntityId> EntityImportResult<E> importEntity(User user, EntityExportData<E> exportData, EntityImportSettings importSettings) throws Exception {
EntitiesImportCtx ctx = new EntitiesImportCtx(getSecurityUser(user), null, importSettings);
EntitiesImportCtx ctx = new EntitiesImportCtx(UUID.randomUUID(), getSecurityUser(user), null, importSettings);
ctx.setFinalImportAttempt(true);
exportData = JacksonUtil.treeToValue(JacksonUtil.valueToTree(exportData), EntityExportData.class);
EntityImportResult<E> importResult = exportImportService.importEntity(ctx, exportData);

4
common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionLoadResult.java
View File

@ -34,11 +34,11 @@ public class VersionLoadResult implements Serializable {
private boolean done;
public static VersionLoadResult success(List<EntityTypeLoadResult> result) {
return VersionLoadResult.builder().result(result).done(true).build();
return VersionLoadResult.builder().result(result).build();
}
public static VersionLoadResult success(EntityTypeLoadResult result) {
return VersionLoadResult.builder().result(List.of(result)).done(true).build();
return VersionLoadResult.builder().result(List.of(result)).build();
}
public static VersionLoadResult error(EntityLoadError error) {