dendi-wang/thingsboard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #10377 from thingsboard/fix/vc-performance

Browse Source 
VC restore: optional rollback on error to vastly increase performance
This commit is contained in:
Andrew Shvayka 2024-04-01 16:35:56 +03:00 committed by GitHub
commit 20c4ec1917
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 70 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java
View File

@ -101,7 +101,11 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS
ctx.putInternalId(exportData.getExternalId(), importResult.getSavedEntity().getId());
ctx.addReferenceCallback(exportData.getExternalId(), importResult.getSaveReferencesCallback());
if (ctx.isRollbackOnError()) {
ctx.addEventCallback(importResult.getSendEventsCallback());
} else {
importResult.getSendEventsCallback().run();
}
return importResult;
}

60
application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java
View File

@ -39,6 +39,7 @@ import org.thingsboard.server.common.data.id.EntityIdFactory;
import org.thingsboard.server.common.data.id.HasId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageDataIterable;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.sync.ie.EntityExportData;
import org.thingsboard.server.common.data.sync.ie.EntityExportSettings;
@ -89,6 +90,7 @@ import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ExecutionException;
import java.util.function.Function;
@ -246,16 +248,18 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
switch (request.getType()) {
case SINGLE_ENTITY: {
SingleEntityVersionLoadRequest versionLoadRequest = (SingleEntityVersionLoadRequest) request;
ctx.setRollbackOnError(true);
VersionLoadConfig config = versionLoadRequest.getConfig();
ListenableFuture<EntityExportData> future = gitServiceQueue.getEntity(user.getTenantId(), request.getVersionId(), versionLoadRequest.getExternalEntityId());
DonAsynchron.withCallback(future,
entityData -> doInTemplate(ctx, request, c -> loadSingleEntity(c, config, entityData)),
entityData -> load(ctx, request, c -> loadSingleEntity(c, config, entityData)),
e -> processLoadError(ctx, e), executor);
break;
}
case ENTITY_TYPE: {
EntityTypeVersionLoadRequest versionLoadRequest = (EntityTypeVersionLoadRequest) request;
executor.submit(() -> doInTemplate(ctx, request, c -> loadMultipleEntities(c, versionLoadRequest)));
ctx.setRollbackOnError(versionLoadRequest.isRollbackOnError());
executor.submit(() -> load(ctx, request, c -> loadMultipleEntities(c, versionLoadRequest)));
break;
}
default:
@ -265,19 +269,24 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
return ctx.getRequestId();
}
private <R> VersionLoadResult doInTemplate(EntitiesImportCtx ctx, VersionLoadRequest request, Function<EntitiesImportCtx, VersionLoadResult> function) {
private <R> VersionLoadResult load(EntitiesImportCtx ctx, VersionLoadRequest request, Function<EntitiesImportCtx, VersionLoadResult> loadFunction) {
try {
VersionLoadResult result = transactionTemplate.execute(status -> {
VersionLoadResult result;
if (ctx.isRollbackOnError()) {
result = transactionTemplate.execute(status -> {
try {
return function.apply(ctx);
return loadFunction.apply(ctx);
} catch (RuntimeException e) {
throw e;
} catch (Exception e) {
throw new RuntimeException(e); // to prevent UndeclaredThrowableException
}
});
for (ThrowingRunnable throwingRunnable : ctx.getEventCallbacks()) {
throwingRunnable.run();
for (ThrowingRunnable eventCallback : ctx.getEventCallbacks()) {
eventCallback.run();
}
} else {
result = loadFunction.apply(ctx);
}
result.setDone(true);
return cachePut(ctx.getRequestId(), result);
@ -324,7 +333,6 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
sw.startNew("Entities " + entityType.name());
ctx.setSettings(getEntityImportSettings(request, entityType));
importEntities(ctx, entityType);
persistToCache(ctx);
}
sw.startNew("Reimport");
@ -336,7 +344,6 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
.filter(entityType -> request.getEntityTypes().get(entityType).isRemoveOtherEntities())
.sorted(exportImportService.getEntityTypeComparatorForImport().reversed())
.forEach(entityType -> removeOtherEntities(ctx, entityType));
persistToCache(ctx);
sw.startNew("References and Relations");
exportImportService.saveReferencesAndRelations(ctx);
@ -389,6 +396,8 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
ctx.getImportedEntities().computeIfAbsent(entityType, t -> new HashSet<>())
.add(importResult.getSavedEntity().getId());
}
persistToCache(ctx);
log.debug("Imported {} pack ({}) for tenant {}", entityType, entityDataList.size(), ctx.getTenantId());
offset += limit;
} while (entityDataList.size() == limit);
@ -413,17 +422,34 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
}
private void removeOtherEntities(EntitiesImportCtx ctx, EntityType entityType) {
DaoUtil.processInBatches(pageLink -> {
return exportableEntitiesService.findEntitiesByTenantId(ctx.getTenantId(), entityType, pageLink);
}, 100, entity -> {
if (ctx.getImportedEntities().get(entityType) == null || !ctx.getImportedEntities().get(entityType).contains(entity.getId())) {
exportableEntitiesService.removeById(ctx.getTenantId(), entity.getId());
var entities = new PageDataIterable<>(link -> exportableEntitiesService.findEntitiesIdsByTenantId(ctx.getTenantId(), entityType, link), 100);
Set<EntityId> toRemove = new HashSet<>();
for (EntityId entityId : entities) {
if (ctx.getImportedEntities().get(entityType) == null || !ctx.getImportedEntities().get(entityType).contains(entityId)) {
toRemove.add(entityId);
}
}
ctx.addEventCallback(() -> logEntityActionService.logEntityAction(ctx.getTenantId(), entity.getId(), entity, null,
ActionType.DELETED, ctx.getUser()));
for (EntityId entityId : toRemove) {
ExportableEntity<EntityId> entity = exportableEntitiesService.findEntityById(entityId);
exportableEntitiesService.removeById(ctx.getTenantId(), entityId);
ThrowingRunnable callback = () -> {
logEntityActionService.logEntityAction(ctx.getTenantId(), entity.getId(), entity, null,
ActionType.DELETED, ctx.getUser());
};
if (ctx.isRollbackOnError()) {
ctx.addEventCallback(callback);
} else {
try {
callback.run();
} catch (ThingsboardException e) {
throw new RuntimeException(e);
}
}
ctx.registerDeleted(entityType);
}
});
persistToCache(ctx);
}
private VersionLoadResult onError(EntityId externalId, Throwable e) {

1
application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java
View File

@ -58,6 +58,7 @@ public class EntitiesImportCtx {
private boolean finalImportAttempt = false;
private EntityImportSettings settings;
private EntityImportResult<?> currentImportResult;
private boolean rollbackOnError;
public EntitiesImportCtx(UUID requestId, User user, String versionId) {
this(requestId, user, versionId, null);

1
common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/EntityTypeVersionLoadRequest.java
View File

@ -26,6 +26,7 @@ import java.util.Map;
public class EntityTypeVersionLoadRequest extends VersionLoadRequest {
private Map<EntityType, EntityTypeVersionLoadConfig> entityTypes;
private boolean rollbackOnError;
@Override
public VersionLoadRequestType getType() {

7
ui-ngx/src/app/modules/home/components/vc/complex-version-load.component.html
View File

@ -23,11 +23,16 @@
<mat-progress-bar color="warn" style="z-index: 10; width: 100%; margin-bottom: -4px;" mode="indeterminate"
*ngIf="isLoading$ | async">
</mat-progress-bar>
<form [formGroup]="loadVersionFormGroup" fxLayout="column" style="flex: 1; padding-top: 16px; overflow: auto;">
<form [formGroup]="loadVersionFormGroup" fxLayout="column" style="flex: 1; padding: 16px 0; overflow-y: auto; overflow-x: hidden">
<tb-entity-types-version-load
formControlName="entityTypes">
</tb-entity-types-version-load>
</form>
<div tb-hint-tooltip-icon="{{ 'version-control.rollback-on-error-hint' | translate }}" [formGroup]="loadVersionFormGroup">
<mat-slide-toggle formControlName="rollbackOnError">
{{ 'version-control.rollback-on-error' | translate }}
</mat-slide-toggle>
</div>
<div fxLayoutAlign="end center" fxLayoutGap="8px" style="padding-top: 16px;">
<button mat-button color="primary"
type="button"

2
ui-ngx/src/app/modules/home/components/vc/complex-version-load.component.ts
View File

@ -79,6 +79,7 @@ export class ComplexVersionLoadComponent extends PageComponent implements OnInit
ngOnInit(): void {
this.loadVersionFormGroup = this.fb.group({
entityTypes: [createDefaultEntityTypesVersionLoad(), []],
rollbackOnError: [true]
});
}
@ -116,6 +117,7 @@ export class ComplexVersionLoadComponent extends PageComponent implements OnInit
const request: EntityTypeVersionLoadRequest = {
versionId: this.versionId,
entityTypes: this.loadVersionFormGroup.get('entityTypes').value,
rollbackOnError: this.loadVersionFormGroup.get('rollbackOnError').value,
type: VersionLoadRequestType.ENTITY_TYPE
};
this.versionLoadResult$ = this.entitiesVersionControlService.loadEntitiesVersion(request, {ignoreErrors: true}).pipe(

1
ui-ngx/src/app/shared/models/vc.models.ts
View File

@ -144,6 +144,7 @@ export interface EntityTypeVersionLoadConfig extends VersionLoadConfig {
export interface EntityTypeVersionLoadRequest extends VersionLoadRequest {
entityTypes: {[entityType: string]: EntityTypeVersionLoadConfig};
type: VersionLoadRequestType.ENTITY_TYPE;
rollbackOnError: boolean;
}
export function createDefaultEntityTypesVersionLoad(): {[entityType: string]: EntityTypeVersionLoadConfig} {

4
ui-ngx/src/assets/locale/locale.constant-en_US.json
View File

@ -5016,7 +5016,9 @@
"device-credentials-conflict": "Failed to load the device with external id <b>{{entityId}}</b><br/>due to the same credentials are already present in the database for another device.<br/>Please consider disabling the <b>load credentials</b> setting in the restore form.",
"missing-referenced-entity": "Failed to load the <b>{{sourceEntityTypeName}}</b> with external id <b>{{sourceEntityId}}</b><br/>because it references missing <b>{{targetEntityTypeName}}</b> with id <b>{{targetEntityId}}</b>.",
"runtime-failed": "<b>Failed:</b> {{message}}",
"auto-commit-settings-read-only-hint": "Auto-commit feature doesn't work with enabled read-only option in Repository settings."
"auto-commit-settings-read-only-hint": "Auto-commit feature doesn't work with enabled read-only option in Repository settings.",
"rollback-on-error": "Rollback on error",
"rollback-on-error-hint": "If you have a large amount of entities to restore, consider disabling this option to increase performance.\n Note, if an error occurs over the course of version loading, already persisted entities (with relations, attributes, etc.) will stay as is"
},
"widget": {
"widget-library": "Widgets library",