Add Load Entities Progress

application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java

@@ -126,8 +126,7 @@ public class EntitiesVersionControlController extends BaseController {
     @ApiOperation(value = "", notes = "")
     @GetMapping(value = "/version/{requestId}/status")
     public VersionCreationResult getVersionCreateRequestStatus(@ApiParam(value = VC_REQUEST_ID_PARAM_DESCRIPTION, required = true)
-                                                               @PathVariable UUID requestId
+                                                               @PathVariable UUID requestId) throws Exception {
-    ) throws Exception {
         accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ);
         return versionControlService.getVersionCreateStatus(getCurrentUser(), requestId);
     }
@@ -290,6 +289,14 @@ public class EntitiesVersionControlController extends BaseController {
         return versionControlService.loadEntitiesVersion(user, request);
     }
 
+    @ApiOperation(value = "", notes = "")
+    @GetMapping(value = "/entity/{requestId}/status")
+    public VersionLoadResult getVersionLoadRequestStatus(@ApiParam(value = VC_REQUEST_ID_PARAM_DESCRIPTION, required = true)
+                                                               @PathVariable UUID requestId) throws Exception {
+        accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ);
+        return versionControlService.getVersionLoadStatus(getCurrentUser(), requestId);
+    }
+
 
     @ApiOperation(value = "", notes = "" +
             "```\n[\n" +

application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java

@@ -259,7 +259,8 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
                 SingleEntityVersionLoadRequest versionLoadRequest = (SingleEntityVersionLoadRequest) request;
                 VersionLoadConfig config = versionLoadRequest.getConfig();
                 ListenableFuture<EntityExportData> future = gitServiceQueue.getEntity(user.getTenantId(), request.getVersionId(), versionLoadRequest.getExternalEntityId());
-                DonAsynchron.withCallback(future, entityData -> doInTemplate(ctx, request, c -> loadSingleEntity(c, config, entityData)),
+                DonAsynchron.withCallback(future,
+                        entityData -> doInTemplate(ctx, request, c -> loadSingleEntity(c, config, entityData)),
                         e -> processLoadError(ctx, e), executor);
                 break;
             }
@@ -278,19 +279,16 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
     private <R> VersionLoadResult doInTemplate(EntitiesImportCtx ctx, VersionLoadRequest request, Function<EntitiesImportCtx, VersionLoadResult> function) {
         try {
             VersionLoadResult result = transactionTemplate.execute(status -> function.apply(ctx));
-            try {
+            for (ThrowingRunnable throwingRunnable : ctx.getEventCallbacks()) {
-                for (ThrowingRunnable throwingRunnable : ctx.getEventCallbacks()) {
+                throwingRunnable.run();
-                    throwingRunnable.run();
-                }
-            } catch (ThingsboardException e) {
-                throw new RuntimeException(e);
             }
-            return result;
+            result.setDone(true);
+            return cachePut(ctx.getRequestId(), result);
         } catch (LoadEntityException e) {
-            return onError(e.getData(), e.getCause());
+            return cachePut(ctx.getRequestId(), onError(e.getData(), e.getCause()));
         } catch (Exception e) {
             log.info("[{}] Failed to process request [{}] due to: ", ctx.getTenantId(), request, e);
-            throw e;
+            return cachePut(ctx.getRequestId(), VersionLoadResult.error(EntityLoadError.runtimeError(e.getMessage())));
         }
     }
 
@@ -329,19 +327,21 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
             sw.startNew("Entities " + entityType.name());
             ctx.setSettings(getEntityImportSettings(request, entityType));
             importEntities(ctx, entityType);
+            persistToCache(ctx);
         }
 
         sw.startNew("Reimport");
         reimport(ctx);
+        persistToCache(ctx);
 
         sw.startNew("Remove Others");
         request.getEntityTypes().keySet().stream()
                 .filter(entityType -> request.getEntityTypes().get(entityType).isRemoveOtherEntities())
                 .sorted(exportImportService.getEntityTypeComparatorForImport().reversed())
                 .forEach(entityType -> removeOtherEntities(ctx, entityType));
+        persistToCache(ctx);
 
         sw.startNew("References and Relations");
-
         exportImportService.saveReferencesAndRelations(ctx);
 
         sw.stop();
@@ -430,7 +430,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
     }
 
     private VersionLoadResult onError(EntityExportData<?> entityData, Throwable e) {
-        return analyze(e, entityData).orElseThrow(() -> new RuntimeException(e));
+        return analyze(e, entityData).orElse(VersionLoadResult.error(EntityLoadError.runtimeError(e.getMessage())));
     }
 
     private Optional<VersionLoadResult> analyze(Throwable e, EntityExportData<?> entityData) {
@@ -607,7 +607,14 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
         taskCache.put(requestId, VersionControlTaskCacheEntry.newForExport(result));
     }
 
-    private void cachePut(UUID requestId, VersionLoadResult result) {
+    private VersionLoadResult cachePut(UUID requestId, VersionLoadResult result) {
+        log.debug("[{}] Cache put: {}", requestId, result);
         taskCache.put(requestId, VersionControlTaskCacheEntry.newForImport(result));
+        return result;
     }
+
+    private void persistToCache(EntitiesImportCtx ctx) {
+        cachePut(ctx.getRequestId(), VersionLoadResult.success(new ArrayList<>(ctx.getResults().values())));
+    }
+
 }

application/src/main/resources/thingsboard.yml

@@ -422,7 +422,7 @@ cache:
       timeToLiveInMinutes: "${CACHE_SPECS_TWO_FA_VERIFICATION_CODES_TTL:60}"
       maxSize: "${CACHE_SPECS_TWO_FA_VERIFICATION_CODES_MAX_SIZE:100000}"
     versionControlTask:
-      timeToLiveInMinutes: "${CACHE_SPECS_VERSION_CONTROL_TASK_TTL:60}"
+      timeToLiveInMinutes: "${CACHE_SPECS_VERSION_CONTROL_TASK_TTL:5}"
       maxSize: "${CACHE_SPECS_VERSION_CONTROL_TASK_MAX_SIZE:100000}"
 
 redis:

application/src/test/java/org/thingsboard/server/service/sync/ie/BaseExportImportServiceTest.java

@@ -82,6 +82,7 @@ import org.thingsboard.server.service.sync.vc.data.SimpleEntitiesExportCtx;
 import java.nio.ByteBuffer;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.UUID;
 
 import static org.assertj.core.api.Assertions.assertThat;
 
@@ -433,7 +434,7 @@ public abstract class BaseExportImportServiceTest extends AbstractControllerTest
     }
 
     protected <E extends ExportableEntity<I>, I extends EntityId> EntityImportResult<E> importEntity(User user, EntityExportData<E> exportData, EntityImportSettings importSettings) throws Exception {
-        EntitiesImportCtx ctx = new EntitiesImportCtx(getSecurityUser(user), null, importSettings);
+        EntitiesImportCtx ctx = new EntitiesImportCtx(UUID.randomUUID(), getSecurityUser(user), null, importSettings);
         ctx.setFinalImportAttempt(true);
         exportData = JacksonUtil.treeToValue(JacksonUtil.valueToTree(exportData), EntityExportData.class);
         EntityImportResult<E> importResult = exportImportService.importEntity(ctx, exportData);

common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionLoadResult.java

@@ -34,11 +34,11 @@ public class VersionLoadResult implements Serializable {
     private boolean done;
 
     public static VersionLoadResult success(List<EntityTypeLoadResult> result) {
-        return VersionLoadResult.builder().result(result).done(true).build();
+        return VersionLoadResult.builder().result(result).build();
     }
 
     public static VersionLoadResult success(EntityTypeLoadResult result) {
-        return VersionLoadResult.builder().result(List.of(result)).done(true).build();
+        return VersionLoadResult.builder().result(List.of(result)).build();
     }
 
     public static VersionLoadResult error(EntityLoadError error) {