Merge pull request #9897 from thingsboard/fix/image-vc

Fixes for images version control

application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/ResourceExportService.java

@@ -18,9 +18,11 @@ package org.thingsboard.server.service.sync.ie.exporting.impl;
 import org.springframework.stereotype.Service;
 import org.thingsboard.server.common.data.EntityType;
 import org.thingsboard.server.common.data.TbResource;
+import org.thingsboard.server.common.data.exception.ThingsboardException;
 import org.thingsboard.server.common.data.id.TbResourceId;
 import org.thingsboard.server.common.data.sync.ie.EntityExportData;
 import org.thingsboard.server.queue.util.TbCoreComponent;
+import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx;
 
 import java.util.Set;
 
@@ -28,6 +30,12 @@ import java.util.Set;
 @TbCoreComponent
 public class ResourceExportService extends BaseEntityExportService<TbResourceId, TbResource, EntityExportData<TbResource>> {
 
+    @Override
+    protected void setAdditionalExportData(EntitiesExportCtx<?> ctx, TbResource resource, EntityExportData<TbResource> exportData) throws ThingsboardException {
+        super.setAdditionalExportData(ctx, resource, exportData);
+        resource.setPreview(null); // will be generated on import
+    }
+
     @Override
     public Set<EntityType> getSupportedEntityTypes() {
         return Set.of(EntityType.TB_RESOURCE);

application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/ResourceImportService.java

@@ -18,12 +18,14 @@ package org.thingsboard.server.service.sync.ie.importing.impl;
 import lombok.RequiredArgsConstructor;
 import org.springframework.stereotype.Service;
 import org.thingsboard.server.common.data.EntityType;
+import org.thingsboard.server.common.data.ResourceType;
 import org.thingsboard.server.common.data.TbResource;
 import org.thingsboard.server.common.data.User;
 import org.thingsboard.server.common.data.exception.ThingsboardException;
 import org.thingsboard.server.common.data.id.TbResourceId;
 import org.thingsboard.server.common.data.id.TenantId;
 import org.thingsboard.server.common.data.sync.ie.EntityExportData;
+import org.thingsboard.server.dao.resource.ImageService;
 import org.thingsboard.server.dao.resource.ResourceService;
 import org.thingsboard.server.queue.util.TbCoreComponent;
 import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx;
@@ -34,6 +36,7 @@ import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx;
 public class ResourceImportService extends BaseEntityImportService<TbResourceId, TbResource, EntityExportData<TbResource>> {
 
     private final ResourceService resourceService;
+    private final ImageService imageService;
 
     @Override
     protected void setOwner(TenantId tenantId, TbResource resource, IdProvider idProvider) {
@@ -66,7 +69,14 @@ public class ResourceImportService extends BaseEntityImportService<TbResourceId,
 
     @Override
     protected TbResource saveOrUpdate(EntitiesImportCtx ctx, TbResource resource, EntityExportData<TbResource> exportData, IdProvider idProvider) {
-        return resourceService.saveResource(resource);
+        if (resource.getResourceType() == ResourceType.IMAGE) {
+            return new TbResource(imageService.saveImage(resource));
+        } else {
+            resource = resourceService.saveResource(resource);
+            resource.setData(null);
+            resource.setPreview(null);
+            return resource;
+        }
     }
 
     @Override

application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java

@@ -287,7 +287,15 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
 
     private <R> VersionLoadResult doInTemplate(EntitiesImportCtx ctx, VersionLoadRequest request, Function<EntitiesImportCtx, VersionLoadResult> function) {
         try {
-            VersionLoadResult result = transactionTemplate.execute(status -> function.apply(ctx));
+            VersionLoadResult result = transactionTemplate.execute(status -> {
+                try {
+                    return function.apply(ctx);
+                } catch (RuntimeException e) {
+                    throw e;
+                } catch (Exception e) {
+                    throw new RuntimeException(e); // to prevent UndeclaredThrowableException
+                }
+            });
             for (ThrowingRunnable throwingRunnable : ctx.getEventCallbacks()) {
                 throwingRunnable.run();
             }
@@ -355,9 +363,9 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
 
         sw.stop();
         for (var task : sw.getTaskInfo()) {
-            log.info("[{}] Executed: {} in {}ms", ctx.getTenantId(), task.getTaskName(), task.getTimeMillis());
+            log.debug("[{}] Executed: {} in {}ms", ctx.getTenantId(), task.getTaskName(), task.getTimeMillis());
         }
-        log.info("[{}] Total time: {}ms", ctx.getTenantId(), sw.getTotalTimeMillis());
+        log.debug("[{}] Total time: {}ms", ctx.getTenantId(), sw.getTotalTimeMillis());
         return VersionLoadResult.success(new ArrayList<>(ctx.getResults().values()));
     }
 
@@ -380,8 +388,8 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
         do {
             try {
                 entityDataList = gitServiceQueue.getEntities(ctx.getTenantId(), ctx.getVersionId(), entityType, offset, limit).get();
-            } catch (InterruptedException | ExecutionException e) {
+            } catch (ExecutionException e) {
-                throw new RuntimeException(e);
+                throw e.getCause();
             }
             log.debug("[{}] Loading {} entities pack ({})", ctx.getTenantId(), entityType, entityDataList.size());
             for (EntityExportData entityData : entityDataList) {

common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java

@@ -24,6 +24,7 @@ import com.google.common.util.concurrent.MoreExecutors;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.checkerframework.checker.nullness.qual.Nullable;
+import org.eclipse.jgit.errors.LargeObjectException;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.boot.context.event.ApplicationReadyEvent;
 import org.springframework.context.event.EventListener;
@@ -261,7 +262,7 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe
                 }
             }
         } catch (Exception e) {
-            reply(ctx, Optional.of(e));
+            reply(ctx, Optional.of(handleError(e)));
         } finally {
             lock.unlock();
         }
@@ -496,6 +497,13 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe
         }
     }
 
+    private Exception handleError(Exception e) {
+        if (e instanceof LargeObjectException) {
+            return new RuntimeException("Version is too big");
+        }
+        return e;
+    }
+
     private void reply(VersionControlRequestCtx ctx, VersionCreationResult result) {
         var responseBuilder = CommitResponseMsg.newBuilder().setAdded(result.getAdded())
                 .setModified(result.getModified())

common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java

@@ -39,6 +39,7 @@ import org.eclipse.jgit.diff.EditList;
 import org.eclipse.jgit.diff.HistogramDiff;
 import org.eclipse.jgit.diff.RawText;
 import org.eclipse.jgit.diff.RawTextComparator;
+import org.eclipse.jgit.errors.LargeObjectException;
 import org.eclipse.jgit.lib.Constants;
 import org.eclipse.jgit.lib.ObjectId;
 import org.eclipse.jgit.lib.ObjectLoader;
@@ -236,8 +237,12 @@ public class GitRepository {
             ObjectId blobId = treeWalk.getObjectId(0);
             try (ObjectReader objectReader = git.getRepository().newObjectReader()) {
                 ObjectLoader objectLoader = objectReader.open(blobId);
+                try {
                     byte[] bytes = objectLoader.getBytes();
                     return new String(bytes, StandardCharsets.UTF_8);
+                } catch (LargeObjectException e) {
+                    throw new RuntimeException("File " + file + " is too big to load");
+                }
             }
         }
     }

dao/src/main/java/org/thingsboard/server/dao/resource/BaseImageService.java

@@ -150,7 +150,6 @@ public class BaseImageService extends BaseResourceService implements ImageServic
         image.setDescriptorValue(descriptor);
         image.setPreview(result.getRight());
 
-        if (image.getId() == null) {
         if (StringUtils.isEmpty(image.getPublicResourceKey())) {
             image.setPublicResourceKey(generatePublicResourceKey());
         } else {
@@ -158,7 +157,6 @@ public class BaseImageService extends BaseResourceService implements ImageServic
                 image.setPublicResourceKey(generatePublicResourceKey());
             }
         }
-        }
         log.debug("[{}] Creating image {} ('{}')", image.getTenantId(), image.getResourceKey(), image.getName());
         return new TbResourceInfo(doSaveResource(image));
     }