dendi-wang/thingsboard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #9897 from thingsboard/fix/image-vc

Browse Source 
Fixes for images version control
This commit is contained in:
Andrew Shvayka 2023-12-22 17:00:57 +02:00 committed by GitHub
commit 86ecc6f742
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 52 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/ResourceExportService.java
View File

@ -18,9 +18,11 @@ package org.thingsboard.server.service.sync.ie.exporting.impl;
import org.springframework.stereotype.Service;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.TbResource;
import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.TbResourceId;
import org.thingsboard.server.common.data.sync.ie.EntityExportData;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx;
import java.util.Set;
@ -28,6 +30,12 @@ import java.util.Set;
@TbCoreComponent
public class ResourceExportService extends BaseEntityExportService<TbResourceId, TbResource, EntityExportData<TbResource>> {
@Override
protected void setAdditionalExportData(EntitiesExportCtx<?> ctx, TbResource resource, EntityExportData<TbResource> exportData) throws ThingsboardException {
super.setAdditionalExportData(ctx, resource, exportData);
resource.setPreview(null); // will be generated on import
}
@Override
public Set<EntityType> getSupportedEntityTypes() {
return Set.of(EntityType.TB_RESOURCE);

12
application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/ResourceImportService.java
View File

@ -18,12 +18,14 @@ package org.thingsboard.server.service.sync.ie.importing.impl;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Service;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.ResourceType;
import org.thingsboard.server.common.data.TbResource;
import org.thingsboard.server.common.data.User;
import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.TbResourceId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.sync.ie.EntityExportData;
import org.thingsboard.server.dao.resource.ImageService;
import org.thingsboard.server.dao.resource.ResourceService;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx;
@ -34,6 +36,7 @@ import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx;
public class ResourceImportService extends BaseEntityImportService<TbResourceId, TbResource, EntityExportData<TbResource>> {
private final ResourceService resourceService;
private final ImageService imageService;
@Override
protected void setOwner(TenantId tenantId, TbResource resource, IdProvider idProvider) {
@ -66,7 +69,14 @@ public class ResourceImportService extends BaseEntityImportService<TbResourceId,
@Override
protected TbResource saveOrUpdate(EntitiesImportCtx ctx, TbResource resource, EntityExportData<TbResource> exportData, IdProvider idProvider) {
return resourceService.saveResource(resource);
if (resource.getResourceType() == ResourceType.IMAGE) {
return new TbResource(imageService.saveImage(resource));
} else {
resource = resourceService.saveResource(resource);
resource.setData(null);
resource.setPreview(null);
return resource;
}
}
@Override

18
application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java
View File

@ -287,7 +287,15 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
private <R> VersionLoadResult doInTemplate(EntitiesImportCtx ctx, VersionLoadRequest request, Function<EntitiesImportCtx, VersionLoadResult> function) {
try {
VersionLoadResult result = transactionTemplate.execute(status -> function.apply(ctx));
VersionLoadResult result = transactionTemplate.execute(status -> {
try {
return function.apply(ctx);
} catch (RuntimeException e) {
throw e;
} catch (Exception e) {
throw new RuntimeException(e); // to prevent UndeclaredThrowableException
}
});
for (ThrowingRunnable throwingRunnable : ctx.getEventCallbacks()) {
throwingRunnable.run();
}
@ -355,9 +363,9 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
sw.stop();
for (var task : sw.getTaskInfo()) {
log.info("[{}] Executed: {} in {}ms", ctx.getTenantId(), task.getTaskName(), task.getTimeMillis());
log.debug("[{}] Executed: {} in {}ms", ctx.getTenantId(), task.getTaskName(), task.getTimeMillis());
}
log.info("[{}] Total time: {}ms", ctx.getTenantId(), sw.getTotalTimeMillis());
log.debug("[{}] Total time: {}ms", ctx.getTenantId(), sw.getTotalTimeMillis());
return VersionLoadResult.success(new ArrayList<>(ctx.getResults().values()));
}
@ -380,8 +388,8 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
do {
try {
entityDataList = gitServiceQueue.getEntities(ctx.getTenantId(), ctx.getVersionId(), entityType, offset, limit).get();
} catch (InterruptedException | ExecutionException e) {
throw new RuntimeException(e);
} catch (ExecutionException e) {
throw e.getCause();
}
log.debug("[{}] Loading {} entities pack ({})", ctx.getTenantId(), entityType, entityDataList.size());
for (EntityExportData entityData : entityDataList) {

10
common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java
View File

@ -24,6 +24,7 @@ import com.google.common.util.concurrent.MoreExecutors;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.checkerframework.checker.nullness.qual.Nullable;
import org.eclipse.jgit.errors.LargeObjectException;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.context.event.ApplicationReadyEvent;
import org.springframework.context.event.EventListener;
@ -261,7 +262,7 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe
}
}
} catch (Exception e) {
reply(ctx, Optional.of(e));
reply(ctx, Optional.of(handleError(e)));
} finally {
lock.unlock();
}
@ -496,6 +497,13 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe
}
}
private Exception handleError(Exception e) {
if (e instanceof LargeObjectException) {
return new RuntimeException("Version is too big");
}
return e;
}
private void reply(VersionControlRequestCtx ctx, VersionCreationResult result) {
var responseBuilder = CommitResponseMsg.newBuilder().setAdded(result.getAdded())
.setModified(result.getModified())

9
common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java
View File

@ -39,6 +39,7 @@ import org.eclipse.jgit.diff.EditList;
import org.eclipse.jgit.diff.HistogramDiff;
import org.eclipse.jgit.diff.RawText;
import org.eclipse.jgit.diff.RawTextComparator;
import org.eclipse.jgit.errors.LargeObjectException;
import org.eclipse.jgit.lib.Constants;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.ObjectLoader;
@ -236,8 +237,12 @@ public class GitRepository {
ObjectId blobId = treeWalk.getObjectId(0);
try (ObjectReader objectReader = git.getRepository().newObjectReader()) {
ObjectLoader objectLoader = objectReader.open(blobId);
byte[] bytes = objectLoader.getBytes();
return new String(bytes, StandardCharsets.UTF_8);
try {
byte[] bytes = objectLoader.getBytes();
return new String(bytes, StandardCharsets.UTF_8);
} catch (LargeObjectException e) {
throw new RuntimeException("File " + file + " is too big to load");
}
}
}
}

10
dao/src/main/java/org/thingsboard/server/dao/resource/BaseImageService.java
View File

@ -150,13 +150,11 @@ public class BaseImageService extends BaseResourceService implements ImageServic
image.setDescriptorValue(descriptor);
image.setPreview(result.getRight());
if (image.getId() == null) {
if (StringUtils.isEmpty(image.getPublicResourceKey())) {
if (StringUtils.isEmpty(image.getPublicResourceKey())) {
image.setPublicResourceKey(generatePublicResourceKey());
} else {
if (resourceInfoDao.existsByPublicResourceKey(ResourceType.IMAGE, image.getPublicResourceKey())) {
image.setPublicResourceKey(generatePublicResourceKey());
} else {
if (resourceInfoDao.existsByPublicResourceKey(ResourceType.IMAGE, image.getPublicResourceKey())) {
image.setPublicResourceKey(generatePublicResourceKey());
}
}
}
log.debug("[{}] Creating image {} ('{}')", image.getTenantId(), image.getResourceKey(), image.getName());