dendi-wang/thingsboard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #9819 from thingsboard/image-gallery-improvements

Browse Source 
Fixes and improvements for image API
This commit is contained in:
Andrew Shvayka 2023-12-12 18:02:21 +02:00 committed by GitHub
commit cd0d2d4212
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
20 changed files with 66 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
application/src/main/java/org/thingsboard/server/controller/ImageController.java
View File

@ -50,9 +50,10 @@ import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.security.Authority;
import org.thingsboard.server.dao.resource.ImageService;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.dao.resource.ImageCacheKey;
import org.thingsboard.server.dao.resource.ImageService;
import org.thingsboard.server.dao.service.validator.ResourceDataValidator;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.resource.TbImageService;
import org.thingsboard.server.service.security.model.SecurityUser;
import org.thingsboard.server.service.security.permission.Operation;
@ -77,6 +78,8 @@ public class ImageController extends BaseController {
private final ImageService imageService;
private final TbImageService tbImageService;
private final ResourceDataValidator resourceValidator;
@Value("${cache.image.systemImagesBrowserTtlInMinutes:0}")
private int systemImagesBrowserTtlInMinutes;
@Value("${cache.image.tenantImagesBrowserTtlInMinutes:0}")
@ -94,6 +97,7 @@ public class ImageController extends BaseController {
TbResource image = new TbResource();
image.setTenantId(user.getTenantId());
accessControlService.checkPermission(user, Resource.TB_RESOURCE, Operation.CREATE, null, image);
resourceValidator.validateResourceSize(user.getTenantId(), null, file.getSize());
image.setFileName(file.getOriginalFilename());
if (StringUtils.isNotEmpty(title)) {
@ -115,6 +119,8 @@ public class ImageController extends BaseController {
@PathVariable String key,
@RequestPart MultipartFile file) throws Exception {
TbResourceInfo imageInfo = checkImageInfo(type, key, Operation.WRITE);
resourceValidator.validateResourceSize(getTenantId(), imageInfo.getId(), file.getSize());
TbResource image = new TbResource(imageInfo);
image.setData(file.getBytes());
image.setFileName(file.getOriginalFilename());

7
application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java
View File

@ -271,9 +271,8 @@ public class ThingsboardInstallService {
case "3.6.1":
log.info("Upgrading ThingsBoard from version 3.6.1 to 3.6.2 ...");
databaseEntitiesUpgradeService.upgradeDatabase("3.6.1");
installScripts.loadSystemImages();
if (!getEnv("SKIP_IMAGES_MIGRATION", false)) {
installScripts.updateImages();
installScripts.setUpdateImages(true);
} else {
log.info("Skipping images migration. Run the upgrade with fromVersion as '3.6.2-images' to migrate");
}
@ -288,6 +287,10 @@ public class ThingsboardInstallService {
dataUpdateService.upgradeRuleNodes();
systemDataLoaderService.loadSystemWidgets();
installScripts.loadSystemLwm2mResources();
installScripts.loadSystemImages();
if (installScripts.isUpdateImages()) {
installScripts.updateImages();
}
}
log.info("Upgrade finished successfully!");

4
application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java
View File

@ -16,6 +16,8 @@
package org.thingsboard.server.service.install;
import com.fasterxml.jackson.databind.JsonNode;
import lombok.Getter;
import lombok.Setter;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
@ -113,6 +115,8 @@ public class InstallScripts {
@Autowired
private ImagesUpdater imagesUpdater;
@Getter @Setter
private boolean updateImages = false;
Path getTenantRuleChainsDir() {
return Paths.get(getDataDir(), JSON_DIR, TENANT_DIR, RULE_CHAINS_DIR);

9
application/src/main/java/org/thingsboard/server/service/resource/DefaultTbImageService.java
View File

@ -145,6 +145,15 @@ public class DefaultTbImageService extends AbstractTbEntityService implements Tb
TbImageDeleteResult result = imageService.deleteImage(imageInfo, force);
if (result.isSuccess()) {
notificationEntityService.logEntityAction(tenantId, imageId, imageInfo, ActionType.DELETED, user, imageId.toString());
evictETag(new ImageCacheKey(tenantId, imageInfo.getResourceKey(), false));
evictETag(new ImageCacheKey(tenantId, imageInfo.getResourceKey(), true));
clusterService.broadcastToCore(TransportProtos.ToCoreNotificationMsg.newBuilder()
.setResourceCacheInvalidateMsg(TransportProtos.ResourceCacheInvalidateMsg.newBuilder()
.setTenantIdMSB(tenantId.getId().getMostSignificantBits())
.setTenantIdLSB(tenantId.getId().getLeastSignificantBits())
.setResourceKey(imageInfo.getResourceKey())
.build())
.build());
}
return result;
} catch (Exception e) {

4
application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java
View File

@ -65,8 +65,8 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS
private final TbNotificationEntityService entityNotificationService;
protected static final List<EntityType> SUPPORTED_ENTITY_TYPES = List.of(
EntityType.CUSTOMER, EntityType.RULE_CHAIN, EntityType.DASHBOARD,
EntityType.ASSET_PROFILE, EntityType.ASSET,
EntityType.CUSTOMER, EntityType.RULE_CHAIN, EntityType.TB_RESOURCE,
EntityType.DASHBOARD, EntityType.ASSET_PROFILE, EntityType.ASSET,
EntityType.DEVICE_PROFILE, EntityType.DEVICE,
EntityType.ENTITY_VIEW, EntityType.WIDGET_TYPE, EntityType.WIDGETS_BUNDLE,
EntityType.NOTIFICATION_TEMPLATE, EntityType.NOTIFICATION_TARGET, EntityType.NOTIFICATION_RULE

1
application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/AssetProfileExportService.java
View File

@ -34,6 +34,7 @@ public class AssetProfileExportService extends BaseEntityExportService<AssetProf
assetProfile.setDefaultDashboardId(getExternalIdOrElseInternal(ctx, assetProfile.getDefaultDashboardId()));
assetProfile.setDefaultRuleChainId(getExternalIdOrElseInternal(ctx, assetProfile.getDefaultRuleChainId()));
assetProfile.setDefaultEdgeRuleChainId(getExternalIdOrElseInternal(ctx, assetProfile.getDefaultEdgeRuleChainId()));
imageService.inlineImage(assetProfile);
}
@Override

1
application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DashboardExportService.java
View File

@ -48,6 +48,7 @@ public class DashboardExportService extends BaseEntityExportService<DashboardId,
for (JsonNode widgetConfig : dashboard.getWidgetsConfig()) {
replaceUuidsRecursively(ctx, JacksonUtil.getSafely(widgetConfig, "config", "actions"), Collections.emptySet(), WIDGET_CONFIG_PROCESSED_FIELDS_PATTERN);
}
imageService.inlineImages(dashboard);
}
@Override

3
application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java
View File

@ -31,6 +31,7 @@ import org.thingsboard.server.common.data.sync.ie.AttributeExportData;
import org.thingsboard.server.common.data.sync.ie.EntityExportData;
import org.thingsboard.server.dao.attributes.AttributesService;
import org.thingsboard.server.dao.relation.RelationDao;
import org.thingsboard.server.dao.resource.ImageService;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.sync.ie.exporting.EntityExportService;
import org.thingsboard.server.service.sync.ie.exporting.ExportableEntitiesService;
@ -58,6 +59,8 @@ public class DefaultEntityExportService<I extends EntityId, E extends Exportable
private RelationDao relationDao;
@Autowired
private AttributesService attributesService;
@Autowired
protected ImageService imageService;
@Override
public final D getExportData(EntitiesExportCtx<?> ctx, I entityId) throws ThingsboardException {

1
application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DeviceProfileExportService.java
View File

@ -34,6 +34,7 @@ public class DeviceProfileExportService extends BaseEntityExportService<DevicePr
deviceProfile.setDefaultDashboardId(getExternalIdOrElseInternal(ctx, deviceProfile.getDefaultDashboardId()));
deviceProfile.setDefaultRuleChainId(getExternalIdOrElseInternal(ctx, deviceProfile.getDefaultRuleChainId()));
deviceProfile.setDefaultEdgeRuleChainId(getExternalIdOrElseInternal(ctx, deviceProfile.getDefaultEdgeRuleChainId()));
imageService.inlineImage(deviceProfile);
}
@Override

5
application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/WidgetTypeExportService.java
View File

@ -32,10 +32,11 @@ import java.util.Set;
public class WidgetTypeExportService extends BaseEntityExportService<WidgetTypeId, WidgetTypeDetails, WidgetTypeExportData> {
@Override
protected void setRelatedEntities(EntitiesExportCtx<?> ctx, WidgetTypeDetails widgetsBundle, WidgetTypeExportData exportData) {
if (widgetsBundle.getTenantId() == null || widgetsBundle.getTenantId().isNullUid()) {
protected void setRelatedEntities(EntitiesExportCtx<?> ctx, WidgetTypeDetails widgetTypeDetails, WidgetTypeExportData exportData) {
if (widgetTypeDetails.getTenantId() == null || widgetTypeDetails.getTenantId().isNullUid()) {
throw new IllegalArgumentException("Export of system Widget Type is not allowed");
}
imageService.inlineImages(widgetTypeDetails);
}
@Override

1
application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/WidgetsBundleExportService.java
View File

@ -41,6 +41,7 @@ public class WidgetsBundleExportService extends BaseEntityExportService<WidgetsB
if (widgetsBundle.getTenantId() == null || widgetsBundle.getTenantId().isNullUid()) {
throw new IllegalArgumentException("Export of system Widget Bundles is not allowed");
}
imageService.inlineImage(widgetsBundle);
List<String> fqns = widgetTypeService.findWidgetFqnsByWidgetsBundleId(ctx.getTenantId(), widgetsBundle.getId());
exportData.setFqns(fqns);

1
common/data/src/main/java/org/thingsboard/server/common/data/DashboardInfo.java
View File

@ -38,7 +38,6 @@ public class DashboardInfo extends BaseData<DashboardId> implements HasName, Has
@NoXss
@Length(fieldName = "title")
private String title;
@Length(fieldName = "image", max = 1000000)
private String image;
@Valid
private Set<ShortCustomerInfo> assignedCustomers;

1
common/data/src/main/java/org/thingsboard/server/common/data/DeviceProfile.java
View File

@ -54,7 +54,6 @@ public class DeviceProfile extends BaseData<DeviceProfileId> implements HasName,
@NoXss
@ApiModelProperty(position = 11, value = "Device Profile description. ")
private String description;
@Length(fieldName = "image", max = 1000000)
@ApiModelProperty(position = 12, value = "Either URL or Base64 data of the icon. Used in the mobile application to visualize set of device profiles in the grid view. ")
private String image;
private boolean isDefault;

1
common/data/src/main/java/org/thingsboard/server/common/data/asset/AssetProfile.java
View File

@ -52,7 +52,6 @@ public class AssetProfile extends BaseData<AssetProfileId> implements HasName, H
@NoXss
@ApiModelProperty(position = 11, value = "Asset Profile description. ")
private String description;
@Length(fieldName = "image", max = 1000000)
@ApiModelProperty(position = 12, value = "Either URL or Base64 data of the icon. Used in the mobile application to visualize set of asset profiles in the grid view. ")
private String image;
private boolean isDefault;

1
common/data/src/main/java/org/thingsboard/server/common/data/widget/WidgetTypeDetails.java
View File

@ -32,7 +32,6 @@ import org.thingsboard.server.common.data.validation.NoXss;
@JsonPropertyOrder({ "fqn", "name", "deprecated", "image", "description", "descriptor", "externalId" })
public class WidgetTypeDetails extends WidgetType implements HasName, HasTenantId, HasImage, ExportableEntity<WidgetTypeId> {
@Length(fieldName = "image", max = 1000000)
@ApiModelProperty(position = 9, value = "Relative or external image URL. Replaced with image data URL (Base64) in case of relative URL and 'inlineImages' option enabled.")
private String image;
@NoXss

2
common/data/src/main/java/org/thingsboard/server/common/data/widget/WidgetsBundle.java
View File

@ -57,7 +57,6 @@ public class WidgetsBundle extends BaseData<WidgetsBundleId> implements HasName,
@ApiModelProperty(position = 5, value = "Title used in search and UI", accessMode = ApiModelProperty.AccessMode.READ_ONLY)
private String title;
@Length(fieldName = "image", max = 1000000)
@Getter
@Setter
@ApiModelProperty(position = 6, value = "Relative or external image URL. Replaced with image data URL (Base64) in case of relative URL and 'inlineImages' option enabled.", accessMode = ApiModelProperty.AccessMode.READ_ONLY)
@ -66,7 +65,6 @@ public class WidgetsBundle extends BaseData<WidgetsBundleId> implements HasName,
@NoXss
@Length(fieldName = "description", max = 1024)
@Getter
@Setter
@ApiModelProperty(position = 7, value = "Description", accessMode = ApiModelProperty.AccessMode.READ_ONLY)
private String description;

9
dao/src/main/java/org/thingsboard/server/dao/resource/BaseImageService.java
View File

@ -27,7 +27,6 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Propagation;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.util.Base64Utils;
import org.thingsboard.common.util.JacksonUtil;
@ -274,7 +273,7 @@ public class BaseImageService extends BaseResourceService implements ImageServic
return resourceInfoDao.findSystemOrTenantImageByEtag(tenantId, ResourceType.IMAGE, etag);
}
@Transactional(propagation = Propagation.NOT_SUPPORTED)// we don't want transaction to rollback in case of an image processing failure
@Transactional(noRollbackFor = Exception.class) // we don't want transaction to rollback in case of an image processing failure
@Override
public boolean replaceBase64WithImageUrl(HasImage entity, String type) {
log.trace("Executing replaceBase64WithImageUrl [{}] [{}] [{}]", entity.getTenantId(), type, entity.getName());
@ -289,7 +288,7 @@ public class BaseImageService extends BaseResourceService implements ImageServic
return result.isUpdated();
}
@Transactional(propagation = Propagation.NOT_SUPPORTED)// we don't want transaction to rollback in case of an image processing failure
@Transactional(noRollbackFor = Exception.class) // we don't want transaction to rollback in case of an image processing failure
@Override
public boolean replaceBase64WithImageUrl(WidgetTypeDetails entity) {
log.trace("Executing replaceBase64WithImageUrl [{}] [WidgetTypeDetails] [{}]", entity.getTenantId(), entity.getId());
@ -315,7 +314,7 @@ public class BaseImageService extends BaseResourceService implements ImageServic
return updated;
}
@Transactional(propagation = Propagation.NOT_SUPPORTED)// we don't want transaction to rollback in case of an image processing failure
@Transactional(noRollbackFor = Exception.class) // we don't want transaction to rollback in case of an image processing failure
@Override
public boolean replaceBase64WithImageUrl(Dashboard entity) {
log.trace("Executing replaceBase64WithImageUrl [{}] [Dashboard] [{}]", entity.getTenantId(), entity.getId());
@ -463,6 +462,8 @@ public class BaseImageService extends BaseResourceService implements ImageServic
}
return UpdateResult.of(false, data);
}
} else {
log.debug("[{}] Using existing image {} ({} - '{}') for '{}'", tenantId, imageInfo.getResourceKey(), imageInfo.getTenantId(), imageInfo.getName(), name);
}
return UpdateResult.of(true, DataConstants.TB_IMAGE_PREFIX + imageInfo.getLink());
}

3
dao/src/main/java/org/thingsboard/server/dao/service/DataValidator.java
View File

@ -17,6 +17,7 @@ package org.thingsboard.server.dao.service;
import com.fasterxml.jackson.databind.JsonNode;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FileUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Lazy;
import org.thingsboard.server.common.data.BaseData;
@ -129,7 +130,7 @@ public abstract class DataValidator<D extends BaseData<?>> {
EntityType entityType) {
if (maxSumDataSize > 0) {
if (dataDao.sumDataSizeByTenantId(tenantId) + currentDataSize > maxSumDataSize) {
throw new DataValidationException(String.format("%ss total size exceeds the maximum of " + maxSumDataSize + " bytes", entityType.getNormalName()));
throw new DataValidationException(String.format("%ss total size exceeds the maximum of " + FileUtils.byteCountToDisplaySize(maxSumDataSize), entityType.getNormalName()));
}
}
}

33
dao/src/main/java/org/thingsboard/server/dao/service/validator/ResourceDataValidator.java
View File

@ -15,12 +15,14 @@
*/
package org.thingsboard.server.dao.service.validator;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Component;
import org.thingsboard.server.common.data.TbResource;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.TbResourceId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration;
import org.thingsboard.server.common.data.widget.BaseWidgetType;
@ -83,19 +85,8 @@ public class ResourceDataValidator extends DataValidator<TbResource> {
if (resource.getResourceType() == null) {
throw new DataValidationException("Resource type should be specified!");
}
if (!resource.getTenantId().isSysTenantId() && resource.getData() != null) {
DefaultTenantProfileConfiguration profileConfiguration = tenantProfileCache.get(tenantId).getDefaultProfileConfiguration();
long maxResourceSize = profileConfiguration.getMaxResourceSize();
if (maxResourceSize > 0 && resource.getData().length > maxResourceSize) {
throw new IllegalArgumentException("Resource exceeds the maximum size of " + maxResourceSize + " bytes");
}
long maxSumResourcesDataInBytes = profileConfiguration.getMaxResourcesInBytes();
int dataSize = resource.getData().length;
if (resource.getId() != null) {
long prevSize = resourceDao.getResourceSize(tenantId, resource.getId());
dataSize -= prevSize;
}
validateMaxSumDataSizePerTenant(tenantId, resourceDao, maxSumResourcesDataInBytes, dataSize, TB_RESOURCE);
if (resource.getData() != null) {
validateResourceSize(resource.getTenantId(), resource.getId(), resource.getData().length);
}
if (StringUtils.isEmpty(resource.getFileName())) {
throw new DataValidationException("Resource file name should be specified!");
@ -108,6 +99,22 @@ public class ResourceDataValidator extends DataValidator<TbResource> {
}
}
public void validateResourceSize(TenantId tenantId, TbResourceId resourceId, long dataSize) {
if (!tenantId.isSysTenantId()) {
DefaultTenantProfileConfiguration profileConfiguration = tenantProfileCache.get(tenantId).getDefaultProfileConfiguration();
long maxResourceSize = profileConfiguration.getMaxResourceSize();
if (maxResourceSize > 0 && dataSize > maxResourceSize) {
throw new IllegalArgumentException("Resource exceeds the maximum size of " + FileUtils.byteCountToDisplaySize(maxResourceSize));
}
long maxSumResourcesDataInBytes = profileConfiguration.getMaxResourcesInBytes();
if (resourceId != null) {
long prevSize = resourceDao.getResourceSize(tenantId, resourceId);
dataSize -= prevSize;
}
validateMaxSumDataSizePerTenant(tenantId, resourceDao, maxSumResourcesDataInBytes, dataSize, TB_RESOURCE);
}
}
@Override
public void validateDelete(TenantId tenantId, EntityId resourceId) {
List<WidgetTypeDetails> widgets = widgetTypeDao.findWidgetTypesInfosByTenantIdAndResourceId(tenantId.getId(),

2
dao/src/main/java/org/thingsboard/server/dao/sql/resource/TbResourceInfoRepository.java
View File

@ -66,7 +66,7 @@ public interface TbResourceInfoRepository extends JpaRepository<TbResourceInfoEn
List<TbResourceInfoEntity> findByTenantIdAndEtagAndResourceKeyStartingWith(UUID tenantId, String etag, String query);
@Query(value = "SELECT * FROM resource r WHERE (r.tenant_id = '13814000-1dd2-11b2-8080-808080808080' OR r.tenant_id = :tenantId) " +
"AND r.resource_type = :resourceType AND r.etag = :etag LIMIT 1", nativeQuery = true)
"AND r.resource_type = :resourceType AND r.etag = :etag ORDER BY created_time, id LIMIT 1", nativeQuery = true)
TbResourceInfoEntity findSystemOrTenantImageByEtag(@Param("tenantId") UUID tenantId,
@Param("resourceType") String resourceType,
@Param("etag") String etag);