dendi-wang/thingsboard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixes and improvements for images processing

Browse Source
This commit is contained in:
ViacheslavKlimov 2023-11-30 20:49:09 +02:00
parent ddfa55ba6e
commit 2d26fdcb95
8 changed files with 139 additions and 109 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

131
application/src/main/java/org/thingsboard/server/service/install/update/ImagesUpdater.java
View File

@ -18,15 +18,11 @@ package org.thingsboard.server.service.install.update;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import org.thingsboard.server.common.data.Dashboard;
import org.thingsboard.server.common.data.DeviceProfile;
import org.thingsboard.server.common.data.asset.AssetProfile;
import org.thingsboard.server.common.data.id.DashboardId;
import org.thingsboard.server.common.data.HasImage;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.id.WidgetTypeId;
import org.thingsboard.server.common.data.page.PageDataIterable;
import org.thingsboard.server.common.data.widget.WidgetTypeDetails;
import org.thingsboard.server.common.data.widget.WidgetsBundle;
import org.thingsboard.server.dao.Dao;
import org.thingsboard.server.dao.asset.AssetProfileDao;
import org.thingsboard.server.dao.dashboard.DashboardDao;
import org.thingsboard.server.dao.device.DeviceProfileDao;
@ -34,6 +30,9 @@ import org.thingsboard.server.dao.resource.ImageService;
import org.thingsboard.server.dao.widget.WidgetTypeDao;
import org.thingsboard.server.dao.widget.WidgetsBundleDao;
import java.util.function.BiFunction;
import java.util.function.Function;
@Component
@RequiredArgsConstructor
@Slf4j
@ -48,108 +47,78 @@ public class ImagesUpdater {
public void updateWidgetsBundlesImages() {
log.info("Updating widgets bundles images...");
var widgetsBundles = new PageDataIterable<>(widgetsBundleDao::findAllWidgetsBundles, 128);
int updatedCount = 0;
int totalCount = 0;
for (WidgetsBundle widgetsBundle : widgetsBundles) {
totalCount++;
try {
boolean updated = imageService.replaceBase64WithImageUrl(widgetsBundle, "bundle");
if (updated) {
widgetsBundleDao.save(widgetsBundle.getTenantId(), widgetsBundle);
log.debug("[{}][{}][{}] Updated widgets bundle images", widgetsBundle.getTenantId(), widgetsBundle.getId(), widgetsBundle.getTitle());
updatedCount++;
}
} catch (Exception e) {
log.error("[{}][{}][{}] Failed to update widgets bundle images", widgetsBundle.getTenantId(), widgetsBundle.getId(), widgetsBundle.getTitle(), e);
}
}
log.info("Updated {} widgets bundles out of {}", updatedCount, totalCount);
updateImages(widgetsBundles, "bundle", imageService::replaceBase64WithImageUrl, widgetsBundleDao);
}
public void updateWidgetTypesImages() {
log.info("Updating widget types images...");
var widgetTypes = new PageDataIterable<>(widgetTypeDao::findAllWidgetTypesIds, 1024);
int updatedCount = 0;
int totalCount = 0;
for (WidgetTypeId widgetTypeId : widgetTypes) {
totalCount++;
WidgetTypeDetails widgetTypeDetails = widgetTypeDao.findById(TenantId.SYS_TENANT_ID, widgetTypeId.getId());
try {
boolean updated = imageService.replaceBase64WithImageUrl(widgetTypeDetails);
if (updated) {
widgetTypeDao.save(widgetTypeDetails.getTenantId(), widgetTypeDetails);
log.debug("[{}][{}][{}] Updated widget type images", widgetTypeDetails.getTenantId(), widgetTypeDetails.getId(), widgetTypeDetails.getName());
updatedCount++;
}
} catch (Exception e) {
log.error("[{}][{}][{}] Failed to update widget type images", widgetTypeDetails.getTenantId(), widgetTypeDetails.getId(), widgetTypeDetails.getName(), e);
}
}
log.info("Updated {} widget types out of {}", updatedCount, totalCount);
var widgetTypesIds = new PageDataIterable<>(widgetTypeDao::findAllWidgetTypesIds, 1024);
updateImages(widgetTypesIds, "widget type", imageService::replaceBase64WithImageUrl, widgetTypeDao);
}
public void updateDashboardsImages() {
log.info("Updating dashboards images...");
var dashboards = new PageDataIterable<>(dashboardDao::findAllIds, 1024);
int updatedCount = 0;
int totalCount = 0;
for (DashboardId dashboardId : dashboards) {
totalCount++;
Dashboard dashboard = dashboardDao.findById(TenantId.SYS_TENANT_ID, dashboardId.getId());
try {
boolean updated = imageService.replaceBase64WithImageUrl(dashboard);
if (updated) {
dashboardDao.save(dashboard.getTenantId(), dashboard);
log.info("[{}][{}][{}] Updated dashboard images", dashboard.getTenantId(), dashboardId, dashboard.getTitle());
updatedCount++;
}
} catch (Exception e) {
log.error("[{}][{}][{}] Failed to update dashboard images", dashboard.getTenantId(), dashboardId, dashboard.getTitle(), e);
}
}
log.info("Updated {} dashboards out of {}", updatedCount, totalCount);
var dashboardsIds = new PageDataIterable<>(dashboardDao::findAllIds, 1024);
updateImages(dashboardsIds, "dashboard", imageService::replaceBase64WithImageUrl, dashboardDao);
}
public void updateDeviceProfilesImages() {
log.info("Updating device profiles images...");
var deviceProfiles = new PageDataIterable<>(deviceProfileDao::findAll, 256);
int updatedCount = 0;
int totalCount = 0;
for (DeviceProfile deviceProfile : deviceProfiles) {
totalCount++;
try {
boolean updated = imageService.replaceBase64WithImageUrl(deviceProfile, "device profile");
if (updated) {
deviceProfileDao.save(deviceProfile.getTenantId(), deviceProfile);
log.debug("[{}][{}][{}] Updated device profile images", deviceProfile.getTenantId(), deviceProfile.getId(), deviceProfile.getName());
updatedCount++;
}
} catch (Exception e) {
log.error("[{}][{}][{}] Failed to update device profile images", deviceProfile.getTenantId(), deviceProfile.getId(), deviceProfile.getName(), e);
}
}
log.info("Updated {} device profiles out of {}", updatedCount, totalCount);
updateImages(deviceProfiles, "device profile", imageService::replaceBase64WithImageUrl, deviceProfileDao);
}
public void updateAssetProfilesImages() {
log.info("Updating asset profiles images...");
var assetProfiles = new PageDataIterable<>(assetProfileDao::findAll, 256);
updateImages(assetProfiles, "asset profile", imageService::replaceBase64WithImageUrl, assetProfileDao);
}
private <E extends HasImage> void updateImages(Iterable<E> entities, String type,
BiFunction<E, String, Boolean> updater, Dao<E> dao) {
int updatedCount = 0;
int totalCount = 0;
for (AssetProfile assetProfile : assetProfiles) {
for (E entity : entities) {
totalCount++;
try {
boolean updated = imageService.replaceBase64WithImageUrl(assetProfile, "asset profile");
boolean updated = updater.apply(entity, type);
if (updated) {
assetProfileDao.save(assetProfile.getTenantId(), assetProfile);
log.debug("[{}][{}][{}] Updated asset profile images", assetProfile.getTenantId(), assetProfile.getId(), assetProfile.getName());
dao.save(entity.getTenantId(), entity);
log.debug("[{}][{}] Updated {} images", entity.getTenantId(), entity.getName(), type);
updatedCount++;
}
} catch (Exception e) {
log.error("[{}][{}][{}] Failed to update asset profile images", assetProfile.getTenantId(), assetProfile.getId(), assetProfile.getName(), e);
log.error("[{}][{}] Failed to update {} images", entity.getTenantId(), entity.getName(), type, e);
}
if (totalCount % 100 == 0) {
log.info("Processed {} {}s so far", totalCount, type);
}
}
log.info("Updated {} asset profiles out of {}", updatedCount, totalCount);
log.info("Updated {} {}s out of {}", updatedCount, type, totalCount);
}
private <E extends HasImage> void updateImages(Iterable<? extends EntityId> entitiesIds, String type,
Function<E, Boolean> updater, Dao<E> dao) {
int updatedCount = 0;
int totalCount = 0;
for (EntityId id : entitiesIds) {
totalCount++;
E entity = dao.findById(TenantId.SYS_TENANT_ID, id.getId());
try {
boolean updated = updater.apply(entity);
if (updated) {
dao.save(entity.getTenantId(), entity);
log.debug("[{}][{}] Updated {} images", entity.getTenantId(), entity.getName(), type);
updatedCount++;
}
} catch (Exception e) {
log.error("[{}][{}] Failed to update {} images", entity.getTenantId(), entity.getName(), type, e);
}
if (totalCount % 100 == 0) {
log.info("Processed {} {}s so far", totalCount, type);
}
}
log.info("Updated {} {}s out of {}", updatedCount, type, totalCount);
}
}

4
dao/pom.xml
View File

@ -242,6 +242,10 @@
<groupId>org.apache.xmlgraphics</groupId>
<artifactId>batik-codec</artifactId>
</dependency>
<dependency>
<groupId>com.drewnoakes</groupId>
<artifactId>metadata-extractor</artifactId>
</dependency>
</dependencies>
<build>
<plugins>

25
dao/src/main/java/org/thingsboard/server/dao/resource/BaseImageService.java
View File

@ -19,6 +19,7 @@ import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.common.base.Strings;
import lombok.Data;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
@ -285,8 +286,10 @@ public class BaseImageService extends BaseResourceService implements ImageServic
if (entity.getDescriptor().isObject()) {
ObjectNode descriptor = (ObjectNode) entity.getDescriptor();
JsonNode defaultConfig = JacksonUtil.toJsonNode(descriptor.get("defaultConfig").asText());
updated |= base64ToImageUrlUsingMapping(entity.getTenantId(), WIDGET_TYPE_BASE64_MAPPING, Collections.singletonMap("prefix", prefix), defaultConfig);
descriptor.put("defaultConfig", defaultConfig.toString());
if (defaultConfig != null && defaultConfig.isObject()) {
updated |= base64ToImageUrlUsingMapping(entity.getTenantId(), WIDGET_TYPE_BASE64_MAPPING, Collections.singletonMap("prefix", prefix), defaultConfig);
descriptor.put("defaultConfig", defaultConfig.toString());
}
}
updated |= base64ToImageUrlRecursively(entity.getTenantId(), prefix, entity.getDescriptor());
return updated;
@ -313,6 +316,9 @@ public class BaseImageService extends BaseResourceService implements ImageServic
JsonPathProcessingTask task = tasks.poll();
String token = task.currentToken();
JsonNode node = task.getNode();
if (node == null) {
continue;
}
if (token.equals("*") || token.startsWith("$")) {
String variableName = token.startsWith("$") ? token.substring(1) : null;
if (node.isArray()) {
@ -345,8 +351,8 @@ public class BaseImageService extends BaseResourceService implements ImageServic
}
if (task.isLast()) {
String name = expression;
for (var replacements : task.getVariables().entrySet()) {
name = name.replace("$" + replacements.getKey(), replacements.getValue());
for (var replacement : task.getVariables().entrySet()) {
name = name.replace("$" + replacement.getKey(), Strings.nullToEmpty(replacement.getValue()));
}
if (node.isObject() && value.isTextual()) {
var result = base64ToImageUrl(tenantId, name, value.asText());
@ -362,7 +368,7 @@ public class BaseImageService extends BaseResourceService implements ImageServic
}
}
} else {
if (StringUtils.isNotEmpty(variableName) && StringUtils.isNotEmpty(variableValue)) {
if (StringUtils.isNotEmpty(variableName)) {
tasks.add(task.next(value, variableName, variableValue));
} else {
tasks.add(task.next(value));
@ -404,6 +410,9 @@ public class BaseImageService extends BaseResourceService implements ImageServic
byte[] imageData = Base64.getDecoder().decode(base64Data);
String etag = calculateEtag(imageData);
var imageInfo = findImageByTenantIdAndEtag(tenantId, etag);
if (imageInfo == null && !tenantId.isSysTenantId()) {
imageInfo = findImageByTenantIdAndEtag(TenantId.SYS_TENANT_ID, etag);
}
if (imageInfo == null) {
TbResource image = new TbResource();
image.setTenantId(tenantId);
@ -415,9 +424,9 @@ public class BaseImageService extends BaseResourceService implements ImageServic
String fileName;
if (StringUtils.isBlank(mdResourceKey)) {
fileName = mdResourceName.toLowerCase()
.replace("'", "").replace("\"", "")
.replace(" ", "_").replace("/", "_")
fileName = StringUtils.strip(mdResourceName.toLowerCase()
.replaceAll("['\"]", "")
.replaceAll("[^\\pL\\d]+", "_"), "_") // leaving only letters and numbers
+ "." + extension;
} else {
fileName = mdResourceKey;

2
dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/DashboardRepository.java
View File

@ -43,7 +43,7 @@ public interface DashboardRepository extends JpaRepository<DashboardEntity, UUID
@Query("SELECT d.id FROM DashboardEntity d WHERE d.tenantId = :tenantId")
Page<UUID> findIdsByTenantId(@Param("tenantId") UUID tenantId, Pageable pageable);
@Query("SELECT id FROM DashboardEntity")
@Query("SELECT d.id FROM DashboardEntity d")
Page<UUID> findAllIds(Pageable pageable);
}

2
dao/src/main/java/org/thingsboard/server/dao/sql/resource/TbResourceRepository.java
View File

@ -91,7 +91,7 @@ public interface TbResourceRepository extends JpaRepository<TbResourceEntity, UU
@Query("SELECT externalId FROM TbResourceInfoEntity WHERE id = :id")
UUID getExternalIdByInternal(@Param("id") UUID internalId);
@Query("SELECT id FROM TbResourceInfoEntity WHERE tenantId = :tenantId")
@Query("SELECT r.id FROM TbResourceInfoEntity r WHERE r.tenantId = :tenantId")
Page<UUID> findIdsByTenantId(@Param("tenantId") UUID tenantId, Pageable pageable);
}

8
dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetTypeRepository.java
View File

@ -69,17 +69,17 @@ public interface WidgetTypeRepository extends JpaRepository<WidgetTypeDetailsEnt
@Query(value = "SELECT * FROM widget_type wt " +
"WHERE wt.tenant_id = :tenantId AND cast(wt.descriptor as json) ->> 'resources' LIKE LOWER(CONCAT('%', :resourceId, '%'))",
nativeQuery = true)
nativeQuery = true)
List<WidgetTypeDetailsEntity> findWidgetTypesInfosByTenantIdAndResourceId(@Param("tenantId") UUID tenantId,
@Param("resourceId") UUID resourceId);
@Param("resourceId") UUID resourceId);
@Query("SELECT externalId FROM WidgetTypeDetailsEntity WHERE id = :id")
UUID getExternalIdById(@Param("id") UUID id);
@Query("SELECT id FROM WidgetTypeDetailsEntity")
@Query("SELECT w.id FROM WidgetTypeDetailsEntity w")
Page<UUID> findAllIds(Pageable pageable);
@Query("SELECT id FROM WidgetTypeDetailsEntity WHERE tenantId = :tenantId")
@Query("SELECT w.id FROM WidgetTypeDetailsEntity w WHERE w.tenantId = :tenantId")
Page<UUID> findIdsByTenantId(@Param("tenantId") UUID tenantId, Pageable pageable);
}

70
dao/src/main/java/org/thingsboard/server/dao/util/ImageUtils.java
View File

@ -15,9 +15,14 @@
*/
package org.thingsboard.server.dao.util;
import com.drew.imaging.ImageMetadataReader;
import com.drew.metadata.Directory;
import com.drew.metadata.Metadata;
import com.drew.metadata.Tag;
import lombok.AccessLevel;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.batik.anim.dom.SAXSVGDocumentFactory;
import org.apache.batik.bridge.BridgeContext;
import org.apache.batik.bridge.DocumentLoader;
@ -41,11 +46,13 @@ import java.io.ByteArrayOutputStream;
import java.util.Map;
@NoArgsConstructor(access = AccessLevel.PRIVATE)
@Slf4j
public class ImageUtils {
private static final Map<String, String> mediaTypeMappings = Map.of(
"jpeg", "jpg",
"svg+xml", "svg"
"svg+xml", "svg",
"x-icon", "ico"
);
public static String mediaTypeToFileExtension(String mimeType) {
@ -64,15 +71,49 @@ public class ImageUtils {
if (mediaTypeToFileExtension(mediaType).equals("svg")) {
return processSvgImage(data, mediaType, thumbnailMaxDimension);
}
BufferedImage bufferedImage = ImageIO.read(new ByteArrayInputStream(data));
ProcessedImage image = new ProcessedImage();
image.setMediaType(mediaType);
image.setWidth(bufferedImage.getWidth());
image.setHeight(bufferedImage.getHeight());
image.setData(data);
image.setSize(data.length);
BufferedImage bufferedImage = null;
try {
bufferedImage = ImageIO.read(new ByteArrayInputStream(data));
} catch (Exception ignored) {
}
if (bufferedImage == null) { // means that media type is not supported by ImageIO; extracting width and height from metadata and leaving preview as original image
Metadata metadata = ImageMetadataReader.readMetadata(new ByteArrayInputStream(data));
for (Directory dir : metadata.getDirectories()) {
Tag widthTag = dir.getTags().stream()
.filter(tag -> tag.getTagName().toLowerCase().contains("width"))
.findFirst().orElse(null);
Tag heightTag = dir.getTags().stream()
.filter(tag -> tag.getTagName().toLowerCase().contains("height"))
.findFirst().orElse(null);
if (widthTag == null || heightTag == null) {
continue;
}
int width = Integer.parseInt(dir.getObject(widthTag.getTagType()).toString());
int height = Integer.parseInt(dir.getObject(widthTag.getTagType()).toString());
image.setWidth(width);
image.setHeight(height);
ProcessedImage preview = new ProcessedImage();
preview.setWidth(image.getWidth());
preview.setHeight(image.getHeight());
preview.setMediaType(mediaType);
preview.setData(null);
preview.setSize(data.length);
image.setPreview(preview);
log.debug("Couldn't parse {} ({}) with ImageIO, got width {} and height {} from metadata", mediaType, dir.getName(), width, height);
return image;
}
throw new IllegalArgumentException("Media type " + mediaType + " not supported");
}
image.setWidth(bufferedImage.getWidth());
image.setHeight(bufferedImage.getHeight());
ProcessedImage preview = new ProcessedImage();
int[] thumbnailDimensions = getThumbnailDimensions(image.getWidth(), image.getHeight(), thumbnailMaxDimension);
preview.setWidth(thumbnailDimensions[0]);
@ -101,18 +142,19 @@ public class ImageUtils {
}
public static ProcessedImage processSvgImage(byte[] data, String mediaType, int thumbnailMaxDimension) throws Exception {
SAXSVGDocumentFactory factory = new SAXSVGDocumentFactory(
XMLResourceDescriptor.getXMLParserClassName());
Document document = factory.createDocument(
null, new ByteArrayInputStream(data));
SAXSVGDocumentFactory factory = new SAXSVGDocumentFactory(XMLResourceDescriptor.getXMLParserClassName());
Document document = factory.createDocument(null, new ByteArrayInputStream(data));
Integer width = null;
Integer height = null;
String strWidth = document.getDocumentElement().getAttribute("width");
String strHeight = document.getDocumentElement().getAttribute("height");
if (StringUtils.isNotEmpty(strWidth) && StringUtils.isNotEmpty(strHeight)) {
width = (int) Double.parseDouble(strWidth);
height = (int) Double.parseDouble(strHeight);;
} else {
try {
width = (int) Double.parseDouble(strWidth);
height = (int) Double.parseDouble(strHeight);
} catch (NumberFormatException ignored) {} // in case width and height are in %, mm, etc.
}
if (width == null || height == null) {
String viewBox = document.getDocumentElement().getAttribute("viewBox");
if (StringUtils.isNotEmpty(viewBox)) {
String[] viewBoxValues = viewBox.split(" ");
@ -124,10 +166,10 @@ public class ImageUtils {
}
if (width == null) {
UserAgent agent = new UserAgentAdapter();
DocumentLoader loader= new DocumentLoader(agent);
DocumentLoader loader = new DocumentLoader(agent);
BridgeContext context = new BridgeContext(agent, loader);
context.setDynamic(true);
GVTBuilder builder= new GVTBuilder();
GVTBuilder builder = new GVTBuilder();
GraphicsNode root = builder.build(context, document);
var bounds = root.getPrimitiveBounds();
if (bounds != null) {

6
pom.xml
View File

@ -154,6 +154,7 @@
<oshi.version>6.4.2</oshi.version>
<google-oauth-client.version>1.34.1</google-oauth-client.version>
<apache-xmlgraphics.version>1.17</apache-xmlgraphics.version>
<drewnoakes-metadata-extractor.version>2.19.0</drewnoakes-metadata-extractor.version>
</properties>
<modules>
@ -2040,6 +2041,11 @@
<artifactId>batik-codec</artifactId>
<version>${apache-xmlgraphics.version}</version>
</dependency>
<dependency>
<groupId>com.drewnoakes</groupId>
<artifactId>metadata-extractor</artifactId>
<version>${drewnoakes-metadata-extractor.version}</version>
</dependency>
</dependencies>
</dependencyManagement>