diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/AbstractBulkImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/AbstractBulkImportService.java index 9850e2d1a1..768adf98ff 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/AbstractBulkImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/AbstractBulkImportService.java @@ -17,6 +17,7 @@ package org.thingsboard.server.service.sync.ie.importing.csv; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.util.concurrent.FutureCallback; +import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.gson.JsonPrimitive; import jakarta.annotation.Nullable; @@ -183,7 +184,13 @@ public abstract class AbstractBulkImportService dataEntry.getKey().getType() == kvType && StringUtils.isNotEmpty(dataEntry.getKey().getKey())) - .forEach(dataEntry -> kvs.add(dataEntry.getKey().getKey(), dataEntry.getValue().toJsonPrimitive())); + .forEach(dataEntry -> { + ParsedValue value = dataEntry.getValue(); + JsonElement kvValue = (value.getDataType() == DataType.JSON) + ? (JsonElement) value.getValue() + : value.toJsonPrimitive(); + kvs.add(dataEntry.getKey().getKey(), kvValue); + }); return Map.entry(kvType, kvs); }) .filter(kvsEntry -> kvsEntry.getValue().entrySet().size() > 0) diff --git a/application/src/main/java/org/thingsboard/server/utils/CsvUtils.java b/application/src/main/java/org/thingsboard/server/utils/CsvUtils.java index 3f75a4918f..0fde72a3b0 100644 --- a/application/src/main/java/org/thingsboard/server/utils/CsvUtils.java +++ b/application/src/main/java/org/thingsboard/server/utils/CsvUtils.java @@ -17,10 +17,15 @@ package org.thingsboard.server.utils; import lombok.AccessLevel; import lombok.NoArgsConstructor; +import lombok.SneakyThrows; import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVPrinter; import org.apache.commons.csv.CSVRecord; import org.apache.commons.io.input.CharSequenceReader; +import java.io.ByteArrayOutputStream; +import java.io.OutputStreamWriter; +import java.nio.charset.StandardCharsets; import java.util.List; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -43,4 +48,18 @@ public class CsvUtils { .collect(Collectors.toList()); } + @SneakyThrows + public static byte[] generateCsv(List> rows) { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + try (OutputStreamWriter writer = new OutputStreamWriter(out, StandardCharsets.UTF_8); + CSVPrinter csvPrinter = new CSVPrinter(writer, CSVFormat.DEFAULT)) { + + for (List row : rows) { + csvPrinter.printRecord(row); + } + csvPrinter.flush(); + } + return out.toByteArray(); + } + } diff --git a/application/src/test/java/org/thingsboard/server/controller/DeviceControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/DeviceControllerTest.java index ca5f146a88..2a2f7bc888 100644 --- a/application/src/test/java/org/thingsboard/server/controller/DeviceControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/DeviceControllerTest.java @@ -38,6 +38,7 @@ import org.springframework.test.context.ContextConfiguration; import org.testcontainers.shaded.org.awaitility.Awaitility; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardExecutors; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceInfo; @@ -59,6 +60,7 @@ import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceCredentialsId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.relation.EntityRelation; @@ -77,10 +79,15 @@ import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.service.gateway_device.GatewayNotificationsService; import org.thingsboard.server.service.state.DeviceStateService; +import org.thingsboard.server.utils.CsvUtils; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.concurrent.TimeUnit; import static org.assertj.core.api.Assertions.assertThat; @@ -1586,6 +1593,56 @@ public class DeviceControllerTest extends AbstractControllerTest { Assert.assertEquals(newAttributeValue, actualAttribute.get("value")); } + @Test + public void testBulkImportDeviceWithJsonAttr() throws Exception { + String deviceName = "some_device"; + String deviceType = "some_type"; + String deviceAttr = "{\"threshold\":45}"; + + List> content = new LinkedList<>(); + content.add(Arrays.asList("NAME", "TYPE", "ATTR")); + content.add(Arrays.asList(deviceName, deviceType, deviceAttr)); + + byte[] bytes = CsvUtils.generateCsv(content); + BulkImportRequest request = new BulkImportRequest(); + request.setFile(new String(bytes, StandardCharsets.UTF_8)); + BulkImportRequest.Mapping mapping = new BulkImportRequest.Mapping(); + BulkImportRequest.ColumnMapping name = new BulkImportRequest.ColumnMapping(); + name.setType(BulkImportColumnType.NAME); + BulkImportRequest.ColumnMapping type = new BulkImportRequest.ColumnMapping(); + type.setType(BulkImportColumnType.TYPE); + BulkImportRequest.ColumnMapping attr = new BulkImportRequest.ColumnMapping(); + attr.setType(BulkImportColumnType.SERVER_ATTRIBUTE); + attr.setKey("attr"); + List columns = new ArrayList<>(); + columns.add(name); + columns.add(type); + columns.add(attr); + + mapping.setColumns(columns); + mapping.setDelimiter(','); + mapping.setUpdate(true); + mapping.setHeader(true); + request.setMapping(mapping); + + BulkImportResult deviceBulkImportResult = doPostWithTypedResponse("/api/device/bulk_import", request, new TypeReference<>() {}); + + Assert.assertEquals(1, deviceBulkImportResult.getCreated().get()); + Assert.assertEquals(0, deviceBulkImportResult.getErrors().get()); + Assert.assertEquals(0, deviceBulkImportResult.getUpdated().get()); + Assert.assertTrue(deviceBulkImportResult.getErrorsList().isEmpty()); + + Device savedDevice = doGet("/api/tenant/devices?deviceName=" + deviceName, Device.class); + + Assert.assertNotNull(savedDevice); + Assert.assertEquals(deviceName, savedDevice.getName()); + Assert.assertEquals(deviceType, savedDevice.getType()); + + Optional retrieved = attributesService.find(tenantId, savedDevice.getId(), AttributeScope.SERVER_SCOPE, "attr").get(); + assertThat(retrieved.get().getJsonValue().get()).isEqualTo(deviceAttr); + assertThat(retrieved.get().getStrValue()).isNotPresent(); + } + @Test public void testSaveDeviceWithOutdatedVersion() throws Exception { Device device = createDevice("Device v1.0"); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/util/TypeCastUtil.java b/common/data/src/main/java/org/thingsboard/server/common/data/util/TypeCastUtil.java index 85071b3cc9..82de579d3d 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/util/TypeCastUtil.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/util/TypeCastUtil.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.common.data.util; +import com.google.gson.JsonParser; import org.apache.commons.lang3.math.NumberUtils; import org.apache.commons.lang3.tuple.Pair; import org.thingsboard.server.common.data.kv.DataType; @@ -40,6 +41,11 @@ public class TypeCastUtil { } catch (RuntimeException ignored) {} } else if (value.equalsIgnoreCase("true") || value.equalsIgnoreCase("false")) { return Pair.of(DataType.BOOLEAN, Boolean.parseBoolean(value)); + } else if (looksLikeJson(value)) { + try { + return Pair.of(DataType.JSON, JsonParser.parseString(value)); + } catch (Exception ignored) { + } } return Pair.of(DataType.STRING, value); } @@ -70,4 +76,10 @@ public class TypeCastUtil { return valueAsString.contains(".") && !valueAsString.contains("E") && !valueAsString.contains("e"); } + private static boolean looksLikeJson(String value) { + String trimmed = value.trim(); + return (trimmed.startsWith("{") && trimmed.endsWith("}")) || + (trimmed.startsWith("[") && trimmed.endsWith("]")); + } + }