Merge pull request #13786 from dashevchenko/jsonAttrType

Added JSON ts type support for bulk import

application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/AbstractBulkImportService.java

@@ -17,6 +17,7 @@ package org.thingsboard.server.service.sync.ie.importing.csv;
 
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.google.common.util.concurrent.FutureCallback;
+import com.google.gson.JsonElement;
 import com.google.gson.JsonObject;
 import com.google.gson.JsonPrimitive;
 import jakarta.annotation.Nullable;
@@ -183,7 +184,13 @@ public abstract class AbstractBulkImportService<E extends HasId<? extends Entity
                     data.entrySet().stream()
                             .filter(dataEntry -> dataEntry.getKey().getType() == kvType &&
                                     StringUtils.isNotEmpty(dataEntry.getKey().getKey()))
-                            .forEach(dataEntry -> kvs.add(dataEntry.getKey().getKey(), dataEntry.getValue().toJsonPrimitive()));
+                            .forEach(dataEntry -> {
+                                ParsedValue value = dataEntry.getValue();
+                                JsonElement kvValue = (value.getDataType() == DataType.JSON)
+                                        ? (JsonElement) value.getValue()
+                                        : value.toJsonPrimitive();
+                                kvs.add(dataEntry.getKey().getKey(), kvValue);
+                            });
                     return Map.entry(kvType, kvs);
                 })
                 .filter(kvsEntry -> kvsEntry.getValue().entrySet().size() > 0)

application/src/main/java/org/thingsboard/server/utils/CsvUtils.java

@@ -17,10 +17,15 @@ package org.thingsboard.server.utils;
 
 import lombok.AccessLevel;
 import lombok.NoArgsConstructor;
+import lombok.SneakyThrows;
 import org.apache.commons.csv.CSVFormat;
+import org.apache.commons.csv.CSVPrinter;
 import org.apache.commons.csv.CSVRecord;
 import org.apache.commons.io.input.CharSequenceReader;
 
+import java.io.ByteArrayOutputStream;
+import java.io.OutputStreamWriter;
+import java.nio.charset.StandardCharsets;
 import java.util.List;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
@@ -43,4 +48,18 @@ public class CsvUtils {
                 .collect(Collectors.toList());
     }
 
+    @SneakyThrows
+    public static byte[] generateCsv(List<List<String>> rows) {
+        ByteArrayOutputStream out = new ByteArrayOutputStream();
+        try (OutputStreamWriter writer = new OutputStreamWriter(out, StandardCharsets.UTF_8);
+             CSVPrinter csvPrinter = new CSVPrinter(writer, CSVFormat.DEFAULT)) {
+
+            for (List<String> row : rows) {
+                csvPrinter.printRecord(row);
+            }
+            csvPrinter.flush();
+        }
+        return out.toByteArray();
+    }
+
 }

application/src/test/java/org/thingsboard/server/controller/DeviceControllerTest.java

@@ -38,6 +38,7 @@ import org.springframework.test.context.ContextConfiguration;
 import org.testcontainers.shaded.org.awaitility.Awaitility;
 import org.thingsboard.common.util.JacksonUtil;
 import org.thingsboard.common.util.ThingsBoardExecutors;
+import org.thingsboard.server.common.data.AttributeScope;
 import org.thingsboard.server.common.data.Customer;
 import org.thingsboard.server.common.data.Device;
 import org.thingsboard.server.common.data.DeviceInfo;
@@ -59,6 +60,7 @@ import org.thingsboard.server.common.data.id.CustomerId;
 import org.thingsboard.server.common.data.id.DeviceCredentialsId;
 import org.thingsboard.server.common.data.id.DeviceId;
 import org.thingsboard.server.common.data.id.EntityId;
+import org.thingsboard.server.common.data.kv.AttributeKvEntry;
 import org.thingsboard.server.common.data.page.PageData;
 import org.thingsboard.server.common.data.page.PageLink;
 import org.thingsboard.server.common.data.relation.EntityRelation;
@@ -77,10 +79,15 @@ import org.thingsboard.server.dao.service.DaoSqlTest;
 import org.thingsboard.server.gen.transport.TransportProtos;
 import org.thingsboard.server.service.gateway_device.GatewayNotificationsService;
 import org.thingsboard.server.service.state.DeviceStateService;
+import org.thingsboard.server.utils.CsvUtils;
 
+import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.concurrent.TimeUnit;
 
 import static org.assertj.core.api.Assertions.assertThat;
@@ -1586,6 +1593,56 @@ public class DeviceControllerTest extends AbstractControllerTest {
         Assert.assertEquals(newAttributeValue, actualAttribute.get("value"));
     }
 
+    @Test
+    public void testBulkImportDeviceWithJsonAttr() throws Exception {
+        String deviceName = "some_device";
+        String deviceType = "some_type";
+        String deviceAttr = "{\"threshold\":45}";
+
+        List<List<String>> content = new LinkedList<>();
+        content.add(Arrays.asList("NAME", "TYPE", "ATTR"));
+        content.add(Arrays.asList(deviceName, deviceType, deviceAttr));
+
+        byte[] bytes = CsvUtils.generateCsv(content);
+        BulkImportRequest request = new BulkImportRequest();
+        request.setFile(new String(bytes, StandardCharsets.UTF_8));
+        BulkImportRequest.Mapping mapping = new BulkImportRequest.Mapping();
+        BulkImportRequest.ColumnMapping name = new BulkImportRequest.ColumnMapping();
+        name.setType(BulkImportColumnType.NAME);
+        BulkImportRequest.ColumnMapping type = new BulkImportRequest.ColumnMapping();
+        type.setType(BulkImportColumnType.TYPE);
+        BulkImportRequest.ColumnMapping attr = new BulkImportRequest.ColumnMapping();
+        attr.setType(BulkImportColumnType.SERVER_ATTRIBUTE);
+        attr.setKey("attr");
+        List<BulkImportRequest.ColumnMapping> columns = new ArrayList<>();
+        columns.add(name);
+        columns.add(type);
+        columns.add(attr);
+
+        mapping.setColumns(columns);
+        mapping.setDelimiter(',');
+        mapping.setUpdate(true);
+        mapping.setHeader(true);
+        request.setMapping(mapping);
+
+        BulkImportResult<Device> deviceBulkImportResult = doPostWithTypedResponse("/api/device/bulk_import", request, new TypeReference<>() {});
+
+        Assert.assertEquals(1, deviceBulkImportResult.getCreated().get());
+        Assert.assertEquals(0, deviceBulkImportResult.getErrors().get());
+        Assert.assertEquals(0, deviceBulkImportResult.getUpdated().get());
+        Assert.assertTrue(deviceBulkImportResult.getErrorsList().isEmpty());
+
+        Device savedDevice = doGet("/api/tenant/devices?deviceName=" + deviceName, Device.class);
+
+        Assert.assertNotNull(savedDevice);
+        Assert.assertEquals(deviceName, savedDevice.getName());
+        Assert.assertEquals(deviceType, savedDevice.getType());
+
+        Optional<AttributeKvEntry> retrieved = attributesService.find(tenantId, savedDevice.getId(), AttributeScope.SERVER_SCOPE, "attr").get();
+        assertThat(retrieved.get().getJsonValue().get()).isEqualTo(deviceAttr);
+        assertThat(retrieved.get().getStrValue()).isNotPresent();
+    }
+
     @Test
     public void testSaveDeviceWithOutdatedVersion() throws Exception {
         Device device = createDevice("Device v1.0");

common/data/src/main/java/org/thingsboard/server/common/data/util/TypeCastUtil.java

@@ -15,6 +15,7 @@
  */
 package org.thingsboard.server.common.data.util;
 
+import com.google.gson.JsonParser;
 import org.apache.commons.lang3.math.NumberUtils;
 import org.apache.commons.lang3.tuple.Pair;
 import org.thingsboard.server.common.data.kv.DataType;
@@ -40,6 +41,11 @@ public class TypeCastUtil {
             } catch (RuntimeException ignored) {}
         } else if (value.equalsIgnoreCase("true") || value.equalsIgnoreCase("false")) {
             return Pair.of(DataType.BOOLEAN, Boolean.parseBoolean(value));
+        } else if (looksLikeJson(value)) {
+            try {
+                return Pair.of(DataType.JSON, JsonParser.parseString(value));
+            } catch (Exception ignored) {
+            }
         }
         return Pair.of(DataType.STRING, value);
     }
@@ -70,4 +76,10 @@ public class TypeCastUtil {
         return valueAsString.contains(".") && !valueAsString.contains("E") && !valueAsString.contains("e");
     }
 
+    private static boolean looksLikeJson(String value) {
+        String trimmed = value.trim();
+        return (trimmed.startsWith("{") && trimmed.endsWith("}")) ||
+                (trimmed.startsWith("[") && trimmed.endsWith("]"));
+    }
+
 }