From 7be97f379c69a73ce30d485c6736a2ac62e5e021 Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Tue, 29 Jul 2025 14:30:59 +0300 Subject: [PATCH] added json timeseries type for bulk import --- .../csv/AbstractBulkImportService.java | 9 ++- .../thingsboard/server/utils/CsvUtils.java | 19 ++++++ .../controller/DeviceControllerTest.java | 60 +++++++++++++++++++ .../server/common/data/util/TypeCastUtil.java | 11 ++++ 4 files changed, 98 insertions(+), 1 deletion(-) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/AbstractBulkImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/AbstractBulkImportService.java index 9850e2d1a1..768adf98ff 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/AbstractBulkImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/AbstractBulkImportService.java @@ -17,6 +17,7 @@ package org.thingsboard.server.service.sync.ie.importing.csv; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.util.concurrent.FutureCallback; +import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.gson.JsonPrimitive; import jakarta.annotation.Nullable; @@ -183,7 +184,13 @@ public abstract class AbstractBulkImportService dataEntry.getKey().getType() == kvType && StringUtils.isNotEmpty(dataEntry.getKey().getKey())) - .forEach(dataEntry -> kvs.add(dataEntry.getKey().getKey(), dataEntry.getValue().toJsonPrimitive())); + .forEach(dataEntry -> { + ParsedValue value = dataEntry.getValue(); + JsonElement kvValue = (value.getDataType() == DataType.JSON) + ? (JsonElement) value.getValue() + : value.toJsonPrimitive(); + kvs.add(dataEntry.getKey().getKey(), kvValue); + }); return Map.entry(kvType, kvs); }) .filter(kvsEntry -> kvsEntry.getValue().entrySet().size() > 0) diff --git a/application/src/main/java/org/thingsboard/server/utils/CsvUtils.java b/application/src/main/java/org/thingsboard/server/utils/CsvUtils.java index 3f75a4918f..0fde72a3b0 100644 --- a/application/src/main/java/org/thingsboard/server/utils/CsvUtils.java +++ b/application/src/main/java/org/thingsboard/server/utils/CsvUtils.java @@ -17,10 +17,15 @@ package org.thingsboard.server.utils; import lombok.AccessLevel; import lombok.NoArgsConstructor; +import lombok.SneakyThrows; import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVPrinter; import org.apache.commons.csv.CSVRecord; import org.apache.commons.io.input.CharSequenceReader; +import java.io.ByteArrayOutputStream; +import java.io.OutputStreamWriter; +import java.nio.charset.StandardCharsets; import java.util.List; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -43,4 +48,18 @@ public class CsvUtils { .collect(Collectors.toList()); } + @SneakyThrows + public static byte[] generateCsv(List> rows) { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + try (OutputStreamWriter writer = new OutputStreamWriter(out, StandardCharsets.UTF_8); + CSVPrinter csvPrinter = new CSVPrinter(writer, CSVFormat.DEFAULT)) { + + for (List row : rows) { + csvPrinter.printRecord(row); + } + csvPrinter.flush(); + } + return out.toByteArray(); + } + } diff --git a/application/src/test/java/org/thingsboard/server/controller/DeviceControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/DeviceControllerTest.java index 36cede9e84..6b31825640 100644 --- a/application/src/test/java/org/thingsboard/server/controller/DeviceControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/DeviceControllerTest.java @@ -77,8 +77,13 @@ import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.service.gateway_device.GatewayNotificationsService; import org.thingsboard.server.service.state.DeviceStateService; +import org.thingsboard.server.utils.CsvUtils; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; @@ -1586,6 +1591,61 @@ public class DeviceControllerTest extends AbstractControllerTest { Assert.assertEquals(newAttributeValue, actualAttribute.get("value")); } + @Test + public void testBulkImportDeviceWithJsonAttr() throws Exception { + String deviceName = "some_device"; + String deviceType = "some_type"; + JsonNode deviceAttr = JacksonUtil.toJsonNode("{\"threshold\": 45}"); + + List> content = new LinkedList<>(); + content.add(Arrays.asList("NAME", "TYPE", "ATTR")); + content.add(Arrays.asList(deviceName, deviceType, deviceAttr.toString())); + + byte[] bytes = CsvUtils.generateCsv(content); + BulkImportRequest request = new BulkImportRequest(); + request.setFile(new String(bytes, StandardCharsets.UTF_8)); + BulkImportRequest.Mapping mapping = new BulkImportRequest.Mapping(); + BulkImportRequest.ColumnMapping name = new BulkImportRequest.ColumnMapping(); + name.setType(BulkImportColumnType.NAME); + BulkImportRequest.ColumnMapping type = new BulkImportRequest.ColumnMapping(); + type.setType(BulkImportColumnType.TYPE); + BulkImportRequest.ColumnMapping attr = new BulkImportRequest.ColumnMapping(); + attr.setType(BulkImportColumnType.SERVER_ATTRIBUTE); + attr.setKey("attr"); + List columns = new ArrayList<>(); + columns.add(name); + columns.add(type); + columns.add(attr); + + mapping.setColumns(columns); + mapping.setDelimiter(','); + mapping.setUpdate(true); + mapping.setHeader(true); + request.setMapping(mapping); + + BulkImportResult deviceBulkImportResult = doPostWithTypedResponse("/api/device/bulk_import", request, new TypeReference<>() {}); + + Assert.assertEquals(1, deviceBulkImportResult.getCreated().get()); + Assert.assertEquals(0, deviceBulkImportResult.getErrors().get()); + Assert.assertEquals(0, deviceBulkImportResult.getUpdated().get()); + Assert.assertTrue(deviceBulkImportResult.getErrorsList().isEmpty()); + + Device savedDevice = doGet("/api/tenant/devices?deviceName=" + deviceName, Device.class); + + Assert.assertNotNull(savedDevice); + Assert.assertEquals(deviceName, savedDevice.getName()); + Assert.assertEquals(deviceType, savedDevice.getType()); + + //check server attribute value + await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> { + Map actualAttribute = doGetAsyncTyped("/api/plugins/telemetry/DEVICE/" + savedDevice.getId() + + "/values/attributes/SERVER_SCOPE", new TypeReference>>() {}).stream() + .filter(att -> att.get("key").equals("attr")).findFirst().get(); + LinkedHashMap expected = JacksonUtil.convertValue(deviceAttr, new TypeReference<>() {}); + Assert.assertEquals(expected, actualAttribute.get("value")); + }); + } + @Test public void testSaveDeviceWithOutdatedVersion() throws Exception { Device device = createDevice("Device v1.0"); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/util/TypeCastUtil.java b/common/data/src/main/java/org/thingsboard/server/common/data/util/TypeCastUtil.java index 85071b3cc9..94a1ee0ef4 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/util/TypeCastUtil.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/util/TypeCastUtil.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.common.data.util; +import com.google.gson.JsonParser; import org.apache.commons.lang3.math.NumberUtils; import org.apache.commons.lang3.tuple.Pair; import org.thingsboard.server.common.data.kv.DataType; @@ -40,6 +41,11 @@ public class TypeCastUtil { } catch (RuntimeException ignored) {} } else if (value.equalsIgnoreCase("true") || value.equalsIgnoreCase("false")) { return Pair.of(DataType.BOOLEAN, Boolean.parseBoolean(value)); + } else if (looksLikeJson(value)) { + try { + return Pair.of(DataType.JSON, JsonParser.parseString(value)); + } catch (Exception ignored) { + } } return Pair.of(DataType.STRING, value); } @@ -70,4 +76,9 @@ public class TypeCastUtil { return valueAsString.contains(".") && !valueAsString.contains("E") && !valueAsString.contains("e"); } + private static boolean looksLikeJson(String value) { + return (value.startsWith("{") && value.endsWith("}")) || + (value.startsWith("[") && value.endsWith("]")); + } + }