dendi-wang/thingsboard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #13786 from dashevchenko/jsonAttrType

Browse Source 
Added JSON ts type support for bulk import
This commit is contained in:
Viacheslav Klimov 2025-10-14 16:55:23 +03:00 committed by GitHub
commit e39944ee9d
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 96 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/AbstractBulkImportService.java
View File

@ -17,6 +17,7 @@ package org.thingsboard.server.service.sync.ie.importing.csv;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.common.util.concurrent.FutureCallback;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.JsonPrimitive;
import jakarta.annotation.Nullable;
@ -183,7 +184,13 @@ public abstract class AbstractBulkImportService<E extends HasId<? extends Entity
data.entrySet().stream()
.filter(dataEntry -> dataEntry.getKey().getType() == kvType &&
StringUtils.isNotEmpty(dataEntry.getKey().getKey()))
.forEach(dataEntry -> kvs.add(dataEntry.getKey().getKey(), dataEntry.getValue().toJsonPrimitive()));
.forEach(dataEntry -> {
ParsedValue value = dataEntry.getValue();
JsonElement kvValue = (value.getDataType() == DataType.JSON)
? (JsonElement) value.getValue()
: value.toJsonPrimitive();
kvs.add(dataEntry.getKey().getKey(), kvValue);
});
return Map.entry(kvType, kvs);
})
.filter(kvsEntry -> kvsEntry.getValue().entrySet().size() > 0)

19
application/src/main/java/org/thingsboard/server/utils/CsvUtils.java
View File

@ -17,10 +17,15 @@ package org.thingsboard.server.utils;
import lombok.AccessLevel;
import lombok.NoArgsConstructor;
import lombok.SneakyThrows;
import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVPrinter;
import org.apache.commons.csv.CSVRecord;
import org.apache.commons.io.input.CharSequenceReader;
import java.io.ByteArrayOutputStream;
import java.io.OutputStreamWriter;
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@ -43,4 +48,18 @@ public class CsvUtils {
.collect(Collectors.toList());
}
@SneakyThrows
public static byte[] generateCsv(List<List<String>> rows) {
ByteArrayOutputStream out = new ByteArrayOutputStream();
try (OutputStreamWriter writer = new OutputStreamWriter(out, StandardCharsets.UTF_8);
CSVPrinter csvPrinter = new CSVPrinter(writer, CSVFormat.DEFAULT)) {
for (List<String> row : rows) {
csvPrinter.printRecord(row);
}
csvPrinter.flush();
}
return out.toByteArray();
}
}

57
application/src/test/java/org/thingsboard/server/controller/DeviceControllerTest.java
View File

@ -38,6 +38,7 @@ import org.springframework.test.context.ContextConfiguration;
import org.testcontainers.shaded.org.awaitility.Awaitility;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.common.util.ThingsBoardExecutors;
import org.thingsboard.server.common.data.AttributeScope;
import org.thingsboard.server.common.data.Customer;
import org.thingsboard.server.common.data.Device;
import org.thingsboard.server.common.data.DeviceInfo;
@ -59,6 +60,7 @@ import org.thingsboard.server.common.data.id.CustomerId;
import org.thingsboard.server.common.data.id.DeviceCredentialsId;
import org.thingsboard.server.common.data.id.DeviceId;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.kv.AttributeKvEntry;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.relation.EntityRelation;
@ -77,10 +79,15 @@ import org.thingsboard.server.dao.service.DaoSqlTest;
import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.service.gateway_device.GatewayNotificationsService;
import org.thingsboard.server.service.state.DeviceStateService;
import org.thingsboard.server.utils.CsvUtils;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.TimeUnit;
import static org.assertj.core.api.Assertions.assertThat;
@ -1586,6 +1593,56 @@ public class DeviceControllerTest extends AbstractControllerTest {
Assert.assertEquals(newAttributeValue, actualAttribute.get("value"));
}
@Test
public void testBulkImportDeviceWithJsonAttr() throws Exception {
String deviceName = "some_device";
String deviceType = "some_type";
String deviceAttr = "{\"threshold\":45}";
List<List<String>> content = new LinkedList<>();
content.add(Arrays.asList("NAME", "TYPE", "ATTR"));
content.add(Arrays.asList(deviceName, deviceType, deviceAttr));
byte[] bytes = CsvUtils.generateCsv(content);
BulkImportRequest request = new BulkImportRequest();
request.setFile(new String(bytes, StandardCharsets.UTF_8));
BulkImportRequest.Mapping mapping = new BulkImportRequest.Mapping();
BulkImportRequest.ColumnMapping name = new BulkImportRequest.ColumnMapping();
name.setType(BulkImportColumnType.NAME);
BulkImportRequest.ColumnMapping type = new BulkImportRequest.ColumnMapping();
type.setType(BulkImportColumnType.TYPE);
BulkImportRequest.ColumnMapping attr = new BulkImportRequest.ColumnMapping();
attr.setType(BulkImportColumnType.SERVER_ATTRIBUTE);
attr.setKey("attr");
List<BulkImportRequest.ColumnMapping> columns = new ArrayList<>();
columns.add(name);
columns.add(type);
columns.add(attr);
mapping.setColumns(columns);
mapping.setDelimiter(',');
mapping.setUpdate(true);
mapping.setHeader(true);
request.setMapping(mapping);
BulkImportResult<Device> deviceBulkImportResult = doPostWithTypedResponse("/api/device/bulk_import", request, new TypeReference<>() {});
Assert.assertEquals(1, deviceBulkImportResult.getCreated().get());
Assert.assertEquals(0, deviceBulkImportResult.getErrors().get());
Assert.assertEquals(0, deviceBulkImportResult.getUpdated().get());
Assert.assertTrue(deviceBulkImportResult.getErrorsList().isEmpty());
Device savedDevice = doGet("/api/tenant/devices?deviceName=" + deviceName, Device.class);
Assert.assertNotNull(savedDevice);
Assert.assertEquals(deviceName, savedDevice.getName());
Assert.assertEquals(deviceType, savedDevice.getType());
Optional<AttributeKvEntry> retrieved = attributesService.find(tenantId, savedDevice.getId(), AttributeScope.SERVER_SCOPE, "attr").get();
assertThat(retrieved.get().getJsonValue().get()).isEqualTo(deviceAttr);
assertThat(retrieved.get().getStrValue()).isNotPresent();
}
@Test
public void testSaveDeviceWithOutdatedVersion() throws Exception {
Device device = createDevice("Device v1.0");

12
common/data/src/main/java/org/thingsboard/server/common/data/util/TypeCastUtil.java
View File

@ -15,6 +15,7 @@
*/
package org.thingsboard.server.common.data.util;
import com.google.gson.JsonParser;
import org.apache.commons.lang3.math.NumberUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.thingsboard.server.common.data.kv.DataType;
@ -40,6 +41,11 @@ public class TypeCastUtil {
} catch (RuntimeException ignored) {}
} else if (value.equalsIgnoreCase("true") || value.equalsIgnoreCase("false")) {
return Pair.of(DataType.BOOLEAN, Boolean.parseBoolean(value));
} else if (looksLikeJson(value)) {
try {
return Pair.of(DataType.JSON, JsonParser.parseString(value));
} catch (Exception ignored) {
}
}
return Pair.of(DataType.STRING, value);
}
@ -70,4 +76,10 @@ public class TypeCastUtil {
return valueAsString.contains(".") && !valueAsString.contains("E") && !valueAsString.contains("e");
}
private static boolean looksLikeJson(String value) {
String trimmed = value.trim();
return (trimmed.startsWith("{") && trimmed.endsWith("}")) ||
(trimmed.startsWith("[") && trimmed.endsWith("]"));
}
}