JsonConverter improvements (correct value conversion)

common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/secure/TbLwM2MDtlsCertificateVerifier.java

@@ -41,6 +41,7 @@ import org.thingsboard.server.common.transport.TransportServiceCallback;
 import org.thingsboard.server.common.transport.auth.ValidateDeviceCredentialsResponse;
 import org.thingsboard.server.common.transport.util.SslUtil;
 import org.thingsboard.server.gen.transport.TransportProtos;
+import org.thingsboard.server.queue.util.TbLwM2mTransportComponent;
 import org.thingsboard.server.transport.lwm2m.config.LwM2MTransportServerConfig;
 import org.thingsboard.server.transport.lwm2m.secure.credentials.LwM2MCredentials;
 import org.thingsboard.server.common.data.device.credentials.lwm2m.X509ClientCredentials;
@@ -61,6 +62,7 @@ import java.util.concurrent.TimeUnit;
 
 @Slf4j
 @Component
+@TbLwM2mTransportComponent
 @RequiredArgsConstructor
 public class TbLwM2MDtlsCertificateVerifier implements NewAdvancedCertificateVerifier {
 

common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/JsonConverter.java

@@ -228,29 +228,27 @@ public class JsonConverter {
     private static KeyValueProto buildNumericKeyValueProto(JsonPrimitive value, String key) {
         String valueAsString = value.getAsString();
         KeyValueProto.Builder builder = KeyValueProto.newBuilder().setKey(key);
-        if (valueAsString.contains("e") || valueAsString.contains("E")) {
+        var bd = new BigDecimal(valueAsString);
-            //TODO: correct value conversion. We should make sure that if the value can't fit into Long or Double, we should send String
+        if (bd.stripTrailingZeros().scale() <= 0) {
-            var bd = new BigDecimal(valueAsString);
-            if (bd.stripTrailingZeros().scale() <= 0) {
-                try {
-                    return builder.setType(KeyValueType.LONG_V).setLongV(bd.longValueExact()).build();
-                } catch (ArithmeticException e) {
-                    return builder.setType(KeyValueType.DOUBLE_V).setDoubleV(bd.doubleValue()).build();
-                }
-            } else {
-                return builder.setType(KeyValueType.DOUBLE_V).setDoubleV(bd.doubleValue()).build();
-            }
-        } else if (valueAsString.contains(".")) {
-            return builder.setType(KeyValueType.DOUBLE_V).setDoubleV(value.getAsDouble()).build();
-        } else {
             try {
-                long longValue = Long.parseLong(value.getAsString());
+                return builder.setType(KeyValueType.LONG_V).setLongV(bd.longValueExact()).build();
-                return builder.setType(KeyValueType.LONG_V).setLongV(longValue).build();
+            } catch (ArithmeticException e) {
-            } catch (NumberFormatException e) {
+                if (isTypeCastEnabled) {
-                //TODO: correct value conversion. We should make sure that if the value can't fit into Long or Double, we should send String
+                    return builder.setType(KeyValueType.STRING_V).setStringV(bd.toPlainString()).build();
-                return builder.setType(KeyValueType.DOUBLE_V).setDoubleV(new BigDecimal(valueAsString).doubleValue()).build();
+                } else {
+                    throw new JsonSyntaxException("Big integer values are not supported!");
+                }
+            }
+        } else {
+            if (bd.scale() <= 16) {
+                return builder.setType(KeyValueType.DOUBLE_V).setDoubleV(bd.doubleValue()).build();
+            } else if (isTypeCastEnabled) {
+                return builder.setType(KeyValueType.STRING_V).setStringV(bd.toPlainString()).build();
+            } else {
+                throw new JsonSyntaxException("Big integer values are not supported!");
             }
         }
+
     }
 
     public static TransportProtos.ToServerRpcRequestMsg convertToServerRpcRequest(JsonElement json, int requestId) throws JsonSyntaxException {
@@ -261,27 +259,24 @@ public class JsonConverter {
     private static void parseNumericValue(List<KvEntry> result, Entry<String, JsonElement> valueEntry, JsonPrimitive value) {
         String valueAsString = value.getAsString();
         String key = valueEntry.getKey();
-        if (valueAsString.contains("e") || valueAsString.contains("E")) {
+        var bd = new BigDecimal(valueAsString);
-            //TODO: correct value conversion. We should make sure that if the value can't fit into Long or Double, we should send String
+        if (bd.stripTrailingZeros().scale() <= 0) {
-            var bd = new BigDecimal(valueAsString);
-            if (bd.stripTrailingZeros().scale() <= 0) {
-                try {
-                    result.add(new LongDataEntry(key, bd.longValueExact()));
-                } catch (ArithmeticException e) {
-                    result.add(new DoubleDataEntry(key, bd.doubleValue()));
-                }
-            } else {
-                result.add(new DoubleDataEntry(key, bd.doubleValue()));
-            }
-        } else if (valueAsString.contains(".")) {
-            result.add(new DoubleDataEntry(key, value.getAsDouble()));
-        } else {
             try {
-                long longValue = Long.parseLong(value.getAsString());
+                result.add(new LongDataEntry(key, bd.longValueExact()));
-                result.add(new LongDataEntry(key, longValue));
+            } catch (ArithmeticException e) {
-            } catch (NumberFormatException e) {
+                if (isTypeCastEnabled) {
-                //TODO: correct value conversion. We should make sure that if the value can't fit into Long or Double, we should send String
+                    result.add(new StringDataEntry(key, bd.toPlainString()));
-                result.add(new DoubleDataEntry(key, new BigDecimal(valueAsString).doubleValue()));
+                } else {
+                    throw new JsonSyntaxException("Big integer values are not supported!");
+                }
+            }
+        } else {
+            if (bd.scale() <= 16) {
+                result.add(new DoubleDataEntry(key, bd.doubleValue()));
+            } else if (isTypeCastEnabled) {
+                result.add(new StringDataEntry(key, bd.toPlainString()));
+            } else {
+                throw new JsonSyntaxException("Big integer values are not supported!");
             }
         }
     }

common/transport/transport-api/src/test/java/JsonConverterTest.java

@@ -15,7 +15,9 @@
  */
 
 import com.google.gson.JsonParser;
+import com.google.gson.JsonSyntaxException;
 import org.junit.Assert;
+import org.junit.Before;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.mockito.junit.MockitoJUnitRunner;
@@ -28,6 +30,11 @@ public class JsonConverterTest {
 
     private static final JsonParser JSON_PARSER = new JsonParser();
 
+    @Before
+    public void before() {
+        JsonConverter.setTypeCastEnabled(true);
+    }
+
     @Test
     public void testParseBigDecimalAsLong() {
         var result = JsonConverter.convertToTelemetry(JSON_PARSER.parse("{\"meterReadingDelta\": 1E+1}"), 0L);
@@ -58,4 +65,33 @@ public class JsonConverterTest {
         Assert.assertEquals(11L, result.get(0L).get(0).getLongValue().get().longValue());
     }
 
+    @Test
+    public void testParseBigDecimalAsStringOutOfLongRange() {
+        var result = JsonConverter.convertToTelemetry(JSON_PARSER.parse("{\"meterReadingDelta\": 9.9701010061400066E19}"), 0L);
+        Assert.assertEquals("99701010061400066000", result.get(0L).get(0).getStrValue().get());
+    }
+
+    @Test
+    public void testParseBigDecimalAsStringOutOfLongRange2() {
+        var result = JsonConverter.convertToTelemetry(JSON_PARSER.parse("{\"meterReadingDelta\": 99701010061400066001}"), 0L);
+        Assert.assertEquals("99701010061400066001", result.get(0L).get(0).getStrValue().get());
+    }
+
+    @Test
+    public void testParseBigDecimalAsStringOutOfLongRange3() {
+        var result = JsonConverter.convertToTelemetry(JSON_PARSER.parse("{\"meterReadingDelta\": 1E19}"), 0L);
+        Assert.assertEquals("10000000000000000000", result.get(0L).get(0).getStrValue().get());
+    }
+
+    @Test(expected = JsonSyntaxException.class)
+    public void testParseBigDecimalOutOfLongRangeWithoutParsing() {
+        JsonConverter.setTypeCastEnabled(false);
+        JsonConverter.convertToTelemetry(JSON_PARSER.parse("{\"meterReadingDelta\": 89701010051400054084}"), 0L);
+    }
+
+    @Test(expected = JsonSyntaxException.class)
+    public void testParseBigDecimalOutOfLongRangeWithoutParsing2() {
+        JsonConverter.setTypeCastEnabled(false);
+        JsonConverter.convertToTelemetry(JSON_PARSER.parse("{\"meterReadingDelta\": 9.9701010061400066E19}"), 0L);
+    }
 }