Merge branch 'master' of github.com:thingsboard/thingsboard

application/src/main/data/upgrade/2.4.3/schema_update_psql_drop_partitions.sql

@@ -43,7 +43,7 @@ BEGIN
     into max_customer_ttl;
     max_ttl := GREATEST(system_ttl, max_customer_ttl, max_tenant_ttl);
     if max_ttl IS NOT NULL AND max_ttl > 0 THEN
-        date := to_timestamp(EXTRACT(EPOCH FROM current_timestamp) - (max_ttl / 1000));
+        date := to_timestamp(EXTRACT(EPOCH FROM current_timestamp) - max_ttl);
         partition_by_max_ttl_date := get_partition_by_max_ttl_date(partition_type, date);
         RAISE NOTICE 'Partition by max ttl: %', partition_by_max_ttl_date;
         IF partition_by_max_ttl_date IS NOT NULL THEN

application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java

@@ -193,6 +193,9 @@ public class ThingsboardInstallService {
                             databaseEntitiesUpgradeService.upgradeDatabase("3.2.1");
                         case "3.2.2":
                             log.info("Upgrading ThingsBoard from version 3.2.2 to 3.3.0 ...");
+                            if (databaseTsUpgradeService != null) {
+                                databaseTsUpgradeService.upgradeDatabase("3.2.2");
+                            }
                             databaseEntitiesUpgradeService.upgradeDatabase("3.2.2");
 
                             dataUpdateService.updateData("3.2.2");

application/src/main/java/org/thingsboard/server/service/install/PsqlTsDatabaseUpgradeService.java

@@ -209,6 +209,12 @@ public class PsqlTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgradeSe
                     executeQuery(conn, "DROP FUNCTION IF EXISTS delete_customer_records_from_ts_kv(character varying, character varying, bigint);");
                 }
                 break;
+            case "3.2.2":
+                try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) {
+                    log.info("Load Drop Partitions functions ...");
+                    loadSql(conn, LOAD_DROP_PARTITIONS_FUNCTIONS_SQL, "2.4.3");
+                }
+                break;
             default:
                 throw new RuntimeException("Unable to upgrade SQL database, unsupported fromVersion: " + fromVersion);
         }

common/queue/pom.xml

@@ -120,12 +120,16 @@
             <groupId>org.apache.curator</groupId>
             <artifactId>curator-recipes</artifactId>
         </dependency>
-
         <dependency>
             <groupId>junit</groupId>
             <artifactId>junit</artifactId>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.hamcrest</groupId>
+            <artifactId>hamcrest</artifactId>
+            <scope>test</scope>
+        </dependency>
         <dependency>
             <groupId>org.mockito</groupId>
             <artifactId>mockito-core</artifactId>

common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/secure/TbLwM2MDtlsCertificateVerifier.java

@@ -41,6 +41,7 @@ import org.thingsboard.server.common.transport.TransportServiceCallback;
 import org.thingsboard.server.common.transport.auth.ValidateDeviceCredentialsResponse;
 import org.thingsboard.server.common.transport.util.SslUtil;
 import org.thingsboard.server.gen.transport.TransportProtos;
+import org.thingsboard.server.queue.util.TbLwM2mTransportComponent;
 import org.thingsboard.server.transport.lwm2m.config.LwM2MTransportServerConfig;
 import org.thingsboard.server.transport.lwm2m.secure.credentials.LwM2MCredentials;
 import org.thingsboard.server.common.data.device.credentials.lwm2m.X509ClientCredentials;
@@ -61,6 +62,7 @@ import java.util.concurrent.TimeUnit;
 
 @Slf4j
 @Component
+@TbLwM2mTransportComponent
 @RequiredArgsConstructor
 public class TbLwM2MDtlsCertificateVerifier implements NewAdvancedCertificateVerifier {
 

common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/JsonConverter.java

@@ -228,29 +228,27 @@ public class JsonConverter {
     private static KeyValueProto buildNumericKeyValueProto(JsonPrimitive value, String key) {
         String valueAsString = value.getAsString();
         KeyValueProto.Builder builder = KeyValueProto.newBuilder().setKey(key);
-        if (valueAsString.contains("e") || valueAsString.contains("E")) {
+        var bd = new BigDecimal(valueAsString);
-            //TODO: correct value conversion. We should make sure that if the value can't fit into Long or Double, we should send String
+        if (bd.stripTrailingZeros().scale() <= 0) {
-            var bd = new BigDecimal(valueAsString);
-            if (bd.stripTrailingZeros().scale() <= 0) {
-                try {
-                    return builder.setType(KeyValueType.LONG_V).setLongV(bd.longValueExact()).build();
-                } catch (ArithmeticException e) {
-                    return builder.setType(KeyValueType.DOUBLE_V).setDoubleV(bd.doubleValue()).build();
-                }
-            } else {
-                return builder.setType(KeyValueType.DOUBLE_V).setDoubleV(bd.doubleValue()).build();
-            }
-        } else if (valueAsString.contains(".")) {
-            return builder.setType(KeyValueType.DOUBLE_V).setDoubleV(value.getAsDouble()).build();
-        } else {
             try {
-                long longValue = Long.parseLong(value.getAsString());
+                return builder.setType(KeyValueType.LONG_V).setLongV(bd.longValueExact()).build();
-                return builder.setType(KeyValueType.LONG_V).setLongV(longValue).build();
+            } catch (ArithmeticException e) {
-            } catch (NumberFormatException e) {
+                if (isTypeCastEnabled) {
-                //TODO: correct value conversion. We should make sure that if the value can't fit into Long or Double, we should send String
+                    return builder.setType(KeyValueType.STRING_V).setStringV(bd.toPlainString()).build();
-                return builder.setType(KeyValueType.DOUBLE_V).setDoubleV(new BigDecimal(valueAsString).doubleValue()).build();
+                } else {
+                    throw new JsonSyntaxException("Big integer values are not supported!");
+                }
+            }
+        } else {
+            if (bd.scale() <= 16) {
+                return builder.setType(KeyValueType.DOUBLE_V).setDoubleV(bd.doubleValue()).build();
+            } else if (isTypeCastEnabled) {
+                return builder.setType(KeyValueType.STRING_V).setStringV(bd.toPlainString()).build();
+            } else {
+                throw new JsonSyntaxException("Big integer values are not supported!");
             }
         }
+
     }
 
     public static TransportProtos.ToServerRpcRequestMsg convertToServerRpcRequest(JsonElement json, int requestId) throws JsonSyntaxException {
@@ -261,27 +259,24 @@ public class JsonConverter {
     private static void parseNumericValue(List<KvEntry> result, Entry<String, JsonElement> valueEntry, JsonPrimitive value) {
         String valueAsString = value.getAsString();
         String key = valueEntry.getKey();
-        if (valueAsString.contains("e") || valueAsString.contains("E")) {
+        var bd = new BigDecimal(valueAsString);
-            //TODO: correct value conversion. We should make sure that if the value can't fit into Long or Double, we should send String
+        if (bd.stripTrailingZeros().scale() <= 0) {
-            var bd = new BigDecimal(valueAsString);
-            if (bd.stripTrailingZeros().scale() <= 0) {
-                try {
-                    result.add(new LongDataEntry(key, bd.longValueExact()));
-                } catch (ArithmeticException e) {
-                    result.add(new DoubleDataEntry(key, bd.doubleValue()));
-                }
-            } else {
-                result.add(new DoubleDataEntry(key, bd.doubleValue()));
-            }
-        } else if (valueAsString.contains(".")) {
-            result.add(new DoubleDataEntry(key, value.getAsDouble()));
-        } else {
             try {
-                long longValue = Long.parseLong(value.getAsString());
+                result.add(new LongDataEntry(key, bd.longValueExact()));
-                result.add(new LongDataEntry(key, longValue));
+            } catch (ArithmeticException e) {
-            } catch (NumberFormatException e) {
+                if (isTypeCastEnabled) {
-                //TODO: correct value conversion. We should make sure that if the value can't fit into Long or Double, we should send String
+                    result.add(new StringDataEntry(key, bd.toPlainString()));
-                result.add(new DoubleDataEntry(key, new BigDecimal(valueAsString).doubleValue()));
+                } else {
+                    throw new JsonSyntaxException("Big integer values are not supported!");
+                }
+            }
+        } else {
+            if (bd.scale() <= 16) {
+                result.add(new DoubleDataEntry(key, bd.doubleValue()));
+            } else if (isTypeCastEnabled) {
+                result.add(new StringDataEntry(key, bd.toPlainString()));
+            } else {
+                throw new JsonSyntaxException("Big integer values are not supported!");
             }
         }
     }

common/transport/transport-api/src/test/java/JsonConverterTest.java

@@ -15,7 +15,9 @@
  */
 
 import com.google.gson.JsonParser;
+import com.google.gson.JsonSyntaxException;
 import org.junit.Assert;
+import org.junit.Before;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.mockito.junit.MockitoJUnitRunner;
@@ -28,6 +30,11 @@ public class JsonConverterTest {
 
     private static final JsonParser JSON_PARSER = new JsonParser();
 
+    @Before
+    public void before() {
+        JsonConverter.setTypeCastEnabled(true);
+    }
+
     @Test
     public void testParseBigDecimalAsLong() {
         var result = JsonConverter.convertToTelemetry(JSON_PARSER.parse("{\"meterReadingDelta\": 1E+1}"), 0L);
@@ -58,4 +65,33 @@ public class JsonConverterTest {
         Assert.assertEquals(11L, result.get(0L).get(0).getLongValue().get().longValue());
     }
 
+    @Test
+    public void testParseBigDecimalAsStringOutOfLongRange() {
+        var result = JsonConverter.convertToTelemetry(JSON_PARSER.parse("{\"meterReadingDelta\": 9.9701010061400066E19}"), 0L);
+        Assert.assertEquals("99701010061400066000", result.get(0L).get(0).getStrValue().get());
+    }
+
+    @Test
+    public void testParseBigDecimalAsStringOutOfLongRange2() {
+        var result = JsonConverter.convertToTelemetry(JSON_PARSER.parse("{\"meterReadingDelta\": 99701010061400066001}"), 0L);
+        Assert.assertEquals("99701010061400066001", result.get(0L).get(0).getStrValue().get());
+    }
+
+    @Test
+    public void testParseBigDecimalAsStringOutOfLongRange3() {
+        var result = JsonConverter.convertToTelemetry(JSON_PARSER.parse("{\"meterReadingDelta\": 1E19}"), 0L);
+        Assert.assertEquals("10000000000000000000", result.get(0L).get(0).getStrValue().get());
+    }
+
+    @Test(expected = JsonSyntaxException.class)
+    public void testParseBigDecimalOutOfLongRangeWithoutParsing() {
+        JsonConverter.setTypeCastEnabled(false);
+        JsonConverter.convertToTelemetry(JSON_PARSER.parse("{\"meterReadingDelta\": 89701010051400054084}"), 0L);
+    }
+
+    @Test(expected = JsonSyntaxException.class)
+    public void testParseBigDecimalOutOfLongRangeWithoutParsing2() {
+        JsonConverter.setTypeCastEnabled(false);
+        JsonConverter.convertToTelemetry(JSON_PARSER.parse("{\"meterReadingDelta\": 9.9701010061400066E19}"), 0L);
+    }
 }

dao/src/main/resources/sql/schema-ts-psql.sql

@@ -63,7 +63,7 @@ BEGIN
     into max_customer_ttl;
     max_ttl := GREATEST(system_ttl, max_customer_ttl, max_tenant_ttl);
     if max_ttl IS NOT NULL AND max_ttl > 0 THEN
-        date := to_timestamp(EXTRACT(EPOCH FROM current_timestamp) - (max_ttl / 1000));
+        date := to_timestamp(EXTRACT(EPOCH FROM current_timestamp) - max_ttl);
         partition_by_max_ttl_date := get_partition_by_max_ttl_date(partition_type, date);
         RAISE NOTICE 'Partition by max ttl: %', partition_by_max_ttl_date;
         IF partition_by_max_ttl_date IS NOT NULL THEN
@@ -104,11 +104,12 @@ BEGIN
                                 END IF;
                             END IF;
                         END IF;
-                    END IF;
+                        IF partition_to_delete IS NOT NULL THEN
-                    IF partition_to_delete IS NOT NULL THEN
+                            RAISE NOTICE 'Partition to delete by max ttl: %', partition_to_delete;
-                        RAISE NOTICE 'Partition to delete by max ttl: %', partition_to_delete;
+                            EXECUTE format('DROP TABLE IF EXISTS %I', partition_to_delete);
-                        EXECUTE format('DROP TABLE %I', partition_to_delete);
+                            partition_to_delete := NULL;
-                        deleted := deleted + 1;
+                            deleted := deleted + 1;
+                        END IF;
                     END IF;
                 END LOOP;
         END IF;

pom.xml

@@ -48,6 +48,7 @@
         <json-path.version>2.2.0</json-path.version>
         <junit.version>4.12</junit.version>
         <jupiter.version>5.7.1</jupiter.version>
+        <hamcrest.version>2.2</hamcrest.version>
         <slf4j.version>1.7.7</slf4j.version>
         <logback.version>1.2.3</logback.version>
         <mockito.version>3.3.3</mockito.version>
@@ -94,7 +95,7 @@
         <hsqldb.version>2.5.0</hsqldb.version>
         <dbunit.version>2.5.3</dbunit.version>
         <spring-test-dbunit.version>1.2.1</spring-test-dbunit.version>
-        <postgresql.driver.version>42.2.16</postgresql.driver.version>
+        <postgresql.driver.version>42.2.20</postgresql.driver.version>
         <sonar.exclusions>org/thingsboard/server/gen/**/*,
             org/thingsboard/server/extensions/core/plugin/telemetry/gen/**/*
         </sonar.exclusions>
@@ -1371,6 +1372,12 @@
                 <version>${junit.version}</version>
                 <scope>test</scope>
             </dependency>
+            <dependency>
+                <groupId>org.hamcrest</groupId>
+                <artifactId>hamcrest</artifactId>
+                <version>${hamcrest.version}</version>
+                <scope>test</scope>
+            </dependency>
             <dependency>
                 <groupId>org.junit.jupiter</groupId>
                 <artifactId>junit-jupiter-params</artifactId>