diff --git a/application/src/main/data/upgrade/2.4.3/schema_update_psql_drop_partitions.sql b/application/src/main/data/upgrade/2.4.3/schema_update_psql_drop_partitions.sql index 3c2d43e197..fcc5c6f232 100644 --- a/application/src/main/data/upgrade/2.4.3/schema_update_psql_drop_partitions.sql +++ b/application/src/main/data/upgrade/2.4.3/schema_update_psql_drop_partitions.sql @@ -43,7 +43,7 @@ BEGIN into max_customer_ttl; max_ttl := GREATEST(system_ttl, max_customer_ttl, max_tenant_ttl); if max_ttl IS NOT NULL AND max_ttl > 0 THEN - date := to_timestamp(EXTRACT(EPOCH FROM current_timestamp) - (max_ttl / 1000)); + date := to_timestamp(EXTRACT(EPOCH FROM current_timestamp) - max_ttl); partition_by_max_ttl_date := get_partition_by_max_ttl_date(partition_type, date); RAISE NOTICE 'Partition by max ttl: %', partition_by_max_ttl_date; IF partition_by_max_ttl_date IS NOT NULL THEN diff --git a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java index 8ea8b85017..267fb31ad7 100644 --- a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java +++ b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java @@ -193,6 +193,9 @@ public class ThingsboardInstallService { databaseEntitiesUpgradeService.upgradeDatabase("3.2.1"); case "3.2.2": log.info("Upgrading ThingsBoard from version 3.2.2 to 3.3.0 ..."); + if (databaseTsUpgradeService != null) { + databaseTsUpgradeService.upgradeDatabase("3.2.2"); + } databaseEntitiesUpgradeService.upgradeDatabase("3.2.2"); dataUpdateService.updateData("3.2.2"); diff --git a/application/src/main/java/org/thingsboard/server/service/install/PsqlTsDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/PsqlTsDatabaseUpgradeService.java index 8e6f4859e5..835b27b71c 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/PsqlTsDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/PsqlTsDatabaseUpgradeService.java @@ -209,6 +209,12 @@ public class PsqlTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgradeSe executeQuery(conn, "DROP FUNCTION IF EXISTS delete_customer_records_from_ts_kv(character varying, character varying, bigint);"); } break; + case "3.2.2": + try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) { + log.info("Load Drop Partitions functions ..."); + loadSql(conn, LOAD_DROP_PARTITIONS_FUNCTIONS_SQL, "2.4.3"); + } + break; default: throw new RuntimeException("Unable to upgrade SQL database, unsupported fromVersion: " + fromVersion); } diff --git a/common/queue/pom.xml b/common/queue/pom.xml index 31ec44e8f7..2b1d87fc7b 100644 --- a/common/queue/pom.xml +++ b/common/queue/pom.xml @@ -120,12 +120,16 @@ org.apache.curator curator-recipes - junit junit test + + org.hamcrest + hamcrest + test + org.mockito mockito-core diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/secure/TbLwM2MDtlsCertificateVerifier.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/secure/TbLwM2MDtlsCertificateVerifier.java index e83532d2cc..7b81e733bc 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/secure/TbLwM2MDtlsCertificateVerifier.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/secure/TbLwM2MDtlsCertificateVerifier.java @@ -41,6 +41,7 @@ import org.thingsboard.server.common.transport.TransportServiceCallback; import org.thingsboard.server.common.transport.auth.ValidateDeviceCredentialsResponse; import org.thingsboard.server.common.transport.util.SslUtil; import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.util.TbLwM2mTransportComponent; import org.thingsboard.server.transport.lwm2m.config.LwM2MTransportServerConfig; import org.thingsboard.server.transport.lwm2m.secure.credentials.LwM2MCredentials; import org.thingsboard.server.common.data.device.credentials.lwm2m.X509ClientCredentials; @@ -61,6 +62,7 @@ import java.util.concurrent.TimeUnit; @Slf4j @Component +@TbLwM2mTransportComponent @RequiredArgsConstructor public class TbLwM2MDtlsCertificateVerifier implements NewAdvancedCertificateVerifier { diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/JsonConverter.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/JsonConverter.java index 4db4aa9bd8..1abc513b23 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/JsonConverter.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/JsonConverter.java @@ -228,29 +228,27 @@ public class JsonConverter { private static KeyValueProto buildNumericKeyValueProto(JsonPrimitive value, String key) { String valueAsString = value.getAsString(); KeyValueProto.Builder builder = KeyValueProto.newBuilder().setKey(key); - if (valueAsString.contains("e") || valueAsString.contains("E")) { - //TODO: correct value conversion. We should make sure that if the value can't fit into Long or Double, we should send String - var bd = new BigDecimal(valueAsString); - if (bd.stripTrailingZeros().scale() <= 0) { - try { - return builder.setType(KeyValueType.LONG_V).setLongV(bd.longValueExact()).build(); - } catch (ArithmeticException e) { - return builder.setType(KeyValueType.DOUBLE_V).setDoubleV(bd.doubleValue()).build(); - } - } else { - return builder.setType(KeyValueType.DOUBLE_V).setDoubleV(bd.doubleValue()).build(); - } - } else if (valueAsString.contains(".")) { - return builder.setType(KeyValueType.DOUBLE_V).setDoubleV(value.getAsDouble()).build(); - } else { + var bd = new BigDecimal(valueAsString); + if (bd.stripTrailingZeros().scale() <= 0) { try { - long longValue = Long.parseLong(value.getAsString()); - return builder.setType(KeyValueType.LONG_V).setLongV(longValue).build(); - } catch (NumberFormatException e) { - //TODO: correct value conversion. We should make sure that if the value can't fit into Long or Double, we should send String - return builder.setType(KeyValueType.DOUBLE_V).setDoubleV(new BigDecimal(valueAsString).doubleValue()).build(); + return builder.setType(KeyValueType.LONG_V).setLongV(bd.longValueExact()).build(); + } catch (ArithmeticException e) { + if (isTypeCastEnabled) { + return builder.setType(KeyValueType.STRING_V).setStringV(bd.toPlainString()).build(); + } else { + throw new JsonSyntaxException("Big integer values are not supported!"); + } + } + } else { + if (bd.scale() <= 16) { + return builder.setType(KeyValueType.DOUBLE_V).setDoubleV(bd.doubleValue()).build(); + } else if (isTypeCastEnabled) { + return builder.setType(KeyValueType.STRING_V).setStringV(bd.toPlainString()).build(); + } else { + throw new JsonSyntaxException("Big integer values are not supported!"); } } + } public static TransportProtos.ToServerRpcRequestMsg convertToServerRpcRequest(JsonElement json, int requestId) throws JsonSyntaxException { @@ -261,27 +259,24 @@ public class JsonConverter { private static void parseNumericValue(List result, Entry valueEntry, JsonPrimitive value) { String valueAsString = value.getAsString(); String key = valueEntry.getKey(); - if (valueAsString.contains("e") || valueAsString.contains("E")) { - //TODO: correct value conversion. We should make sure that if the value can't fit into Long or Double, we should send String - var bd = new BigDecimal(valueAsString); - if (bd.stripTrailingZeros().scale() <= 0) { - try { - result.add(new LongDataEntry(key, bd.longValueExact())); - } catch (ArithmeticException e) { - result.add(new DoubleDataEntry(key, bd.doubleValue())); - } - } else { - result.add(new DoubleDataEntry(key, bd.doubleValue())); - } - } else if (valueAsString.contains(".")) { - result.add(new DoubleDataEntry(key, value.getAsDouble())); - } else { + var bd = new BigDecimal(valueAsString); + if (bd.stripTrailingZeros().scale() <= 0) { try { - long longValue = Long.parseLong(value.getAsString()); - result.add(new LongDataEntry(key, longValue)); - } catch (NumberFormatException e) { - //TODO: correct value conversion. We should make sure that if the value can't fit into Long or Double, we should send String - result.add(new DoubleDataEntry(key, new BigDecimal(valueAsString).doubleValue())); + result.add(new LongDataEntry(key, bd.longValueExact())); + } catch (ArithmeticException e) { + if (isTypeCastEnabled) { + result.add(new StringDataEntry(key, bd.toPlainString())); + } else { + throw new JsonSyntaxException("Big integer values are not supported!"); + } + } + } else { + if (bd.scale() <= 16) { + result.add(new DoubleDataEntry(key, bd.doubleValue())); + } else if (isTypeCastEnabled) { + result.add(new StringDataEntry(key, bd.toPlainString())); + } else { + throw new JsonSyntaxException("Big integer values are not supported!"); } } } diff --git a/common/transport/transport-api/src/test/java/JsonConverterTest.java b/common/transport/transport-api/src/test/java/JsonConverterTest.java index dc28b268f5..5aa5c28c0e 100644 --- a/common/transport/transport-api/src/test/java/JsonConverterTest.java +++ b/common/transport/transport-api/src/test/java/JsonConverterTest.java @@ -15,7 +15,9 @@ */ import com.google.gson.JsonParser; +import com.google.gson.JsonSyntaxException; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.junit.MockitoJUnitRunner; @@ -28,6 +30,11 @@ public class JsonConverterTest { private static final JsonParser JSON_PARSER = new JsonParser(); + @Before + public void before() { + JsonConverter.setTypeCastEnabled(true); + } + @Test public void testParseBigDecimalAsLong() { var result = JsonConverter.convertToTelemetry(JSON_PARSER.parse("{\"meterReadingDelta\": 1E+1}"), 0L); @@ -58,4 +65,33 @@ public class JsonConverterTest { Assert.assertEquals(11L, result.get(0L).get(0).getLongValue().get().longValue()); } + @Test + public void testParseBigDecimalAsStringOutOfLongRange() { + var result = JsonConverter.convertToTelemetry(JSON_PARSER.parse("{\"meterReadingDelta\": 9.9701010061400066E19}"), 0L); + Assert.assertEquals("99701010061400066000", result.get(0L).get(0).getStrValue().get()); + } + + @Test + public void testParseBigDecimalAsStringOutOfLongRange2() { + var result = JsonConverter.convertToTelemetry(JSON_PARSER.parse("{\"meterReadingDelta\": 99701010061400066001}"), 0L); + Assert.assertEquals("99701010061400066001", result.get(0L).get(0).getStrValue().get()); + } + + @Test + public void testParseBigDecimalAsStringOutOfLongRange3() { + var result = JsonConverter.convertToTelemetry(JSON_PARSER.parse("{\"meterReadingDelta\": 1E19}"), 0L); + Assert.assertEquals("10000000000000000000", result.get(0L).get(0).getStrValue().get()); + } + + @Test(expected = JsonSyntaxException.class) + public void testParseBigDecimalOutOfLongRangeWithoutParsing() { + JsonConverter.setTypeCastEnabled(false); + JsonConverter.convertToTelemetry(JSON_PARSER.parse("{\"meterReadingDelta\": 89701010051400054084}"), 0L); + } + + @Test(expected = JsonSyntaxException.class) + public void testParseBigDecimalOutOfLongRangeWithoutParsing2() { + JsonConverter.setTypeCastEnabled(false); + JsonConverter.convertToTelemetry(JSON_PARSER.parse("{\"meterReadingDelta\": 9.9701010061400066E19}"), 0L); + } } diff --git a/dao/src/main/resources/sql/schema-ts-psql.sql b/dao/src/main/resources/sql/schema-ts-psql.sql index da4ea0748b..5683cc0a17 100644 --- a/dao/src/main/resources/sql/schema-ts-psql.sql +++ b/dao/src/main/resources/sql/schema-ts-psql.sql @@ -63,7 +63,7 @@ BEGIN into max_customer_ttl; max_ttl := GREATEST(system_ttl, max_customer_ttl, max_tenant_ttl); if max_ttl IS NOT NULL AND max_ttl > 0 THEN - date := to_timestamp(EXTRACT(EPOCH FROM current_timestamp) - (max_ttl / 1000)); + date := to_timestamp(EXTRACT(EPOCH FROM current_timestamp) - max_ttl); partition_by_max_ttl_date := get_partition_by_max_ttl_date(partition_type, date); RAISE NOTICE 'Partition by max ttl: %', partition_by_max_ttl_date; IF partition_by_max_ttl_date IS NOT NULL THEN @@ -104,11 +104,12 @@ BEGIN END IF; END IF; END IF; - END IF; - IF partition_to_delete IS NOT NULL THEN - RAISE NOTICE 'Partition to delete by max ttl: %', partition_to_delete; - EXECUTE format('DROP TABLE %I', partition_to_delete); - deleted := deleted + 1; + IF partition_to_delete IS NOT NULL THEN + RAISE NOTICE 'Partition to delete by max ttl: %', partition_to_delete; + EXECUTE format('DROP TABLE IF EXISTS %I', partition_to_delete); + partition_to_delete := NULL; + deleted := deleted + 1; + END IF; END IF; END LOOP; END IF; diff --git a/pom.xml b/pom.xml index 29a373f2e4..e09dedf49f 100755 --- a/pom.xml +++ b/pom.xml @@ -48,6 +48,7 @@ 2.2.0 4.12 5.7.1 + 2.2 1.7.7 1.2.3 3.3.3 @@ -94,7 +95,7 @@ 2.5.0 2.5.3 1.2.1 - 42.2.16 + 42.2.20 org/thingsboard/server/gen/**/*, org/thingsboard/server/extensions/core/plugin/telemetry/gen/**/* @@ -1371,6 +1372,12 @@ ${junit.version} test + + org.hamcrest + hamcrest + ${hamcrest.version} + test + org.junit.jupiter junit-jupiter-params