dendi-wang/thingsboard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' of github.com:thingsboard/thingsboard

Browse Source
This commit is contained in:
Igor Kulikov 2021-05-27 12:34:38 +03:00
commit 1176f0de26
9 changed files with 103 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
application/src/main/data/upgrade/2.4.3/schema_update_psql_drop_partitions.sql
View File

@ -43,7 +43,7 @@ BEGIN
into max_customer_ttl;
max_ttl := GREATEST(system_ttl, max_customer_ttl, max_tenant_ttl);
if max_ttl IS NOT NULL AND max_ttl > 0 THEN
date := to_timestamp(EXTRACT(EPOCH FROM current_timestamp) - (max_ttl / 1000));
date := to_timestamp(EXTRACT(EPOCH FROM current_timestamp) - max_ttl);
partition_by_max_ttl_date := get_partition_by_max_ttl_date(partition_type, date);
RAISE NOTICE 'Partition by max ttl: %', partition_by_max_ttl_date;
IF partition_by_max_ttl_date IS NOT NULL THEN

3
application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java
View File

@ -193,6 +193,9 @@ public class ThingsboardInstallService {
databaseEntitiesUpgradeService.upgradeDatabase("3.2.1");
case "3.2.2":
log.info("Upgrading ThingsBoard from version 3.2.2 to 3.3.0 ...");
if (databaseTsUpgradeService != null) {
databaseTsUpgradeService.upgradeDatabase("3.2.2");
}
databaseEntitiesUpgradeService.upgradeDatabase("3.2.2");
dataUpdateService.updateData("3.2.2");

6
application/src/main/java/org/thingsboard/server/service/install/PsqlTsDatabaseUpgradeService.java
View File

@ -209,6 +209,12 @@ public class PsqlTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgradeSe
executeQuery(conn, "DROP FUNCTION IF EXISTS delete_customer_records_from_ts_kv(character varying, character varying, bigint);");
}
break;
case "3.2.2":
try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) {
log.info("Load Drop Partitions functions ...");
loadSql(conn, LOAD_DROP_PARTITIONS_FUNCTIONS_SQL, "2.4.3");
}
break;
default:
throw new RuntimeException("Unable to upgrade SQL database, unsupported fromVersion: " + fromVersion);
}

6
common/queue/pom.xml
View File

@ -120,12 +120,16 @@
<groupId>org.apache.curator</groupId>
<artifactId>curator-recipes</artifactId>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.hamcrest</groupId>
<artifactId>hamcrest</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>

2
common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/secure/TbLwM2MDtlsCertificateVerifier.java
View File

@ -41,6 +41,7 @@ import org.thingsboard.server.common.transport.TransportServiceCallback;
import org.thingsboard.server.common.transport.auth.ValidateDeviceCredentialsResponse;
import org.thingsboard.server.common.transport.util.SslUtil;
import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.queue.util.TbLwM2mTransportComponent;
import org.thingsboard.server.transport.lwm2m.config.LwM2MTransportServerConfig;
import org.thingsboard.server.transport.lwm2m.secure.credentials.LwM2MCredentials;
import org.thingsboard.server.common.data.device.credentials.lwm2m.X509ClientCredentials;
@ -61,6 +62,7 @@ import java.util.concurrent.TimeUnit;
@Slf4j
@Component
@TbLwM2mTransportComponent
@RequiredArgsConstructor
public class TbLwM2MDtlsCertificateVerifier implements NewAdvancedCertificateVerifier {

43
common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/JsonConverter.java
View File

@ -228,29 +228,27 @@ public class JsonConverter {
private static KeyValueProto buildNumericKeyValueProto(JsonPrimitive value, String key) {
String valueAsString = value.getAsString();
KeyValueProto.Builder builder = KeyValueProto.newBuilder().setKey(key);
if (valueAsString.contains("e") || valueAsString.contains("E")) {
//TODO: correct value conversion. We should make sure that if the value can't fit into Long or Double, we should send String
var bd = new BigDecimal(valueAsString);
if (bd.stripTrailingZeros().scale() <= 0) {
try {
return builder.setType(KeyValueType.LONG_V).setLongV(bd.longValueExact()).build();
} catch (ArithmeticException e) {
return builder.setType(KeyValueType.DOUBLE_V).setDoubleV(bd.doubleValue()).build();
if (isTypeCastEnabled) {
return builder.setType(KeyValueType.STRING_V).setStringV(bd.toPlainString()).build();
} else {
throw new JsonSyntaxException("Big integer values are not supported!");
}
}
} else {
if (bd.scale() <= 16) {
return builder.setType(KeyValueType.DOUBLE_V).setDoubleV(bd.doubleValue()).build();
}
} else if (valueAsString.contains(".")) {
return builder.setType(KeyValueType.DOUBLE_V).setDoubleV(value.getAsDouble()).build();
} else if (isTypeCastEnabled) {
return builder.setType(KeyValueType.STRING_V).setStringV(bd.toPlainString()).build();
} else {
try {
long longValue = Long.parseLong(value.getAsString());
return builder.setType(KeyValueType.LONG_V).setLongV(longValue).build();
} catch (NumberFormatException e) {
//TODO: correct value conversion. We should make sure that if the value can't fit into Long or Double, we should send String
return builder.setType(KeyValueType.DOUBLE_V).setDoubleV(new BigDecimal(valueAsString).doubleValue()).build();
throw new JsonSyntaxException("Big integer values are not supported!");
}
}
}
public static TransportProtos.ToServerRpcRequestMsg convertToServerRpcRequest(JsonElement json, int requestId) throws JsonSyntaxException {
@ -261,27 +259,24 @@ public class JsonConverter {
private static void parseNumericValue(List<KvEntry> result, Entry<String, JsonElement> valueEntry, JsonPrimitive value) {
String valueAsString = value.getAsString();
String key = valueEntry.getKey();
if (valueAsString.contains("e") || valueAsString.contains("E")) {
//TODO: correct value conversion. We should make sure that if the value can't fit into Long or Double, we should send String
var bd = new BigDecimal(valueAsString);
if (bd.stripTrailingZeros().scale() <= 0) {
try {
result.add(new LongDataEntry(key, bd.longValueExact()));
} catch (ArithmeticException e) {
result.add(new DoubleDataEntry(key, bd.doubleValue()));
if (isTypeCastEnabled) {
result.add(new StringDataEntry(key, bd.toPlainString()));
} else {
throw new JsonSyntaxException("Big integer values are not supported!");
}
}
} else {
if (bd.scale() <= 16) {
result.add(new DoubleDataEntry(key, bd.doubleValue()));
}
} else if (valueAsString.contains(".")) {
result.add(new DoubleDataEntry(key, value.getAsDouble()));
} else if (isTypeCastEnabled) {
result.add(new StringDataEntry(key, bd.toPlainString()));
} else {
try {
long longValue = Long.parseLong(value.getAsString());
result.add(new LongDataEntry(key, longValue));
} catch (NumberFormatException e) {
//TODO: correct value conversion. We should make sure that if the value can't fit into Long or Double, we should send String
result.add(new DoubleDataEntry(key, new BigDecimal(valueAsString).doubleValue()));
throw new JsonSyntaxException("Big integer values are not supported!");
}
}
}

36
common/transport/transport-api/src/test/java/JsonConverterTest.java
View File

@ -15,7 +15,9 @@
*/
import com.google.gson.JsonParser;
import com.google.gson.JsonSyntaxException;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.junit.MockitoJUnitRunner;
@ -28,6 +30,11 @@ public class JsonConverterTest {
private static final JsonParser JSON_PARSER = new JsonParser();
@Before
public void before() {
JsonConverter.setTypeCastEnabled(true);
}
@Test
public void testParseBigDecimalAsLong() {
var result = JsonConverter.convertToTelemetry(JSON_PARSER.parse("{\"meterReadingDelta\": 1E+1}"), 0L);
@ -58,4 +65,33 @@ public class JsonConverterTest {
Assert.assertEquals(11L, result.get(0L).get(0).getLongValue().get().longValue());
}
@Test
public void testParseBigDecimalAsStringOutOfLongRange() {
var result = JsonConverter.convertToTelemetry(JSON_PARSER.parse("{\"meterReadingDelta\": 9.9701010061400066E19}"), 0L);
Assert.assertEquals("99701010061400066000", result.get(0L).get(0).getStrValue().get());
}
@Test
public void testParseBigDecimalAsStringOutOfLongRange2() {
var result = JsonConverter.convertToTelemetry(JSON_PARSER.parse("{\"meterReadingDelta\": 99701010061400066001}"), 0L);
Assert.assertEquals("99701010061400066001", result.get(0L).get(0).getStrValue().get());
}
@Test
public void testParseBigDecimalAsStringOutOfLongRange3() {
var result = JsonConverter.convertToTelemetry(JSON_PARSER.parse("{\"meterReadingDelta\": 1E19}"), 0L);
Assert.assertEquals("10000000000000000000", result.get(0L).get(0).getStrValue().get());
}
@Test(expected = JsonSyntaxException.class)
public void testParseBigDecimalOutOfLongRangeWithoutParsing() {
JsonConverter.setTypeCastEnabled(false);
JsonConverter.convertToTelemetry(JSON_PARSER.parse("{\"meterReadingDelta\": 89701010051400054084}"), 0L);
}
@Test(expected = JsonSyntaxException.class)
public void testParseBigDecimalOutOfLongRangeWithoutParsing2() {
JsonConverter.setTypeCastEnabled(false);
JsonConverter.convertToTelemetry(JSON_PARSER.parse("{\"meterReadingDelta\": 9.9701010061400066E19}"), 0L);
}
}

7
dao/src/main/resources/sql/schema-ts-psql.sql
View File

@ -63,7 +63,7 @@ BEGIN
into max_customer_ttl;
max_ttl := GREATEST(system_ttl, max_customer_ttl, max_tenant_ttl);
if max_ttl IS NOT NULL AND max_ttl > 0 THEN
date := to_timestamp(EXTRACT(EPOCH FROM current_timestamp) - (max_ttl / 1000));
date := to_timestamp(EXTRACT(EPOCH FROM current_timestamp) - max_ttl);
partition_by_max_ttl_date := get_partition_by_max_ttl_date(partition_type, date);
RAISE NOTICE 'Partition by max ttl: %', partition_by_max_ttl_date;
IF partition_by_max_ttl_date IS NOT NULL THEN
@ -104,12 +104,13 @@ BEGIN
END IF;
END IF;
END IF;
END IF;
IF partition_to_delete IS NOT NULL THEN
RAISE NOTICE 'Partition to delete by max ttl: %', partition_to_delete;
EXECUTE format('DROP TABLE %I', partition_to_delete);
EXECUTE format('DROP TABLE IF EXISTS %I', partition_to_delete);
partition_to_delete := NULL;
deleted := deleted + 1;
END IF;
END IF;
END LOOP;
END IF;
END IF;

9
pom.xml
View File

@ -48,6 +48,7 @@
<json-path.version>2.2.0</json-path.version>
<junit.version>4.12</junit.version>
<jupiter.version>5.7.1</jupiter.version>
<hamcrest.version>2.2</hamcrest.version>
<slf4j.version>1.7.7</slf4j.version>
<logback.version>1.2.3</logback.version>
<mockito.version>3.3.3</mockito.version>
@ -94,7 +95,7 @@
<hsqldb.version>2.5.0</hsqldb.version>
<dbunit.version>2.5.3</dbunit.version>
<spring-test-dbunit.version>1.2.1</spring-test-dbunit.version>
<postgresql.driver.version>42.2.16</postgresql.driver.version>
<postgresql.driver.version>42.2.20</postgresql.driver.version>
<sonar.exclusions>org/thingsboard/server/gen/**/*,
org/thingsboard/server/extensions/core/plugin/telemetry/gen/**/*
</sonar.exclusions>
@ -1371,6 +1372,12 @@
<version>${junit.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.hamcrest</groupId>
<artifactId>hamcrest</artifactId>
<version>${hamcrest.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-params</artifactId>