dendi-wang/thingsboard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

dao tests: PostgreSqlDaoServiceTestSuite added using org.testcontainers

Browse Source
This commit is contained in:
Sergey Matvienko 2021-09-07 20:23:24 +03:00
parent 1adaef31f5
commit adb9a8d150
8 changed files with 363 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
dao/pom.xml
View File

@ -201,6 +201,11 @@
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-jpa</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-test</artifactId>
@ -211,6 +216,16 @@
<artifactId>hsqldb</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>postgresql</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>jdbc</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-context-support</artifactId>

76
dao/src/test/java/org/thingsboard/server/dao/PostgreSqlDaoServiceTestSuite.java Normal file
View File

@ -0,0 +1,76 @@
/**
* ThingsBoard, Inc. ("COMPANY") CONFIDENTIAL
*
* Copyright © 2016-2021 ThingsBoard, Inc. All Rights Reserved.
*
* NOTICE: All information contained herein is, and remains
* the property of ThingsBoard, Inc. and its suppliers,
* if any. The intellectual and technical concepts contained
* herein are proprietary to ThingsBoard, Inc.
* and its suppliers and may be covered by U.S. and Foreign Patents,
* patents in process, and are protected by trade secret or copyright law.
*
* Dissemination of this information or reproduction of this material is strictly forbidden
* unless prior written permission is obtained from COMPANY.
*
* Access to the source code contained herein is hereby forbidden to anyone except current COMPANY employees,
* managers or contractors who have executed Confidentiality and Non-disclosure agreements
* explicitly covering such access.
*
* The copyright notice above does not evidence any actual or intended publication
* or disclosure of this source code, which includes
* information that is confidential and/or proprietary, and is a trade secret, of COMPANY.
* ANY REPRODUCTION, MODIFICATION, DISTRIBUTION, PUBLIC PERFORMANCE,
* OR PUBLIC DISPLAY OF OR THROUGH USE OF THIS SOURCE CODE WITHOUT
* THE EXPRESS WRITTEN CONSENT OF COMPANY IS STRICTLY PROHIBITED,
* AND IN VIOLATION OF APPLICABLE LAWS AND INTERNATIONAL TREATIES.
* THE RECEIPT OR POSSESSION OF THIS SOURCE CODE AND/OR RELATED INFORMATION
* DOES NOT CONVEY OR IMPLY ANY RIGHTS TO REPRODUCE, DISCLOSE OR DISTRIBUTE ITS CONTENTS,
* OR TO MANUFACTURE, USE, OR SELL ANYTHING THAT IT MAY DESCRIBE, IN WHOLE OR IN PART.
*/
package org.thingsboard.server.dao;
import org.junit.ClassRule;
import org.junit.extensions.cpsuite.ClasspathSuite;
import org.junit.extensions.cpsuite.ClasspathSuite.ClassnameFilters;
import org.junit.runner.RunWith;
import java.util.Arrays;
@RunWith(ClasspathSuite.class)
@ClassnameFilters({
"org.thingsboard.server.dao.service.psql.*SqlTest",
"org.thingsboard.server.dao.service.attributes.psql.*SqlTest",
"org.thingsboard.server.dao.service.event.psql.*SqlTest",
"org.thingsboard.server.dao.service.timeseries.psql.*SqlTest"
})
public class PostgreSqlDaoServiceTestSuite {
@ClassRule
public static CustomSqlUnit sqlUnit = new CustomSqlUnit(
Arrays.asList("sql/schema-ts-psql.sql", "sql/schema-entities.sql", "sql/schema-entities-idx.sql"
, "sql/system-data.sql"
, "sql/system-test-psql.sql"
),
"sql/psql/drop-all-tables.sql",
"psql-test.properties"
);
// @ClassRule
// public static CustomSqlUnit sqlUnit = new CustomSqlUnit(
// Arrays.asList("sql/schema-ts-psql.sql"
// , "sql/schema-entities.sql", "sql/schema-entities-idx.sql"
// , "sql/system-data.sql", "sql/system-test.sql"
// ),
// "sql/psql/drop-all-tables.sql",
// "sql-test.properties"
// );
// @ClassRule
// public static CustomSqlUnit sqlUnit = new CustomSqlUnit(
// Arrays.asList("sql/schema-timescale.sql", "sql/schema-entities.sql", "sql/schema-entities-idx.sql", "sql/system-data.sql", "sql/system-test.sql"),
// "sql/timescale/drop-all-tables.sql",
// "sql-test.properties"
// );
}

177
dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java
View File

@ -17,15 +17,18 @@ package org.thingsboard.server.dao.service;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.RandomStringUtils;
import org.apache.commons.lang3.RandomUtils;
import org.hamcrest.Matchers;
import org.jetbrains.annotations.NotNull;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.core.ResultSetExtractor;
import org.thingsboard.server.common.data.DataConstants;
import org.thingsboard.server.common.data.Device;
import org.thingsboard.server.common.data.EntityType;
@ -80,14 +83,17 @@ import java.util.Comparator;
import java.util.List;
import java.util.Random;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import static org.junit.Assert.assertEquals;
import static org.hamcrest.MatcherAssert.assertThat;
@Slf4j
public abstract class BaseEntityServiceTest extends AbstractServiceTest {
static final int ENTITY_COUNT = 5;
@Autowired
private AttributesService attributesService;
@ -525,7 +531,7 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest {
}
private void createTestHierarchy(TenantId tenantId, List<Asset> assets, List<Device> devices, List<Long> consumptions, List<Long> highConsumptions, List<Long> temperatures, List<Long> highTemperatures) throws InterruptedException {
for (int i = 0; i < 5; i++) {
for (int i = 0; i < ENTITY_COUNT; i++) {
Asset asset = new Asset();
asset.setTenantId(tenantId);
asset.setName("Asset" + i);
@ -535,18 +541,19 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest {
//TO make sure devices have different created time
Thread.sleep(1);
assets.add(asset);
EntityRelation er = new EntityRelation();
er.setFrom(tenantId);
er.setTo(asset.getId());
er.setType("Manages");
er.setTypeGroup(RelationTypeGroup.COMMON);
relationService.saveRelation(tenantId, er);
createRelation(tenantId, "Manages", tenantId, asset.getId());
long consumption = (long) (Math.random() * 100);
consumptions.add(consumption);
if (consumption > 50) {
highConsumptions.add(consumption);
}
for (int j = 0; j < 5; j++) {
//tenant -> asset : one-to-one but many edges
for (int n = 0; n < ENTITY_COUNT; n++) {
createRelation(tenantId, "UseCase-" + n, tenantId, asset.getId());
}
for (int j = 0; j < ENTITY_COUNT; j++) {
Device device = new Device();
device.setTenantId(tenantId);
device.setName("A" + i + "Device" + j);
@ -556,27 +563,158 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest {
//TO make sure devices have different created time
Thread.sleep(1);
devices.add(device);
er = new EntityRelation();
er.setFrom(asset.getId());
er.setTo(device.getId());
er.setType("Contains");
er.setTypeGroup(RelationTypeGroup.COMMON);
relationService.saveRelation(tenantId, er);
createRelation(tenantId, "Contains", asset.getId(), device.getId());
long temperature = (long) (Math.random() * 100);
temperatures.add(temperature);
if (temperature > 45) {
highTemperatures.add(temperature);
}
//asset -> device : one-to-one but many edges
for (int n = 0; n < ENTITY_COUNT; n++) {
createRelation(tenantId, "UseCase-" + n, asset.getId(), device.getId());
}
}
}
createManyCustomRelationsBetweenTwoNodes(tenantId, "UseCase", assets, devices);
//asset -> device one-to-many shared with other assets
for (int n = 0; n < devices.size(); n = n + ENTITY_COUNT) {
createRelation(tenantId, "SharedWithAsset0", assets.get(0).getId(), devices.get(n).getId());
}
//createManyCustomRelationsBetweenTwoNodes(tenantId, "UseCase", assets, devices);
createHorizontalRingRelations(tenantId, "Ring(Loop)-Ast", assets);
createLoopRelations(tenantId, "Loop-Tnt-Ast-Dev", tenantId, assets.get(0).getId(), devices.get(0).getId());
createLoopRelations(tenantId, "Loop-Tnt-Ast", tenantId, assets.get(1).getId());
createLoopRelations(tenantId, "Loop-Ast-Tnt-Ast", assets.get(2).getId(), tenantId, assets.get(3).getId());
printAllRelations();
//testQuery();
//printAllRelations();
}
private void testQuery() {
template.query("" +
" DROP TABLE IF EXISTS relation_test;\n" +
" CREATE TABLE IF NOT EXISTS relation_test\n" +
" (\n" +
" from_id uuid,\n" +
" from_type varchar(255),\n" +
" to_id uuid,\n" +
" to_type varchar(255),\n" +
" relation_type_group varchar(255),\n" +
" relation_type varchar(255),\n" +
" additional_info varchar,\n" +
" CONSTRAINT relation_test_pkey PRIMARY KEY (from_id, from_type, relation_type_group, relation_type, to_id, to_type)\n" +
" );\n", r -> null);
log.error("insert test {}", (Object) template.query("" +
"INSERT INTO relation_test (from_id, from_type, to_id, to_type, relation_type_group, relation_type, additional_info) " +
" VALUES " +
" ('11111111-0f19-11ec-ba23-e981fc95500d', 'TENANT', '22222222-0f19-11ec-ba23-e981fc95500d', 'ASSET', 'COMMON', 'Contains', null),\n" +
" ('22222222-0f19-11ec-ba23-e981fc95500d', 'ASSET', '33333333-0f19-11ec-ba23-e981fc95500d', 'DEVICE', 'COMMON', 'Contains', null),\n" +
" ('33333333-0f19-11ec-ba23-e981fc95500d', 'DEVICE', '11111111-0f19-11ec-ba23-e981fc95500d', 'TENANT', 'COMMON', 'Contains', null);\n" +
""
, r -> null));
//log.error("array_position hsql {}", template.queryForObject("select array_position(ARRAY['sun','mon','tue','wed','thu','fri','sat'], 'mon') from relation_test", Long.class));
log.error("array_position hsql {}", template.queryForObject("select position_array('mon' in ARRAY['sun','mon','tue']);", Long.class));
log.error("array_position psql {}", template.queryForObject("select array_position(ARRAY['sun','mon','tue'], 'mon');", Long.class));
Long countTest = template.queryForObject("select count(*) from relation_test", Long.class);
log.error("count test {}", countTest);
Long count = template.queryForObject("select count(*) from relation", Long.class);
log.error("count {}", count);
List<List<String>> result = template.query("" +
"WITH RECURSIVE related_entities(from_id, from_type, to_id, to_type, lvl, path) " +
" AS (SELECT from_id, " +
" from_type, " +
" to_id, " +
" to_type, " +
" 1 as lvl, " +
" ARRAY [from_id] as path " +
" FROM relation_test r " +
" WHERE from_id = '11111111-0f19-11ec-ba23-e981fc95500d' " +
" and from_type = 'TENANT' " +
" and relation_type_group = 'COMMON' " +
" GROUP BY r.from_id, r.from_type, r.to_id, r.to_type , 1, ARRAY [from_id] " +
" UNION ALL " +
" SELECT r.from_id, " +
" r.from_type, " +
" r.to_id, " +
" r.to_type, " +
" (re.lvl + 1) as lvl, " +
" (re.path || ARRAY [r.from_id]) as path " +
" FROM relation_test r " +
" INNER JOIN related_entities re " +
" ON r.from_id = re.to_id and " +
" r.from_type = re.to_type and " +
" relation_type_group = 'COMMON' and " +
" r.from_id NOT IN (SELECT * FROM unnest(re.path)) and " +
" re.lvl <= 7 " +
" GROUP BY r.from_id, r.from_type, r.to_id, r.to_type, " +
" (re.lvl + 1), (re.path || ARRAY [r.from_id])) " +
" " +
" SELECT lvl, from_id, from_type, to_id, to_type, path " + //to_id IN (SELECT * FROM unnest(path)) as is_present,
" from related_entities r_int " +
" " +
" ",
getListResultSetExtractor());
log.error("result {}", result.size() - 1);
AtomicInteger counter = new AtomicInteger();
result.forEach(r -> System.out.printf("%s %s\n", counter.incrementAndGet(), r.toString()));
log.error("end");
log.error("query 1 (expected true): {}", template.queryForObject(
"SELECT UUID('463e5c80-0f38-11ec-8153-55a9f38b54f3') NOT IN (SELECT * FROM unnest(ARRAY[UUID('463e5c80-0f38-11ec-8153-55a9f38b54f3'), UUID('46957d30-0f38-11ec-8153-55a9f38b54f3')])) ",
String.class));
log.error("query 1.1 (expected true): {}", template.queryForObject(
"SELECT UUID('463e5c80-0f38-11ec-8153-55a9f38b54f3') NOT IN (SELECT * FROM unnest(ARRAY[UUID('463e5c80-0f38-11ec-8153-55a9f38b54f3')] || ARRAY[UUID('46957d30-0f38-11ec-8153-55a9f38b54f3')] )) ",
String.class));
log.error("query 2 (expected true): {}", template.queryForObject(
"SELECT UUID('463e5c80-0f38-11ec-8153-55a9f38b54f3') NOT IN (SELECT UUID('463e5c80-0f38-11ec-8153-55a9f38b54f3')) ",
String.class));
// log.error("query true3: ", template.queryForObject(
// "SELECT '463e5c80-0f38-11ec-8153-55a9f38b54f3' IN (SELECT UUID('463e5c80-0f38-11ec-8153-55a9f38b54f3')) ",
// String.class));
log.error("query 3 (expected true): {} ", template.queryForObject(
"SELECT '463e5c80-0f38-11ec-8153-55a9f38b54f3' NOT IN (SELECT '463e5c80-0f38-11ec-8153-55a9f38b54f3') ",
String.class));
List<List<String>> result2 = template.query("SELECT * FROM unnest(ARRAY[UUID('463e5c80-0f38-11ec-8153-55a9f38b54f3'), UUID('46957d30-0f38-11ec-8153-55a9f38b54f3')]) "
, getListResultSetExtractor());
log.error("result2 {}", result2.size() - 1);
AtomicInteger counter2 = new AtomicInteger();
result2.forEach(r -> System.out.printf("%s %s\n", counter2.incrementAndGet(), r.toString()));
log.error("end2");
}
@NotNull
private ResultSetExtractor<List<List<String>>> getListResultSetExtractor() {
return rs -> {
List<List<String>> list = new ArrayList<>();
final int columnCount = rs.getMetaData().getColumnCount();
List<String> columns = new ArrayList<>(columnCount);
for (int i = 1; i <= columnCount; i++) {
columns.add(rs.getMetaData().getColumnName(i));
}
list.add(columns);
while (rs.next()) {
List<String> data = new ArrayList<>(columnCount);
for (int i = 1; i <= columnCount; i++) {
data.add(rs.getString(i));
}
list.add(data);
}
return list;
};
}
/*
@ -642,9 +780,14 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest {
}
void createOneToManyRelations(TenantId tenantId, String type, EntityId from, List<EntityId> toIds) {
toIds.forEach(toId -> relationService.saveRelation(tenantId, new EntityRelation(from, toId, type, RelationTypeGroup.COMMON)));
toIds.forEach(toId -> createRelation(tenantId, type, from, toId));
}
void createRelation(TenantId tenantId, String type, EntityId from, EntityId toId) {
relationService.saveRelation(tenantId, new EntityRelation(from, toId, type, RelationTypeGroup.COMMON));
}
@Test
public void testSimpleFindEntityDataByQuery() throws InterruptedException {
List<Device> devices = new ArrayList<>();

48
dao/src/test/java/org/thingsboard/server/dao/service/DaoPostgreSqlTest.java Normal file
View File

@ -0,0 +1,48 @@
/**
* ThingsBoard, Inc. ("COMPANY") CONFIDENTIAL
*
* Copyright © 2016-2021 ThingsBoard, Inc. All Rights Reserved.
*
* NOTICE: All information contained herein is, and remains
* the property of ThingsBoard, Inc. and its suppliers,
* if any. The intellectual and technical concepts contained
* herein are proprietary to ThingsBoard, Inc.
* and its suppliers and may be covered by U.S. and Foreign Patents,
* patents in process, and are protected by trade secret or copyright law.
*
* Dissemination of this information or reproduction of this material is strictly forbidden
* unless prior written permission is obtained from COMPANY.
*
* Access to the source code contained herein is hereby forbidden to anyone except current COMPANY employees,
* managers or contractors who have executed Confidentiality and Non-disclosure agreements
* explicitly covering such access.
*
* The copyright notice above does not evidence any actual or intended publication
* or disclosure of this source code, which includes
* information that is confidential and/or proprietary, and is a trade secret, of COMPANY.
* ANY REPRODUCTION, MODIFICATION, DISTRIBUTION, PUBLIC PERFORMANCE,
* OR PUBLIC DISPLAY OF OR THROUGH USE OF THIS SOURCE CODE WITHOUT
* THE EXPRESS WRITTEN CONSENT OF COMPANY IS STRICTLY PROHIBITED,
* AND IN VIOLATION OF APPLICABLE LAWS AND INTERNATIONAL TREATIES.
* THE RECEIPT OR POSSESSION OF THIS SOURCE CODE AND/OR RELATED INFORMATION
* DOES NOT CONVEY OR IMPLY ANY RIGHTS TO REPRODUCE, DISCLOSE OR DISTRIBUTE ITS CONTENTS,
* OR TO MANUFACTURE, USE, OR SELL ANYTHING THAT IT MAY DESCRIBE, IN WHOLE OR IN PART.
*/
package org.thingsboard.server.dao.service;
import org.springframework.test.context.TestPropertySource;
import java.lang.annotation.Documented;
import java.lang.annotation.ElementType;
import java.lang.annotation.Inherited;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
@Target(ElementType.TYPE)
@Retention(RetentionPolicy.RUNTIME)
@Inherited
@Documented
@TestPropertySource(locations = {"classpath:application-test.properties", "classpath:psql-test.properties"})
public @interface DaoPostgreSqlTest {
}

8
dao/src/test/java/org/thingsboard/server/dao/service/sql/EntityServiceSqlTest.java → dao/src/test/java/org/thingsboard/server/dao/service/psql/EntityServicePostgreSqlTest.java
View File

@ -13,11 +13,11 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.dao.service.sql;
package org.thingsboard.server.dao.service.psql;
import org.thingsboard.server.dao.service.BaseEntityServiceTest;
import org.thingsboard.server.dao.service.DaoSqlTest;
import org.thingsboard.server.dao.service.DaoPostgreSqlTest;
@DaoSqlTest
public class EntityServiceSqlTest extends BaseEntityServiceTest {
@DaoPostgreSqlTest
public class EntityServicePostgreSqlTest extends BaseEntityServiceTest {
}

47
dao/src/test/resources/psql-test.properties Normal file
View File

@ -0,0 +1,47 @@
database.ts.type=sql
database.ts_latest.type=sql
sql.ts_inserts_executor_type=fixed
sql.ts_inserts_fixed_thread_pool_size=200
sql.ts_key_value_partitioning=MONTHS
#
spring.jpa.properties.hibernate.jdbc.lob.non_contextual_creation=true
spring.jpa.properties.hibernate.order_by.default_null_ordering=last
spring.jpa.properties.hibernate.jdbc.log.warnings=false
spring.jpa.show-sql=false
spring.jpa.hibernate.ddl-auto=none
spring.jpa.database-platform=org.hibernate.dialect.PostgreSQLDialect
spring.datasource.username=postgres
spring.datasource.password=postgres
spring.datasource.url=jdbc:tc:postgresql:12.8:///thingsboard?TC_DAEMON=true&TC_TMPFS=/testtmpfs:rw
spring.datasource.driverClassName=org.testcontainers.jdbc.ContainerDatabaseDriver
#org.postgresql.Driver
spring.datasource.hikari.maximumPoolSize=20
service.type=monolith
#database.ts.type=timescale
#database.ts.type=sql
#database.entities.type=sql
#
#sql.ts_inserts_executor_type=fixed
#sql.ts_inserts_fixed_thread_pool_size=200
#sql.ts_key_value_partitioning=MONTHS
#
#spring.jpa.properties.hibernate.jdbc.lob.non_contextual_creation=true
#spring.jpa.show-sql=false
#spring.jpa.hibernate.ddl-auto=none
#spring.jpa.database-platform=org.hibernate.dialect.PostgreSQLDialect
#
#spring.datasource.username=postgres
#spring.datasource.password=postgres
#spring.datasource.url=jdbc:postgresql://localhost:5432/sqltest
#spring.datasource.driverClassName=org.postgresql.Driver
#spring.datasource.hikari.maximumPoolSize = 50
queue.core.pack-processing-timeout=3000
queue.rule-engine.pack-processing-timeout=3000
queue.rule-engine.queues[0].name=Main
queue.rule-engine.queues[0].topic=tb_rule_engine.main
queue.rule-engine.queues[0].poll-interval=25
queue.rule-engine.queues[0].partitions=3
queue.rule-engine.queues[0].pack-processing-timeout=3000
queue.rule-engine.queues[0].processing-strategy.type=SKIP_ALL_FAILURES
queue.rule-engine.queues[0].submit-strategy.type=BURST
sql.log_entity_queries=true

2
dao/src/test/resources/sql/system-test-psql.sql Normal file
View File

@ -0,0 +1,2 @@
--PostgreSQL specific truncate to fit constraints
TRUNCATE TABLE device_credentials, device, device_profile, rule_node_state, rule_node, rule_chain;

11
pom.xml
View File

@ -1637,6 +1637,17 @@
<groupId>org.hsqldb</groupId>
<artifactId>hsqldb</artifactId>
<version>${hsqldb.version}</version>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>postgresql</artifactId>
<version>${testcontainers.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>jdbc</artifactId>
<version>${testcontainers.version}</version>
<scope>test</scope>
</dependency>
<dependency>