Unifying SQL DAO

application/pom.xml

@@ -249,11 +249,6 @@
             <artifactId>spring-test-dbunit</artifactId>
             <scope>test</scope>
         </dependency>
-        <dependency>
-            <groupId>ru.yandex.qatools.embed</groupId>
-            <artifactId>postgresql-embedded</artifactId>
-            <scope>test</scope>
-        </dependency>
     </dependencies>
 
     <build>

application/src/main/resources/thingsboard.yml

@@ -106,7 +106,7 @@ coap:
   timeout: "${COAP_TIMEOUT:10000}"
 
 database:
-  type: "${DATABASE_TYPE:cassandra}" # cassandra OR postgres
+  type: "${DATABASE_TYPE:cassandra}" # cassandra OR sql
 
 # Cassandra driver configuration parameters
 cassandra:

application/src/test/java/org/thingsboard/server/system/SystemSqlTestSuite.java

@@ -18,7 +18,7 @@ package org.thingsboard.server.system;
 import org.junit.ClassRule;
 import org.junit.extensions.cpsuite.ClasspathSuite;
 import org.junit.runner.RunWith;
-import org.thingsboard.server.dao.CustomPostgresUnit;
+import org.thingsboard.server.dao.CustomSqlUnit;
 
 import java.util.Arrays;
 
@@ -30,9 +30,10 @@ import java.util.Arrays;
 public class SystemSqlTestSuite {
 
     @ClassRule
-    public static CustomPostgresUnit postgresUnit = new CustomPostgresUnit(
+    public static CustomSqlUnit sqlUnit = new CustomSqlUnit(
-            Arrays.asList("postgres/schema.sql", "postgres/system-data.sql"),
+            Arrays.asList("sql/schema.sql", "sql/system-data.sql"),
-            "postgres-embedded-test.properties");
+            "sql-test.properties",
+            "sql/drop-all-tables.sql");
 
 
 }

dao/pom.xml

@@ -170,8 +170,8 @@
             <scope>test</scope>
         </dependency>
         <dependency>
-            <groupId>ru.yandex.qatools.embed</groupId>
+            <groupId>org.hsqldb</groupId>
-            <artifactId>postgresql-embedded</artifactId>
+            <artifactId>hsqldb</artifactId>
             <scope>test</scope>
         </dependency>
     </dependencies>

dao/src/main/java/org/thingsboard/server/dao/sql/JpaAbstractDao.java

@@ -45,7 +45,7 @@ public abstract class JpaAbstractDao<E extends BaseEntity<D>, D>
     protected void setSearchText(E entity) {}
 
     @Override
-    @Transactional(propagation = REQUIRES_NEW)
+    @Transactional
     public D save(D domain) {
         E entity;
         try {
@@ -64,7 +64,6 @@ public abstract class JpaAbstractDao<E extends BaseEntity<D>, D>
     }
 
     @Override
-    @Transactional(propagation = REQUIRES_NEW)
     public D findById(UUID key) {
         log.debug("Get entity by key {}", key);
         E entity = getCrudRepository().findOne(key);
@@ -78,7 +77,7 @@ public abstract class JpaAbstractDao<E extends BaseEntity<D>, D>
     }
 
     @Override
-    @Transactional(propagation = REQUIRES_NEW)
+    @Transactional
     public boolean removeById(UUID key) {
         getCrudRepository().delete(key);
         log.debug("Remove request: {}", key);

dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDao.java

@@ -73,7 +73,6 @@ public class JpaAlarmDao extends JpaAbstractDao<AlarmEntity, Alarm> implements A
     }
 
     @Override
-    @Transactional(propagation = REQUIRES_NEW)
     public ListenableFuture<Alarm> findLatestByOriginatorAndType(TenantId tenantId, EntityId originator, String type) {
         return service.submit(() -> {
             List<AlarmEntity> latest = alarmRepository.findLatestByOriginatorAndType(

dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceRepository.java

@@ -80,7 +80,5 @@ public interface DeviceRepository extends CrudRepository<DeviceEntity, UUID> {
 
     List<DeviceEntity> findDevicesByTenantIdAndCustomerIdAndIdIn(UUID tenantId, UUID customerId, List<UUID> deviceIds);
 
-    List<DeviceEntity> findDevicesByTenantId(UUID tenantId);
-
     List<DeviceEntity> findDevicesByTenantIdAndIdIn(UUID tenantId, List<UUID> deviceIds);
 }

dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java

@@ -30,12 +30,12 @@ import org.thingsboard.server.common.data.page.TimePageLink;
 import org.thingsboard.server.common.data.relation.EntityRelation;
 import org.thingsboard.server.common.data.relation.RelationTypeGroup;
 import org.thingsboard.server.dao.DaoUtil;
-import org.thingsboard.server.dao.util.SqlDao;
 import org.thingsboard.server.dao.model.sql.RelationCompositeKey;
 import org.thingsboard.server.dao.model.sql.RelationEntity;
 import org.thingsboard.server.dao.relation.RelationDao;
 import org.thingsboard.server.dao.sql.JpaAbstractDaoListeningExecutorService;
 import org.thingsboard.server.dao.sql.JpaAbstractSearchTimeDao;
+import org.thingsboard.server.dao.util.SqlDao;
 
 import javax.persistence.criteria.CriteriaBuilder;
 import javax.persistence.criteria.CriteriaQuery;

dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java

@@ -17,11 +17,11 @@ package org.thingsboard.server.dao.sql.relation;
 
 import org.springframework.data.jpa.repository.JpaSpecificationExecutor;
 import org.springframework.data.repository.CrudRepository;
-import org.thingsboard.server.dao.util.SqlDao;
+import org.springframework.transaction.annotation.Transactional;
 import org.thingsboard.server.dao.model.sql.RelationCompositeKey;
 import org.thingsboard.server.dao.model.sql.RelationEntity;
+import org.thingsboard.server.dao.util.SqlDao;
 
-import javax.transaction.Transactional;
 import java.util.List;
 import java.util.UUID;
 
@@ -51,5 +51,5 @@ public interface RelationRepository
                                                     String fromType);
 
     @Transactional
-    List<RelationEntity> deleteByFromIdAndFromType(UUID fromId, String fromType);
+    void deleteByFromIdAndFromType(UUID fromId, String fromType);
 }

dao/src/main/java/org/thingsboard/server/dao/util/SqlDao.java

@@ -17,6 +17,6 @@ package org.thingsboard.server.dao.util;
 
 import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
 
-@ConditionalOnProperty(prefix = "database", value = "type", havingValue = "postgres")
+@ConditionalOnProperty(prefix = "database", value = "type", havingValue = "sql")
 public @interface SqlDao {
 }

dao/src/main/resources/sql/schema.sql

@@ -17,7 +17,7 @@
 
 CREATE TABLE IF NOT EXISTS admin_settings (
     id uuid NOT NULL CONSTRAINT admin_settings_pkey PRIMARY KEY,
-    json_value text,
+    json_value varchar,
     key character varying(255)
 );
 
@@ -25,7 +25,7 @@ CREATE TABLE IF NOT EXISTS alarm (
     id uuid NOT NULL CONSTRAINT alarm_pkey PRIMARY KEY,
     ack_ts bigint,
     clear_ts bigint,
-    additional_info text,
+    additional_info varchar,
     end_ts bigint,
     originator_id uuid,
     originator_type integer,
@@ -39,7 +39,7 @@ CREATE TABLE IF NOT EXISTS alarm (
 
 CREATE TABLE IF NOT EXISTS asset (
     id uuid NOT NULL CONSTRAINT asset_pkey PRIMARY KEY,
-    additional_info text,
+    additional_info varchar,
     customer_id uuid,
     name character varying(255),
     search_text character varying(255),
@@ -64,7 +64,7 @@ CREATE TABLE IF NOT EXISTS component_descriptor (
     id uuid NOT NULL CONSTRAINT component_descriptor_pkey PRIMARY KEY,
     actions character varying(255),
     clazz character varying(255),
-    configuration_descriptor text,
+    configuration_descriptor varchar,
     name character varying(255),
     scope character varying(255),
     search_text character varying(255),
@@ -73,7 +73,7 @@ CREATE TABLE IF NOT EXISTS component_descriptor (
 
 CREATE TABLE IF NOT EXISTS customer (
     id uuid NOT NULL CONSTRAINT customer_pkey PRIMARY KEY,
-    additional_info text,
+    additional_info varchar,
     address character varying(255),
     address2 character varying(255),
     city character varying(255),
@@ -89,7 +89,7 @@ CREATE TABLE IF NOT EXISTS customer (
 
 CREATE TABLE IF NOT EXISTS dashboard (
     id uuid NOT NULL CONSTRAINT dashboard_pkey PRIMARY KEY,
-    configuration text,
+    configuration varchar,
     customer_id uuid,
     search_text character varying(255),
     tenant_id uuid,
@@ -98,7 +98,7 @@ CREATE TABLE IF NOT EXISTS dashboard (
 
 CREATE TABLE IF NOT EXISTS device (
     id uuid NOT NULL CONSTRAINT device_pkey PRIMARY KEY,
-    additional_info text,
+    additional_info varchar,
     customer_id uuid,
     type character varying(255),
     name character varying(255),
@@ -116,7 +116,7 @@ CREATE TABLE IF NOT EXISTS device_credentials (
 
 CREATE TABLE IF NOT EXISTS event (
     id uuid NOT NULL CONSTRAINT event_pkey PRIMARY KEY,
-    body text,
+    body varchar,
     entity_id uuid,
     entity_type character varying(255),
     event_type character varying(255),
@@ -127,10 +127,10 @@ CREATE TABLE IF NOT EXISTS event (
 
 CREATE TABLE IF NOT EXISTS plugin (
     id uuid NOT NULL CONSTRAINT plugin_pkey PRIMARY KEY,
-    additional_info text,
+    additional_info varchar,
     api_token character varying(255),
     plugin_class character varying(255),
-    configuration text,
+    configuration varchar,
     name character varying(255),
     public_access boolean,
     search_text character varying(255),
@@ -145,18 +145,18 @@ CREATE TABLE IF NOT EXISTS relation (
     to_type character varying(255),
     relation_type_group character varying(255),
     relation_type character varying(255),
-    additional_info text,
+    additional_info varchar,
     CONSTRAINT relation_unq_key UNIQUE (from_id, from_type, relation_type_group, relation_type, to_id, to_type)
 );
 
 CREATE TABLE IF NOT EXISTS rule (
     id uuid NOT NULL CONSTRAINT rule_pkey PRIMARY KEY,
-    action text,
+    action varchar,
-    additional_info text,
+    additional_info varchar,
-    filters text,
+    filters varchar,
     name character varying(255),
     plugin_token character varying(255),
-    processor text,
+    processor varchar,
     search_text character varying(255),
     state character varying(255),
     tenant_id uuid,
@@ -165,7 +165,7 @@ CREATE TABLE IF NOT EXISTS rule (
 
 CREATE TABLE IF NOT EXISTS tb_user (
     id uuid NOT NULL CONSTRAINT tb_user_pkey PRIMARY KEY,
-    additional_info text,
+    additional_info varchar,
     authority character varying(255),
     customer_id uuid,
     email character varying(255) UNIQUE,
@@ -177,7 +177,7 @@ CREATE TABLE IF NOT EXISTS tb_user (
 
 CREATE TABLE IF NOT EXISTS tenant (
     id uuid NOT NULL CONSTRAINT tenant_pkey PRIMARY KEY,
-    additional_info text,
+    additional_info varchar,
     address character varying(255),
     address2 character varying(255),
     city character varying(255),
@@ -228,7 +228,7 @@ CREATE TABLE IF NOT EXISTS widget_type (
     id uuid NOT NULL CONSTRAINT widget_type_pkey PRIMARY KEY,
     alias character varying(255),
     bundle_alias character varying(255),
-    descriptor text,
+    descriptor varchar(2000000),
     name character varying(255),
     tenant_id uuid
 );
@@ -236,7 +236,6 @@ CREATE TABLE IF NOT EXISTS widget_type (
 CREATE TABLE IF NOT EXISTS widgets_bundle (
     id uuid NOT NULL CONSTRAINT widgets_bundle_pkey PRIMARY KEY,
     alias character varying(255),
-    image bytea,
     search_text character varying(255),
     tenant_id uuid,
     title character varying(255)

dao/src/test/java/org/thingsboard/server/dao/CustomSqlUnit.java

@@ -19,7 +19,6 @@ import com.google.common.base.Charsets;
 import com.google.common.io.Resources;
 import lombok.extern.slf4j.Slf4j;
 import org.junit.rules.ExternalResource;
-import ru.yandex.qatools.embed.postgresql.EmbeddedPostgres;
 
 import java.io.IOException;
 import java.io.InputStream;
@@ -30,58 +29,58 @@ import java.sql.SQLException;
 import java.util.List;
 import java.util.Properties;
 
-import static ru.yandex.qatools.embed.postgresql.distribution.Version.Main.V9_6;
 
 /**
  * Created by Valerii Sosliuk on 6/24/2017.
  */
 @Slf4j
-public class CustomPostgresUnit extends ExternalResource {
+public class CustomSqlUnit extends ExternalResource {
-
-    private static final String HOST = "host";
-    private static final String PORT = "port";
-    private static final String DATABASE = "database";
-    private static final String USERNAME = "username";
-    private static final String PASSWORD = "password";
 
     private List<String> sqlFiles;
     private Properties properties;
+    private String dropAllTablesSqlFile;
 
-    private EmbeddedPostgres postgres;
+    public CustomSqlUnit(List<String> sqlFiles, String configurationFileName, String dropAllTablesSqlFile) {
-
-    public CustomPostgresUnit(List<String> sqlFiles, String configurationFileName) {
         this.sqlFiles = sqlFiles;
         this.properties = loadProperties(configurationFileName);
+        this.dropAllTablesSqlFile = dropAllTablesSqlFile;
     }
 
     @Override
     public void before() {
-        postgres = new EmbeddedPostgres(V9_6);
-        load();
-    }
-
-    @Override
-    public void after() {
-        postgres.stop();
-    }
-
-    private void load() {
         Connection conn = null;
         try {
-            String url = postgres.start(properties.getProperty(HOST),
+            String url = properties.getProperty("spring.datasource.url");
-                                        Integer.parseInt(properties.getProperty(PORT)),
+            conn = DriverManager.getConnection(url, "sa", "");
-                                        properties.getProperty(DATABASE),
-                                        properties.getProperty(USERNAME),
-                                        properties.getProperty(PASSWORD));
-
-            conn = DriverManager.getConnection(url);
             for (String sqlFile : sqlFiles) {
                 URL sqlFileUrl = Resources.getResource(sqlFile);
                 String sql = Resources.toString(sqlFileUrl, Charsets.UTF_8);
                 conn.createStatement().execute(sql);
             }
         } catch (IOException | SQLException e) {
-            throw new RuntimeException("Unable to start embedded postgres. Reason: " + e.getMessage(), e);
+            throw new RuntimeException("Unable to start embedded hsqldb. Reason: " + e.getMessage(), e);
+        } finally {
+            if (conn != null) {
+                try {
+                    conn.close();
+                } catch (SQLException e) {
+                    log.error(e.getMessage(), e);
+                }
+            }
+        }
+    }
+
+    @Override
+    public void after() {
+        Connection conn = null;
+        try {
+            String url = properties.getProperty("spring.datasource.url");
+            conn = DriverManager.getConnection(url, "sa", "");
+            URL dropAllTableSqlFileUrl = Resources.getResource(dropAllTablesSqlFile);
+            String dropAllTablesSql = Resources.toString(dropAllTableSqlFileUrl, Charsets.UTF_8);
+            conn.createStatement().execute(dropAllTablesSql);
+        } catch (IOException | SQLException e) {
+            throw new RuntimeException("Unable to clean up embedded hsqldb. Reason: " + e.getMessage(), e);
         } finally {
             if (conn != null) {
                 try {

dao/src/test/java/org/thingsboard/server/dao/JpaDaoTestSuite.java

@@ -24,13 +24,14 @@ import java.util.Arrays;
 
 @RunWith(ClasspathSuite.class)
 @ClassnameFilters({
-        "org.thingsboard.server.dao.sql.*Test"
+        "org.thingsboard.server.dao.sql.*ABTest"
 })
 public class JpaDaoTestSuite {
 
     @ClassRule
-    public static CustomPostgresUnit postgresUnit = new CustomPostgresUnit(
+    public static CustomSqlUnit sqlUnit = new CustomSqlUnit(
-            Arrays.asList("postgres/schema.sql", "postgres/system-data.sql"),
+            Arrays.asList("sql/schema.sql", "sql/system-data.sql"),
-            "postgres-embedded-test.properties");
+            "sql-test.properties",
+            "sql/drop-all-tables.sql");
 
 }

dao/src/test/java/org/thingsboard/server/dao/JpaDbunitTestConfig.java

@@ -18,6 +18,7 @@ package org.thingsboard.server.dao;
 import com.github.springtestdbunit.bean.DatabaseConfigBean;
 import com.github.springtestdbunit.bean.DatabaseDataSourceConnectionFactoryBean;
 import org.dbunit.DatabaseUnitException;
+import org.dbunit.ext.hsqldb.HsqldbDataTypeFactory;
 import org.dbunit.ext.postgresql.PostgresqlDataTypeFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.context.annotation.Bean;
@@ -41,7 +42,7 @@ public class JpaDbunitTestConfig {
     @Bean
     public DatabaseConfigBean databaseConfigBean() {
         DatabaseConfigBean databaseConfigBean = new DatabaseConfigBean();
-        databaseConfigBean.setDatatypeFactory(new PostgresqlDataTypeFactory());
+        databaseConfigBean.setDatatypeFactory(new HsqldbDataTypeFactory());
         return databaseConfigBean;
     }
 

dao/src/test/java/org/thingsboard/server/dao/SqlDaoServiceTestSuite.java

@@ -29,8 +29,9 @@ import java.util.Arrays;
 public class SqlDaoServiceTestSuite {
 
     @ClassRule
-    public static CustomPostgresUnit postgresUnit = new CustomPostgresUnit(
+    public static CustomSqlUnit sqlUnit = new CustomSqlUnit(
-            Arrays.asList("postgres/schema.sql", "postgres/system-data.sql", "postgres/system-test.sql"),
+            Arrays.asList("sql/schema.sql", "sql/system-data.sql", "sql/system-test.sql"),
-            "postgres-embedded-test.properties");
+            "sql-test.properties",
+            "sql/drop-all-tables.sql");
 
 }

dao/src/test/resources/postgres-embedded-test.properties

@@ -1,5 +0,0 @@
-host: localhost
-port: 5433
-database: thingsboard-test
-username: postgres
-password: postgres

dao/src/test/resources/sql-test.properties

@@ -1,8 +1,8 @@
-database.type=postgres
+database.type=sql
 
 spring.jpa.show-sql=false
 spring.jpa.hibernate.ddl-auto=validate
 
-spring.datasource.url=jdbc:postgresql://localhost:5433/thingsboard-test
+spring.datasource.username=sa
-spring.datasource.username=postgres
+spring.datasource.password=
-spring.datasource.password=postgres
+spring.datasource.url=jdbc:hsqldb:mem:thingsboardTestDb;sql.enforce_size=false

dao/src/test/resources/sql/drop-all-tables.sql

@@ -0,0 +1,20 @@
+TRUNCATE TABLE admin_settings;
+TRUNCATE TABLE alarm;
+TRUNCATE TABLE asset;
+TRUNCATE TABLE attribute_kv;
+TRUNCATE TABLE component_descriptor;
+TRUNCATE TABLE customer;
+TRUNCATE TABLE dashboard;
+TRUNCATE TABLE device;
+TRUNCATE TABLE device_credentials;
+TRUNCATE TABLE event;
+TRUNCATE TABLE plugin;
+TRUNCATE TABLE relation;
+TRUNCATE TABLE rule;
+TRUNCATE TABLE tb_user;
+TRUNCATE TABLE tenant;
+TRUNCATE TABLE ts_kv;
+TRUNCATE TABLE ts_kv_latest;
+TRUNCATE TABLE user_credentials;
+TRUNCATE TABLE widget_type;
+TRUNCATE TABLE widgets_bundle;

pom.xml

@@ -71,6 +71,7 @@
         <springfox-swagger.version>2.6.1</springfox-swagger.version>
         <bouncycastle.version>1.56</bouncycastle.version>
         <winsw.version>2.0.1</winsw.version>
+        <hsqldb.version>2.4.0</hsqldb.version>
         <dbunit.version>2.5.3</dbunit.version>
         <spring-test-dbunit.version>1.2.1</spring-test-dbunit.version>
         <postgresql.driver.version>9.4.1211</postgresql.driver.version>
@@ -746,6 +747,12 @@
                 <artifactId>bcpkix-jdk15on</artifactId>
                 <version>${bouncycastle.version}</version>
             </dependency>
+            <dependency>
+                <groupId>org.hsqldb</groupId>
+                <artifactId>hsqldb</artifactId>
+                <version>${hsqldb.version}</version>
+                <scope>test</scope>
+            </dependency>
             <dependency>
                 <groupId>ru.yandex.qatools.embed</groupId>
                 <artifactId>postgresql-embedded</artifactId>