From 5809b0d6a82bafb25b559ebafc41d7098175607e Mon Sep 17 00:00:00 2001 From: Yuriy Date: Tue, 5 Apr 2022 17:52:41 +0300 Subject: [PATCH 1/8] QUERY: add SQL string with parameters --- .../server/dao/sql/query/DefaultQueryLogComponent.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponent.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponent.java index 0469b45442..17c60c8be9 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponent.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponent.java @@ -17,6 +17,7 @@ package org.thingsboard.server.dao.sql.query; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; +import org.springframework.jdbc.core.namedparam.NamedParameterUtils; import org.springframework.stereotype.Component; import java.util.Arrays; @@ -33,7 +34,12 @@ public class DefaultQueryLogComponent implements QueryLogComponent { @Override public void logQuery(QueryContext ctx, String query, long duration) { if (logSqlQueries && duration > logQueriesThreshold) { + + String sqlToUse = NamedParameterUtils.substituteNamedParameters(query, ctx); + log.info("QUERY: {} took {} ms", query, duration); + log.info("QUERY SQL TO USE: {} took {} ms", sqlToUse, duration); + Arrays.asList(ctx.getParameterNames()).forEach(param -> log.info("QUERY PARAM: {} -> {}", param, ctx.getValue(param))); } } From 97b9afac9a7bddd17151ad9b729cae6ffc335a3b Mon Sep 17 00:00:00 2001 From: Yuriy Lytvynchuk Date: Fri, 8 Apr 2022 23:01:51 +0300 Subject: [PATCH 2/8] add SQL string with parameters --- .../sql/query/DefaultQueryLogComponent.java | 74 +++++++++- .../query/DefaultQueryLogComponentTest.java | 128 ++++++++++++++++++ 2 files changed, 201 insertions(+), 1 deletion(-) create mode 100644 dao/src/test/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponentTest.java diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponent.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponent.java index 17c60c8be9..4a8d9e4d4d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponent.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponent.java @@ -17,10 +17,16 @@ package org.thingsboard.server.dao.sql.query; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; +import org.springframework.jdbc.core.SqlParameter; +import org.springframework.jdbc.core.SqlParameterValue; import org.springframework.jdbc.core.namedparam.NamedParameterUtils; +import org.springframework.jdbc.core.namedparam.ParsedSql; +import org.springframework.jdbc.core.namedparam.SqlParameterSource; import org.springframework.stereotype.Component; import java.util.Arrays; +import java.util.List; +import java.util.UUID; @Component @Slf4j @@ -35,7 +41,7 @@ public class DefaultQueryLogComponent implements QueryLogComponent { public void logQuery(QueryContext ctx, String query, long duration) { if (logSqlQueries && duration > logQueriesThreshold) { - String sqlToUse = NamedParameterUtils.substituteNamedParameters(query, ctx); + String sqlToUse = substituteParametersInSqlString(query, ctx); log.info("QUERY: {} took {} ms", query, duration); log.info("QUERY SQL TO USE: {} took {} ms", sqlToUse, duration); @@ -43,4 +49,70 @@ public class DefaultQueryLogComponent implements QueryLogComponent { Arrays.asList(ctx.getParameterNames()).forEach(param -> log.info("QUERY PARAM: {} -> {}", param, ctx.getValue(param))); } } + + String substituteParametersInSqlString(String sql, SqlParameterSource paramSource) { + + ParsedSql parsedSql = NamedParameterUtils.parseSqlStatement(sql); + List declaredParams = NamedParameterUtils.buildSqlParameterList(parsedSql, paramSource); + StringBuilder actualSql = new StringBuilder(parsedSql.toString()); + + if (declaredParams.isEmpty()) { + return sql; + } + + for (SqlParameter parSQL: declaredParams) { + String paramName = parSQL.getName(); + if (!paramSource.hasValue(paramName)) { + continue; + } + + Object value = paramSource.getValue(paramName); + if (value instanceof SqlParameterValue) { + value = ((SqlParameterValue)value).getValue(); + } + + if (!(value instanceof Iterable)) { + + String ValueForSQLQuery = getValueForSQLQuery(value); + String sqlTemp = sql.replace(":" + paramName, ValueForSQLQuery); + sql = sqlTemp; + continue; + } + + //Iterable + int count = 0; + String valueArrayStr = ""; + + for (Object valueTemp: (Iterable)value) { + + if (count > 0) { + valueArrayStr+=", "; + } + + String ValueForSQLQuery = getValueForSQLQuery(valueTemp); + valueArrayStr += ValueForSQLQuery; + ++count; + } + + if (!valueArrayStr.isEmpty()){ + String sqlTemp = sql.replace(":" + paramName, valueArrayStr); + sql = sqlTemp; + } + } + + return sql; + } + + String getValueForSQLQuery(Object valueParameter) { + + if (valueParameter instanceof String) { + return "'" + valueParameter + "'"; + } + + if (valueParameter instanceof UUID) { + return "'" + valueParameter.toString() + "'"; + } + + return valueParameter.toString(); + } } diff --git a/dao/src/test/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponentTest.java b/dao/src/test/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponentTest.java new file mode 100644 index 0000000000..db65c8d360 --- /dev/null +++ b/dao/src/test/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponentTest.java @@ -0,0 +1,128 @@ + +package org.thingsboard.server.dao.sql.query; + +import com.datastax.oss.driver.api.core.uuid.Uuids; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.junit4.SpringRunner; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.TenantId; + +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; + +import static org.junit.Assert.assertEquals; + +@RunWith(SpringRunner.class) +@SpringBootTest(classes = DefaultQueryLogComponent.class) +class DefaultQueryLogComponentTest { + + private TenantId tenantId; + private QueryContext ctx; + + @Autowired + private DefaultQueryLogComponent queryLog; + + @BeforeEach + void setUp() { + tenantId = new TenantId(Uuids.timeBased()); + ctx = new QueryContext(new QuerySecurityContext(tenantId, null, EntityType.ALARM)); + } + + @Test + void substituteParametersInSqlString_StringType() { + + String Name = "Mery"; + String id = "ID_1"; + String sql = "Select * from Table Where name = :name AND id = :id"; + String sqlToUse = "Select * from Table Where name = 'Mery' AND id = 'ID_1'"; + + ctx.addStringParameter("name", Name); + ctx.addStringParameter("id", id); + + String sqlToUse2 = queryLog.substituteParametersInSqlString(sql, ctx); + assertEquals(sqlToUse, sqlToUse2); + } + + @Test + void substituteParametersInSqlString_DoubleLongType() { + + double sum = 10.55; + long price = 100000; + String sql = "Select * from Table Where sum = :sum AND price = :price"; + String sqlToUse = "Select * from Table Where sum = 10.55 AND price = 100000"; + + ctx.addDoubleParameter("sum", sum); + ctx.addLongParameter("price", price); + + String sqlToUse2 = queryLog.substituteParametersInSqlString(sql, ctx); + assertEquals(sqlToUse, sqlToUse2); + } + + @Test + void substituteParametersInSqlString_BooleanType() { + + boolean check = true; + String sql = "Select * from Table Where check = :check AND mark = :mark"; + String sqlToUse = "Select * from Table Where check = true AND mark = false"; + + ctx.addBooleanParameter("check", check); + ctx.addBooleanParameter("mark", false); + + String sqlToUse2 = queryLog.substituteParametersInSqlString(sql, ctx); + assertEquals(sqlToUse, sqlToUse2); + } + + @Test + void substituteParametersInSqlString_UuidType() { + + UUID guid = Uuids.timeBased(); + String sql = "Select * from Table Where guid = :guid"; + String sqlToUse = "Select * from Table Where guid = '" + guid.toString() + "'"; + + ctx.addUuidParameter("guid", guid); + + String sqlToUse2 = queryLog.substituteParametersInSqlString(sql, ctx); + assertEquals(sqlToUse, sqlToUse2); + } + + @Test + void substituteParametersInSqlString_StringListType() { + + List ids = new ArrayList<>(); + ids.add("ID_1"); + ids.add("ID_2"); + ids.add("ID_3"); + ids.add("ID_4"); + + String sql = "Select * from Table Where id IN (:ids)"; + String sqlToUse = "Select * from Table Where id IN ('ID_1', 'ID_2', 'ID_3', 'ID_4')"; + + ctx.addStringListParameter("ids", ids); + + String sqlToUse2 = queryLog.substituteParametersInSqlString(sql, ctx); + assertEquals(sqlToUse, sqlToUse2); + } + + @Test + void substituteParametersInSqlString_UuidListType() { + + List guids = new ArrayList<>(); + guids.add(UUID.fromString("634a8d03-6871-4e01-94d0-876bf3e67dff")); + guids.add(UUID.fromString("3adbb5b8-4dc6-4faf-80dc-681a7b518b5e")); + guids.add(UUID.fromString("63a50f0c-2058-4d1d-8f15-812eb7f84412")); + + String sql = "Select * from Table Where guid IN (:guids)"; + String sqlToUse = "Select * from Table Where guid IN ('634a8d03-6871-4e01-94d0-876bf3e67dff', '3adbb5b8-4dc6-4faf-80dc-681a7b518b5e', '63a50f0c-2058-4d1d-8f15-812eb7f84412')"; + + ctx.addUuidListParameter("guids", guids); + + String sqlToUse2 = queryLog.substituteParametersInSqlString(sql, ctx); + assertEquals(sqlToUse, sqlToUse2); + } +} + From ee4a3b90cc1acab95ab8b3164404a87eeade6816 Mon Sep 17 00:00:00 2001 From: Yuriy Lytvynchuk Date: Fri, 8 Apr 2022 23:01:51 +0300 Subject: [PATCH 3/8] Query: add SQL string with parameters --- .../sql/query/DefaultQueryLogComponent.java | 74 +++++++++- .../query/DefaultQueryLogComponentTest.java | 128 ++++++++++++++++++ 2 files changed, 201 insertions(+), 1 deletion(-) create mode 100644 dao/src/test/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponentTest.java diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponent.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponent.java index 17c60c8be9..4a8d9e4d4d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponent.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponent.java @@ -17,10 +17,16 @@ package org.thingsboard.server.dao.sql.query; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; +import org.springframework.jdbc.core.SqlParameter; +import org.springframework.jdbc.core.SqlParameterValue; import org.springframework.jdbc.core.namedparam.NamedParameterUtils; +import org.springframework.jdbc.core.namedparam.ParsedSql; +import org.springframework.jdbc.core.namedparam.SqlParameterSource; import org.springframework.stereotype.Component; import java.util.Arrays; +import java.util.List; +import java.util.UUID; @Component @Slf4j @@ -35,7 +41,7 @@ public class DefaultQueryLogComponent implements QueryLogComponent { public void logQuery(QueryContext ctx, String query, long duration) { if (logSqlQueries && duration > logQueriesThreshold) { - String sqlToUse = NamedParameterUtils.substituteNamedParameters(query, ctx); + String sqlToUse = substituteParametersInSqlString(query, ctx); log.info("QUERY: {} took {} ms", query, duration); log.info("QUERY SQL TO USE: {} took {} ms", sqlToUse, duration); @@ -43,4 +49,70 @@ public class DefaultQueryLogComponent implements QueryLogComponent { Arrays.asList(ctx.getParameterNames()).forEach(param -> log.info("QUERY PARAM: {} -> {}", param, ctx.getValue(param))); } } + + String substituteParametersInSqlString(String sql, SqlParameterSource paramSource) { + + ParsedSql parsedSql = NamedParameterUtils.parseSqlStatement(sql); + List declaredParams = NamedParameterUtils.buildSqlParameterList(parsedSql, paramSource); + StringBuilder actualSql = new StringBuilder(parsedSql.toString()); + + if (declaredParams.isEmpty()) { + return sql; + } + + for (SqlParameter parSQL: declaredParams) { + String paramName = parSQL.getName(); + if (!paramSource.hasValue(paramName)) { + continue; + } + + Object value = paramSource.getValue(paramName); + if (value instanceof SqlParameterValue) { + value = ((SqlParameterValue)value).getValue(); + } + + if (!(value instanceof Iterable)) { + + String ValueForSQLQuery = getValueForSQLQuery(value); + String sqlTemp = sql.replace(":" + paramName, ValueForSQLQuery); + sql = sqlTemp; + continue; + } + + //Iterable + int count = 0; + String valueArrayStr = ""; + + for (Object valueTemp: (Iterable)value) { + + if (count > 0) { + valueArrayStr+=", "; + } + + String ValueForSQLQuery = getValueForSQLQuery(valueTemp); + valueArrayStr += ValueForSQLQuery; + ++count; + } + + if (!valueArrayStr.isEmpty()){ + String sqlTemp = sql.replace(":" + paramName, valueArrayStr); + sql = sqlTemp; + } + } + + return sql; + } + + String getValueForSQLQuery(Object valueParameter) { + + if (valueParameter instanceof String) { + return "'" + valueParameter + "'"; + } + + if (valueParameter instanceof UUID) { + return "'" + valueParameter.toString() + "'"; + } + + return valueParameter.toString(); + } } diff --git a/dao/src/test/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponentTest.java b/dao/src/test/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponentTest.java new file mode 100644 index 0000000000..db65c8d360 --- /dev/null +++ b/dao/src/test/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponentTest.java @@ -0,0 +1,128 @@ + +package org.thingsboard.server.dao.sql.query; + +import com.datastax.oss.driver.api.core.uuid.Uuids; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.junit4.SpringRunner; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.TenantId; + +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; + +import static org.junit.Assert.assertEquals; + +@RunWith(SpringRunner.class) +@SpringBootTest(classes = DefaultQueryLogComponent.class) +class DefaultQueryLogComponentTest { + + private TenantId tenantId; + private QueryContext ctx; + + @Autowired + private DefaultQueryLogComponent queryLog; + + @BeforeEach + void setUp() { + tenantId = new TenantId(Uuids.timeBased()); + ctx = new QueryContext(new QuerySecurityContext(tenantId, null, EntityType.ALARM)); + } + + @Test + void substituteParametersInSqlString_StringType() { + + String Name = "Mery"; + String id = "ID_1"; + String sql = "Select * from Table Where name = :name AND id = :id"; + String sqlToUse = "Select * from Table Where name = 'Mery' AND id = 'ID_1'"; + + ctx.addStringParameter("name", Name); + ctx.addStringParameter("id", id); + + String sqlToUse2 = queryLog.substituteParametersInSqlString(sql, ctx); + assertEquals(sqlToUse, sqlToUse2); + } + + @Test + void substituteParametersInSqlString_DoubleLongType() { + + double sum = 10.55; + long price = 100000; + String sql = "Select * from Table Where sum = :sum AND price = :price"; + String sqlToUse = "Select * from Table Where sum = 10.55 AND price = 100000"; + + ctx.addDoubleParameter("sum", sum); + ctx.addLongParameter("price", price); + + String sqlToUse2 = queryLog.substituteParametersInSqlString(sql, ctx); + assertEquals(sqlToUse, sqlToUse2); + } + + @Test + void substituteParametersInSqlString_BooleanType() { + + boolean check = true; + String sql = "Select * from Table Where check = :check AND mark = :mark"; + String sqlToUse = "Select * from Table Where check = true AND mark = false"; + + ctx.addBooleanParameter("check", check); + ctx.addBooleanParameter("mark", false); + + String sqlToUse2 = queryLog.substituteParametersInSqlString(sql, ctx); + assertEquals(sqlToUse, sqlToUse2); + } + + @Test + void substituteParametersInSqlString_UuidType() { + + UUID guid = Uuids.timeBased(); + String sql = "Select * from Table Where guid = :guid"; + String sqlToUse = "Select * from Table Where guid = '" + guid.toString() + "'"; + + ctx.addUuidParameter("guid", guid); + + String sqlToUse2 = queryLog.substituteParametersInSqlString(sql, ctx); + assertEquals(sqlToUse, sqlToUse2); + } + + @Test + void substituteParametersInSqlString_StringListType() { + + List ids = new ArrayList<>(); + ids.add("ID_1"); + ids.add("ID_2"); + ids.add("ID_3"); + ids.add("ID_4"); + + String sql = "Select * from Table Where id IN (:ids)"; + String sqlToUse = "Select * from Table Where id IN ('ID_1', 'ID_2', 'ID_3', 'ID_4')"; + + ctx.addStringListParameter("ids", ids); + + String sqlToUse2 = queryLog.substituteParametersInSqlString(sql, ctx); + assertEquals(sqlToUse, sqlToUse2); + } + + @Test + void substituteParametersInSqlString_UuidListType() { + + List guids = new ArrayList<>(); + guids.add(UUID.fromString("634a8d03-6871-4e01-94d0-876bf3e67dff")); + guids.add(UUID.fromString("3adbb5b8-4dc6-4faf-80dc-681a7b518b5e")); + guids.add(UUID.fromString("63a50f0c-2058-4d1d-8f15-812eb7f84412")); + + String sql = "Select * from Table Where guid IN (:guids)"; + String sqlToUse = "Select * from Table Where guid IN ('634a8d03-6871-4e01-94d0-876bf3e67dff', '3adbb5b8-4dc6-4faf-80dc-681a7b518b5e', '63a50f0c-2058-4d1d-8f15-812eb7f84412')"; + + ctx.addUuidListParameter("guids", guids); + + String sqlToUse2 = queryLog.substituteParametersInSqlString(sql, ctx); + assertEquals(sqlToUse, sqlToUse2); + } +} + From 1f029983d1f5b826b8a6e7d8128c641caf3d03ce Mon Sep 17 00:00:00 2001 From: Yuriy Lytvynchuk Date: Mon, 11 Apr 2022 19:33:59 +0300 Subject: [PATCH 4/8] Query: add SQL string with parameters --- .../sql/query/DefaultQueryLogComponent.java | 14 ++-- .../query/DefaultQueryLogComponentTest.java | 65 +++++++++---------- 2 files changed, 35 insertions(+), 44 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponent.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponent.java index 4a8d9e4d4d..7291d480f6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponent.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponent.java @@ -54,7 +54,6 @@ public class DefaultQueryLogComponent implements QueryLogComponent { ParsedSql parsedSql = NamedParameterUtils.parseSqlStatement(sql); List declaredParams = NamedParameterUtils.buildSqlParameterList(parsedSql, paramSource); - StringBuilder actualSql = new StringBuilder(parsedSql.toString()); if (declaredParams.isEmpty()) { return sql; @@ -74,8 +73,7 @@ public class DefaultQueryLogComponent implements QueryLogComponent { if (!(value instanceof Iterable)) { String ValueForSQLQuery = getValueForSQLQuery(value); - String sqlTemp = sql.replace(":" + paramName, ValueForSQLQuery); - sql = sqlTemp; + sql = sql.replace(":" + paramName, ValueForSQLQuery); continue; } @@ -94,10 +92,8 @@ public class DefaultQueryLogComponent implements QueryLogComponent { ++count; } - if (!valueArrayStr.isEmpty()){ - String sqlTemp = sql.replace(":" + paramName, valueArrayStr); - sql = sqlTemp; - } + sql = sql.replace(":" + paramName, valueArrayStr); + } return sql; @@ -106,11 +102,11 @@ public class DefaultQueryLogComponent implements QueryLogComponent { String getValueForSQLQuery(Object valueParameter) { if (valueParameter instanceof String) { - return "'" + valueParameter + "'"; + return "'" + ((String) valueParameter).replaceAll("'", "''") + "'"; } if (valueParameter instanceof UUID) { - return "'" + valueParameter.toString() + "'"; + return "'" + valueParameter + "'"; } return valueParameter.toString(); diff --git a/dao/src/test/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponentTest.java b/dao/src/test/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponentTest.java index db65c8d360..9eefa9128a 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponentTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponentTest.java @@ -2,15 +2,14 @@ package org.thingsboard.server.dao.sql.query; import com.datastax.oss.driver.api.core.uuid.Uuids; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; +import org.junit.Before; +import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.test.context.junit4.SpringRunner; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.TenantId; - import java.util.ArrayList; import java.util.List; import java.util.UUID; @@ -19,7 +18,7 @@ import static org.junit.Assert.assertEquals; @RunWith(SpringRunner.class) @SpringBootTest(classes = DefaultQueryLogComponent.class) -class DefaultQueryLogComponentTest { +public class DefaultQueryLogComponentTest { private TenantId tenantId; private QueryContext ctx; @@ -27,44 +26,44 @@ class DefaultQueryLogComponentTest { @Autowired private DefaultQueryLogComponent queryLog; - @BeforeEach - void setUp() { + @Before + public void setUp() { tenantId = new TenantId(Uuids.timeBased()); ctx = new QueryContext(new QuerySecurityContext(tenantId, null, EntityType.ALARM)); } @Test - void substituteParametersInSqlString_StringType() { + public void substituteParametersInSqlString_StringType() { - String Name = "Mery"; + String Name = "Mery's"; String id = "ID_1"; String sql = "Select * from Table Where name = :name AND id = :id"; - String sqlToUse = "Select * from Table Where name = 'Mery' AND id = 'ID_1'"; + String sqlToUse = "Select * from Table Where name = 'Mery''s' AND id = 'ID_1'"; ctx.addStringParameter("name", Name); ctx.addStringParameter("id", id); - String sqlToUse2 = queryLog.substituteParametersInSqlString(sql, ctx); - assertEquals(sqlToUse, sqlToUse2); + String sqlToUseResult = queryLog.substituteParametersInSqlString(sql, ctx); + assertEquals(sqlToUse, sqlToUseResult); } @Test - void substituteParametersInSqlString_DoubleLongType() { + public void substituteParametersInSqlString_DoubleLongType() { - double sum = 10.55; + double sum = 0.00000021d; long price = 100000; String sql = "Select * from Table Where sum = :sum AND price = :price"; - String sqlToUse = "Select * from Table Where sum = 10.55 AND price = 100000"; + String sqlToUse = "Select * from Table Where sum = 2.1E-7 AND price = 100000"; ctx.addDoubleParameter("sum", sum); ctx.addLongParameter("price", price); - String sqlToUse2 = queryLog.substituteParametersInSqlString(sql, ctx); - assertEquals(sqlToUse, sqlToUse2); + String sqlToUseResult = queryLog.substituteParametersInSqlString(sql, ctx); + assertEquals(sqlToUse, sqlToUseResult); } @Test - void substituteParametersInSqlString_BooleanType() { + public void substituteParametersInSqlString_BooleanType() { boolean check = true; String sql = "Select * from Table Where check = :check AND mark = :mark"; @@ -73,43 +72,39 @@ class DefaultQueryLogComponentTest { ctx.addBooleanParameter("check", check); ctx.addBooleanParameter("mark", false); - String sqlToUse2 = queryLog.substituteParametersInSqlString(sql, ctx); - assertEquals(sqlToUse, sqlToUse2); + String sqlToUseResult = queryLog.substituteParametersInSqlString(sql, ctx); + assertEquals(sqlToUse, sqlToUseResult); } @Test - void substituteParametersInSqlString_UuidType() { + public void substituteParametersInSqlString_UuidType() { UUID guid = Uuids.timeBased(); String sql = "Select * from Table Where guid = :guid"; - String sqlToUse = "Select * from Table Where guid = '" + guid.toString() + "'"; + String sqlToUse = "Select * from Table Where guid = '" + guid + "'"; ctx.addUuidParameter("guid", guid); - String sqlToUse2 = queryLog.substituteParametersInSqlString(sql, ctx); - assertEquals(sqlToUse, sqlToUse2); + String sqlToUseResult = queryLog.substituteParametersInSqlString(sql, ctx); + assertEquals(sqlToUse, sqlToUseResult); } @Test - void substituteParametersInSqlString_StringListType() { + public void substituteParametersInSqlString_StringListType() { - List ids = new ArrayList<>(); - ids.add("ID_1"); - ids.add("ID_2"); - ids.add("ID_3"); - ids.add("ID_4"); + List ids = List.of("ID_1'", "ID_2", "ID_3", "ID_4"); String sql = "Select * from Table Where id IN (:ids)"; - String sqlToUse = "Select * from Table Where id IN ('ID_1', 'ID_2', 'ID_3', 'ID_4')"; + String sqlToUse = "Select * from Table Where id IN ('ID_1''', 'ID_2', 'ID_3', 'ID_4')"; ctx.addStringListParameter("ids", ids); - String sqlToUse2 = queryLog.substituteParametersInSqlString(sql, ctx); - assertEquals(sqlToUse, sqlToUse2); + String sqlToUseResult = queryLog.substituteParametersInSqlString(sql, ctx); + assertEquals(sqlToUse, sqlToUseResult); } @Test - void substituteParametersInSqlString_UuidListType() { + public void substituteParametersInSqlString_UuidListType() { List guids = new ArrayList<>(); guids.add(UUID.fromString("634a8d03-6871-4e01-94d0-876bf3e67dff")); @@ -121,8 +116,8 @@ class DefaultQueryLogComponentTest { ctx.addUuidListParameter("guids", guids); - String sqlToUse2 = queryLog.substituteParametersInSqlString(sql, ctx); - assertEquals(sqlToUse, sqlToUse2); + String sqlToUseResult = queryLog.substituteParametersInSqlString(sql, ctx); + assertEquals(sqlToUse, sqlToUseResult); } } From 1a4cf21bb1735312d155b2f3f5e5fe7db2849ed9 Mon Sep 17 00:00:00 2001 From: Yuriy Lytvynchuk Date: Mon, 11 Apr 2022 20:00:07 +0300 Subject: [PATCH 5/8] Query: add SQL string with parameters --- .../server/dao/sql/query/DefaultQueryLogComponent.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponent.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponent.java index 7291d480f6..37de3396ec 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponent.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponent.java @@ -87,8 +87,8 @@ public class DefaultQueryLogComponent implements QueryLogComponent { valueArrayStr+=", "; } - String ValueForSQLQuery = getValueForSQLQuery(valueTemp); - valueArrayStr += ValueForSQLQuery; + String valueForSQLQuery = getValueForSQLQuery(valueTemp); + valueArrayStr += valueForSQLQuery; ++count; } From cecbacfad43ea669a44361c49d30c1f6a8dd08eb Mon Sep 17 00:00:00 2001 From: Yuriy Lytvynchuk Date: Tue, 12 Apr 2022 18:20:55 +0300 Subject: [PATCH 6/8] Query: add SQL string with parameters --- .../sql/query/DefaultQueryLogComponent.java | 5 +- .../query/DefaultQueryLogComponentTest.java | 70 ++++++++++++++----- 2 files changed, 52 insertions(+), 23 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponent.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponent.java index 37de3396ec..21b99d4348 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponent.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponent.java @@ -42,10 +42,7 @@ public class DefaultQueryLogComponent implements QueryLogComponent { if (logSqlQueries && duration > logQueriesThreshold) { String sqlToUse = substituteParametersInSqlString(query, ctx); - - log.info("QUERY: {} took {} ms", query, duration); - log.info("QUERY SQL TO USE: {} took {} ms", sqlToUse, duration); - + log.warn("SLOW QUERY took {} ms: {}", sqlToUse, duration); Arrays.asList(ctx.getParameterNames()).forEach(param -> log.info("QUERY PARAM: {} -> {}", param, ctx.getValue(param))); } } diff --git a/dao/src/test/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponentTest.java b/dao/src/test/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponentTest.java index 9eefa9128a..dc0640b370 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponentTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponentTest.java @@ -1,47 +1,83 @@ - +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.dao.sql.query; -import com.datastax.oss.driver.api.core.uuid.Uuids; import org.junit.Before; import org.junit.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.junit.runner.RunWith; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; +import org.mockito.BDDMockito; +import org.mockito.Mockito; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.boot.test.mock.mockito.SpyBean; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.TestPropertySource; +import org.springframework.test.context.junit.jupiter.SpringExtension; import org.springframework.test.context.junit4.SpringRunner; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.TenantId; -import java.util.ArrayList; + import java.util.List; import java.util.UUID; import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.times; @RunWith(SpringRunner.class) -@SpringBootTest(classes = DefaultQueryLogComponent.class) +@ExtendWith(SpringExtension.class) +@ContextConfiguration(classes = DefaultQueryLogComponent.class) +@EnableConfigurationProperties +@TestPropertySource(properties = { + "sql.log_queries=true", + "sql.log_queries_threshold:2999" +}) + public class DefaultQueryLogComponentTest { private TenantId tenantId; private QueryContext ctx; - @Autowired + @SpyBean private DefaultQueryLogComponent queryLog; @Before public void setUp() { - tenantId = new TenantId(Uuids.timeBased()); + tenantId = new TenantId(UUID.fromString("97275c1c-9cf2-4d25-a68d-933031158f84")); ctx = new QueryContext(new QuerySecurityContext(tenantId, null, EntityType.ALARM)); } + @Test + public void logQuery() { + + BDDMockito.willCallRealMethod().given(queryLog).logQuery(ctx, "", 3000); + BDDMockito.willReturn("").given(queryLog).substituteParametersInSqlString("", ctx); + queryLog.logQuery(ctx, "", 3000); + + Mockito.verify(queryLog, times(1)).substituteParametersInSqlString("", ctx); + + } + @Test public void substituteParametersInSqlString_StringType() { - String Name = "Mery's"; - String id = "ID_1"; String sql = "Select * from Table Where name = :name AND id = :id"; String sqlToUse = "Select * from Table Where name = 'Mery''s' AND id = 'ID_1'"; - ctx.addStringParameter("name", Name); - ctx.addStringParameter("id", id); + ctx.addStringParameter("name", "Mery's"); + ctx.addStringParameter("id", "ID_1"); String sqlToUseResult = queryLog.substituteParametersInSqlString(sql, ctx); assertEquals(sqlToUse, sqlToUseResult); @@ -65,11 +101,10 @@ public class DefaultQueryLogComponentTest { @Test public void substituteParametersInSqlString_BooleanType() { - boolean check = true; String sql = "Select * from Table Where check = :check AND mark = :mark"; String sqlToUse = "Select * from Table Where check = true AND mark = false"; - ctx.addBooleanParameter("check", check); + ctx.addBooleanParameter("check", true); ctx.addBooleanParameter("mark", false); String sqlToUseResult = queryLog.substituteParametersInSqlString(sql, ctx); @@ -79,7 +114,7 @@ public class DefaultQueryLogComponentTest { @Test public void substituteParametersInSqlString_UuidType() { - UUID guid = Uuids.timeBased(); + UUID guid = UUID.randomUUID(); String sql = "Select * from Table Where guid = :guid"; String sqlToUse = "Select * from Table Where guid = '" + guid + "'"; @@ -106,10 +141,7 @@ public class DefaultQueryLogComponentTest { @Test public void substituteParametersInSqlString_UuidListType() { - List guids = new ArrayList<>(); - guids.add(UUID.fromString("634a8d03-6871-4e01-94d0-876bf3e67dff")); - guids.add(UUID.fromString("3adbb5b8-4dc6-4faf-80dc-681a7b518b5e")); - guids.add(UUID.fromString("63a50f0c-2058-4d1d-8f15-812eb7f84412")); + List guids = List.of(UUID.fromString("634a8d03-6871-4e01-94d0-876bf3e67dff"), UUID.fromString("3adbb5b8-4dc6-4faf-80dc-681a7b518b5e"), UUID.fromString("63a50f0c-2058-4d1d-8f15-812eb7f84412")); String sql = "Select * from Table Where guid IN (:guids)"; String sqlToUse = "Select * from Table Where guid IN ('634a8d03-6871-4e01-94d0-876bf3e67dff', '3adbb5b8-4dc6-4faf-80dc-681a7b518b5e', '63a50f0c-2058-4d1d-8f15-812eb7f84412')"; From 4b8e2efc3850ba8d7c89bb91294bdceb72a03a9d Mon Sep 17 00:00:00 2001 From: Yuriy Lytvynchuk Date: Wed, 13 Apr 2022 17:19:11 +0300 Subject: [PATCH 7/8] logQuery: Log format modification --- .../server/dao/sql/query/DefaultQueryLogComponent.java | 5 ++--- .../server/dao/sql/query/DefaultQueryLogComponentTest.java | 1 - 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponent.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponent.java index 21b99d4348..d66bccebc8 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponent.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponent.java @@ -24,7 +24,6 @@ import org.springframework.jdbc.core.namedparam.ParsedSql; import org.springframework.jdbc.core.namedparam.SqlParameterSource; import org.springframework.stereotype.Component; -import java.util.Arrays; import java.util.List; import java.util.UUID; @@ -42,8 +41,8 @@ public class DefaultQueryLogComponent implements QueryLogComponent { if (logSqlQueries && duration > logQueriesThreshold) { String sqlToUse = substituteParametersInSqlString(query, ctx); - log.warn("SLOW QUERY took {} ms: {}", sqlToUse, duration); - Arrays.asList(ctx.getParameterNames()).forEach(param -> log.info("QUERY PARAM: {} -> {}", param, ctx.getValue(param))); + log.warn("SLOW QUERY took {} ms: {}", duration, sqlToUse); + } } diff --git a/dao/src/test/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponentTest.java b/dao/src/test/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponentTest.java index dc0640b370..c3ac1f2c24 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponentTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponentTest.java @@ -62,7 +62,6 @@ public class DefaultQueryLogComponentTest { @Test public void logQuery() { - BDDMockito.willCallRealMethod().given(queryLog).logQuery(ctx, "", 3000); BDDMockito.willReturn("").given(queryLog).substituteParametersInSqlString("", ctx); queryLog.logQuery(ctx, "", 3000); From e1b9f8211b4d86ef6a211efc86540a6b9c590619 Mon Sep 17 00:00:00 2001 From: Yuriy Lytvynchuk Date: Wed, 13 Apr 2022 17:55:38 +0300 Subject: [PATCH 8/8] getValueForSQLQuery: valueParameter.ToString() -> String.valueOf(valueParameter) --- .../server/dao/sql/query/DefaultQueryLogComponent.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponent.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponent.java index d66bccebc8..f879fb21c7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponent.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponent.java @@ -105,6 +105,6 @@ public class DefaultQueryLogComponent implements QueryLogComponent { return "'" + valueParameter + "'"; } - return valueParameter.toString(); + return String.valueOf(valueParameter); } }