diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index d51a2f228f..be1b5984b2 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -255,11 +255,13 @@ sql: batch_max_delay: "${SQL_ATTRIBUTES_BATCH_MAX_DELAY_MS:100}" stats_print_interval_ms: "${SQL_ATTRIBUTES_BATCH_STATS_PRINT_MS:10000}" batch_threads: "${SQL_ATTRIBUTES_BATCH_THREADS:3}" # batch thread count have to be a prime number like 3 or 5 to gain perfect hash distribution + noxss_validation_enabled: "${SQL_ATTRIBUTES_NOXSS_VALIDATION_ENABLED:true}" ts: batch_size: "${SQL_TS_BATCH_SIZE:10000}" batch_max_delay: "${SQL_TS_BATCH_MAX_DELAY_MS:100}" stats_print_interval_ms: "${SQL_TS_BATCH_STATS_PRINT_MS:10000}" batch_threads: "${SQL_TS_BATCH_THREADS:3}" # batch thread count have to be a prime number like 3 or 5 to gain perfect hash distribution + noxss_validation_enabled: "${SQL_TS_NOXSS_VALIDATION_ENABLED:true}" ts_latest: batch_size: "${SQL_TS_LATEST_BATCH_SIZE:10000}" batch_max_delay: "${SQL_TS_LATEST_BATCH_MAX_DELAY_MS:100}" diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseTelemetryControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseTelemetryControllerTest.java index bd67e5b44d..af0d486869 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseTelemetryControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseTelemetryControllerTest.java @@ -29,12 +29,15 @@ public abstract class BaseTelemetryControllerTest extends AbstractControllerTest public void testConstraintValidator() throws Exception { loginTenantAdmin(); Device device = createDevice(); - String requestBody = "{\"data\": \"alert(document)\\\">\"}"; - doPostAsync("/api/plugins/telemetry/" + device.getId() + "/SHARED_SCOPE", requestBody, String.class, status().isOk()); - doPostAsync("/api/plugins/telemetry/DEVICE/" + device.getId() + "/timeseries/smth", requestBody, String.class, status().isOk()); - requestBody = "{\"alert(document)\\\">\": \"data\"}"; - doPostAsync("/api/plugins/telemetry/" + device.getId() + "/SHARED_SCOPE", requestBody, String.class, status().isBadRequest()); - doPostAsync("/api/plugins/telemetry/DEVICE/" + device.getId() + "/timeseries/smth", requestBody, String.class, status().isBadRequest()); + String correctRequestBody = "{\"data\": \"value\"}"; + doPostAsync("/api/plugins/telemetry/" + device.getId() + "/SHARED_SCOPE", correctRequestBody, String.class, status().isOk()); + doPostAsync("/api/plugins/telemetry/DEVICE/" + device.getId() + "/timeseries/smth", correctRequestBody, String.class, status().isOk()); + String invalidRequestBody = "{\"data\": \"alert(document)\\\">\"}"; + doPostAsync("/api/plugins/telemetry/" + device.getId() + "/SHARED_SCOPE", invalidRequestBody, String.class, status().isBadRequest()); + doPostAsync("/api/plugins/telemetry/DEVICE/" + device.getId() + "/timeseries/smth", invalidRequestBody, String.class, status().isBadRequest()); + invalidRequestBody = "{\"alert(document)\\\">\": \"data\"}"; + doPostAsync("/api/plugins/telemetry/" + device.getId() + "/SHARED_SCOPE", invalidRequestBody, String.class, status().isBadRequest()); + doPostAsync("/api/plugins/telemetry/DEVICE/" + device.getId() + "/timeseries/smth", invalidRequestBody, String.class, status().isBadRequest()); } private Device createDevice() throws Exception { @@ -50,7 +53,6 @@ public abstract class BaseTelemetryControllerTest extends AbstractControllerTest SaveDeviceWithCredentialsRequest saveRequest = new SaveDeviceWithCredentialsRequest(device, deviceCredentials); - Device savedDevice = readResponse(doPost("/api/device-with-credentials", saveRequest).andExpect(status().isOk()), Device.class); - return savedDevice; + return readResponse(doPost("/api/device-with-credentials", saveRequest).andExpect(status().isOk()), Device.class); } } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/kv/JsonDataEntry.java b/common/data/src/main/java/org/thingsboard/server/common/data/kv/JsonDataEntry.java index 95712bdd70..4260c512f8 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/kv/JsonDataEntry.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/kv/JsonDataEntry.java @@ -15,10 +15,14 @@ */ package org.thingsboard.server.common.data.kv; +import org.thingsboard.server.common.data.validation.NoXss; + import java.util.Objects; import java.util.Optional; public class JsonDataEntry extends BasicKvEntry { + + @NoXss private final String value; public JsonDataEntry(String key, String value) { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/kv/StringDataEntry.java b/common/data/src/main/java/org/thingsboard/server/common/data/kv/StringDataEntry.java index 06a2b5cac6..9251cc80b3 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/kv/StringDataEntry.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/kv/StringDataEntry.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.common.data.kv; +import org.thingsboard.server.common.data.validation.NoXss; + import java.util.Objects; import java.util.Optional; @@ -22,6 +24,7 @@ public class StringDataEntry extends BasicKvEntry { private static final long serialVersionUID = 1L; + @NoXss private final String value; public StringDataEntry(String key, String value) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributeUtils.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributeUtils.java index 5bcf09cb30..980057a231 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributeUtils.java +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributeUtils.java @@ -21,6 +21,8 @@ import org.thingsboard.server.dao.exception.IncorrectParameterException; import org.thingsboard.server.dao.service.Validator; import org.thingsboard.server.dao.util.KvUtils; +import java.util.List; + public class AttributeUtils { public static void validate(EntityId id, String scope) { @@ -28,8 +30,12 @@ public class AttributeUtils { Validator.validateString(scope, "Incorrect scope " + scope); } - public static void validate(AttributeKvEntry kvEntry) { - KvUtils.validate(kvEntry); + public static void validate(List kvEntries, boolean validateNoxss) { + kvEntries.forEach(kv -> validate(kv, validateNoxss)); + } + + public static void validate(AttributeKvEntry kvEntry, boolean validateNoxss) { + KvUtils.validate(kvEntry, validateNoxss); if (kvEntry.getDataType() == null) { throw new IncorrectParameterException("Incorrect kvEntry. Data type can't be null"); } else { diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java index ef38de352a..08cb8beeae 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java @@ -18,6 +18,7 @@ package org.thingsboard.server.dao.attributes; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Primary; import org.springframework.stereotype.Service; @@ -45,6 +46,9 @@ import static org.thingsboard.server.dao.attributes.AttributeUtils.validate; public class BaseAttributesService implements AttributesService { private final AttributesDao attributesDao; + @Value("${sql.attributes.noxss_validation_enabled:true}") + private boolean noxssValidationEnabled; + public BaseAttributesService(AttributesDao attributesDao) { this.attributesDao = attributesDao; } @@ -82,14 +86,14 @@ public class BaseAttributesService implements AttributesService { @Override public ListenableFuture save(TenantId tenantId, EntityId entityId, String scope, AttributeKvEntry attribute) { validate(entityId, scope); - AttributeUtils.validate(attribute); + AttributeUtils.validate(attribute, noxssValidationEnabled); return attributesDao.save(tenantId, entityId, scope, attribute); } @Override public ListenableFuture> save(TenantId tenantId, EntityId entityId, String scope, List attributes) { validate(entityId, scope); - attributes.forEach(AttributeUtils::validate); + AttributeUtils.validate(attributes, noxssValidationEnabled); List> saveFutures = attributes.stream().map(attribute -> attributesDao.save(tenantId, entityId, scope, attribute)).collect(Collectors.toList()); return Futures.allAsList(saveFutures); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java index 01a6d1201c..ef81ed695b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java @@ -70,6 +70,9 @@ public class CachedAttributesService implements AttributesService { @Value("${cache.type:caffeine}") private String cacheType; + @Value("${sql.attributes.noxss_validation_enabled:true}") + private boolean noxssValidationEnabled; + public CachedAttributesService(AttributesDao attributesDao, StatsFactory statsFactory, CacheExecutorService cacheExecutorService, @@ -134,7 +137,7 @@ public class CachedAttributesService implements AttributesService { @Override public ListenableFuture> find(TenantId tenantId, EntityId entityId, String scope, Collection attributeKeys) { validate(entityId, scope); - attributeKeys = new LinkedHashSet<>(attributeKeys); // deduplicate the attributes + attributeKeys = new LinkedHashSet<>(attributeKeys); // deduplicate the attributes attributeKeys.forEach(attributeKey -> Validator.validateString(attributeKey, "Incorrect attribute key " + attributeKey)); Map> wrappedCachedAttributes = findCachedAttributes(entityId, scope, attributeKeys); @@ -212,7 +215,7 @@ public class CachedAttributesService implements AttributesService { @Override public ListenableFuture save(TenantId tenantId, EntityId entityId, String scope, AttributeKvEntry attribute) { validate(entityId, scope); - AttributeUtils.validate(attribute); + AttributeUtils.validate(attribute, noxssValidationEnabled); ListenableFuture future = attributesDao.save(tenantId, entityId, scope, attribute); return Futures.transform(future, key -> evict(entityId, scope, attribute, key), cacheExecutor); } @@ -220,7 +223,7 @@ public class CachedAttributesService implements AttributesService { @Override public ListenableFuture> save(TenantId tenantId, EntityId entityId, String scope, List attributes) { validate(entityId, scope); - attributes.forEach(AttributeUtils::validate); + AttributeUtils.validate(attributes, noxssValidationEnabled); List> futures = new ArrayList<>(attributes.size()); for (var attribute : attributes) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java index 748c57b5dc..1f5f21acc8 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java @@ -81,6 +81,9 @@ public class BaseTimeseriesService implements TimeseriesService { @Value("${database.ts_max_intervals}") private long maxTsIntervals; + @Value("${sql.ts.noxss_validation_enabled:true}") + private boolean noxssValidationEnabled; + @Autowired private TimeseriesDao timeseriesDao; @@ -156,7 +159,7 @@ public class BaseTimeseriesService implements TimeseriesService { @Override public ListenableFuture save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry) { - KvUtils.validate(tsKvEntry); + KvUtils.validate(tsKvEntry, noxssValidationEnabled); validate(entityId); List> futures = Lists.newArrayListWithExpectedSize(INSERTS_PER_ENTRY); saveAndRegisterFutures(tenantId, futures, entityId, tsKvEntry, 0L); @@ -174,7 +177,7 @@ public class BaseTimeseriesService implements TimeseriesService { } private ListenableFuture doSave(TenantId tenantId, EntityId entityId, List tsKvEntries, long ttl, boolean saveLatest) { - KvUtils.validate(tsKvEntries); + KvUtils.validate(tsKvEntries, noxssValidationEnabled); int inserts = saveLatest ? INSERTS_PER_ENTRY : INSERTS_PER_ENTRY_WITHOUT_LATEST; List> futures = Lists.newArrayListWithExpectedSize(tsKvEntries.size() * inserts); for (TsKvEntry tsKvEntry : tsKvEntries) { @@ -189,7 +192,7 @@ public class BaseTimeseriesService implements TimeseriesService { @Override public ListenableFuture> saveLatest(TenantId tenantId, EntityId entityId, List tsKvEntries) { - KvUtils.validate(tsKvEntries); + KvUtils.validate(tsKvEntries, noxssValidationEnabled); List> futures = Lists.newArrayListWithExpectedSize(tsKvEntries.size()); for (TsKvEntry tsKvEntry : tsKvEntries) { futures.add(timeseriesLatestDao.saveLatest(tenantId, entityId, tsKvEntry)); diff --git a/dao/src/main/java/org/thingsboard/server/dao/util/KvUtils.java b/dao/src/main/java/org/thingsboard/server/dao/util/KvUtils.java index 8e80bd992d..133fb4c85e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/util/KvUtils.java +++ b/dao/src/main/java/org/thingsboard/server/dao/util/KvUtils.java @@ -22,14 +22,16 @@ import org.thingsboard.server.dao.service.ConstraintValidator; import java.util.List; public class KvUtils { - public static void validate(List tsKvEntries) { - tsKvEntries.forEach(KvUtils::validate); + public static void validate(List tsKvEntries, boolean validateNoxss) { + tsKvEntries.forEach(kvEntry -> validate(kvEntry, validateNoxss)); } - public static void validate(KvEntry tsKvEntry) { + public static void validate(KvEntry tsKvEntry, boolean validateNoxss) { if (tsKvEntry == null) { throw new IncorrectParameterException("Key value entry can't be null"); } - ConstraintValidator.validateFields(tsKvEntry); + if (validateNoxss) { + ConstraintValidator.validateFields(tsKvEntry); + } } } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/ConstraintValidatorTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/ConstraintValidatorTest.java index 204f019318..fa3d39df30 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/ConstraintValidatorTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/ConstraintValidatorTest.java @@ -16,6 +16,7 @@ package org.thingsboard.server.dao.service; import org.junit.Assert; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.thingsboard.server.common.data.kv.JsonDataEntry; import org.thingsboard.server.common.data.kv.StringDataEntry; @@ -23,6 +24,9 @@ import org.thingsboard.server.dao.exception.DataValidationException; class ConstraintValidatorTest { + private static final int MIN_IN_MS = 60000; + private static final int _1M = 1_000_000; + @Test void validateFields() { StringDataEntry stringDataEntryValid = new StringDataEntry("key", "value"); @@ -33,8 +37,21 @@ class ConstraintValidatorTest { JsonDataEntry jsonDataEntryInvalid = new JsonDataEntry("key", "{\"value\": }"); Assert.assertThrows(DataValidationException.class, () -> ConstraintValidator.validateFields(stringDataEntryInvalid1)); -// Assert.assertThrows(DataValidationException.class, () -> ConstraintValidator.validateFields(stringDataEntryInvalid2)); -// Assert.assertThrows(DataValidationException.class, () -> ConstraintValidator.validateFields(jsonDataEntryInvalid)); + Assert.assertThrows(DataValidationException.class, () -> ConstraintValidator.validateFields(stringDataEntryInvalid2)); + Assert.assertThrows(DataValidationException.class, () -> ConstraintValidator.validateFields(jsonDataEntryInvalid)); ConstraintValidator.validateFields(stringDataEntryValid); } + + @Test + void validatePerMinute() { + StringDataEntry stringDataEntryValid = new StringDataEntry("key", "value"); + + long start = System.currentTimeMillis(); + for (int i = 0; i < _1M; i++) { + ConstraintValidator.validateFields(stringDataEntryValid); + } + long end = System.currentTimeMillis(); + + Assertions.assertTrue(MIN_IN_MS > end - start); + } } \ No newline at end of file