dendi-wang/thingsboard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

added noxss enable param for ts and attributes

Browse Source
This commit is contained in:
YevhenBondarenko 2023-04-12 11:30:04 +02:00
parent 1a3ee8512d
commit ef86dc49b9
10 changed files with 70 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
application/src/main/resources/thingsboard.yml
View File

@ -255,11 +255,13 @@ sql:
batch_max_delay: "${SQL_ATTRIBUTES_BATCH_MAX_DELAY_MS:100}"
stats_print_interval_ms: "${SQL_ATTRIBUTES_BATCH_STATS_PRINT_MS:10000}"
batch_threads: "${SQL_ATTRIBUTES_BATCH_THREADS:3}" # batch thread count have to be a prime number like 3 or 5 to gain perfect hash distribution
noxss_validation_enabled: "${SQL_ATTRIBUTES_NOXSS_VALIDATION_ENABLED:true}"
ts:
batch_size: "${SQL_TS_BATCH_SIZE:10000}"
batch_max_delay: "${SQL_TS_BATCH_MAX_DELAY_MS:100}"
stats_print_interval_ms: "${SQL_TS_BATCH_STATS_PRINT_MS:10000}"
batch_threads: "${SQL_TS_BATCH_THREADS:3}" # batch thread count have to be a prime number like 3 or 5 to gain perfect hash distribution
noxss_validation_enabled: "${SQL_TS_NOXSS_VALIDATION_ENABLED:true}"
ts_latest:
batch_size: "${SQL_TS_LATEST_BATCH_SIZE:10000}"
batch_max_delay: "${SQL_TS_LATEST_BATCH_MAX_DELAY_MS:100}"

18
application/src/test/java/org/thingsboard/server/controller/BaseTelemetryControllerTest.java
View File

@ -29,12 +29,15 @@ public abstract class BaseTelemetryControllerTest extends AbstractControllerTest
public void testConstraintValidator() throws Exception {
loginTenantAdmin();
Device device = createDevice();
String requestBody = "{\"data\": \"<object data=\\\"data:text/html,<script>alert(document)</script>\\\"></object>\"}";
doPostAsync("/api/plugins/telemetry/" + device.getId() + "/SHARED_SCOPE", requestBody, String.class, status().isOk());
doPostAsync("/api/plugins/telemetry/DEVICE/" + device.getId() + "/timeseries/smth", requestBody, String.class, status().isOk());
requestBody = "{\"<object data=\\\"data:text/html,<script>alert(document)</script>\\\"></object>\": \"data\"}";
doPostAsync("/api/plugins/telemetry/" + device.getId() + "/SHARED_SCOPE", requestBody, String.class, status().isBadRequest());
doPostAsync("/api/plugins/telemetry/DEVICE/" + device.getId() + "/timeseries/smth", requestBody, String.class, status().isBadRequest());
String correctRequestBody = "{\"data\": \"value\"}";
doPostAsync("/api/plugins/telemetry/" + device.getId() + "/SHARED_SCOPE", correctRequestBody, String.class, status().isOk());
doPostAsync("/api/plugins/telemetry/DEVICE/" + device.getId() + "/timeseries/smth", correctRequestBody, String.class, status().isOk());
String invalidRequestBody = "{\"data\": \"<object data=\\\"data:text/html,<script>alert(document)</script>\\\"></object>\"}";
doPostAsync("/api/plugins/telemetry/" + device.getId() + "/SHARED_SCOPE", invalidRequestBody, String.class, status().isBadRequest());
doPostAsync("/api/plugins/telemetry/DEVICE/" + device.getId() + "/timeseries/smth", invalidRequestBody, String.class, status().isBadRequest());
invalidRequestBody = "{\"<object data=\\\"data:text/html,<script>alert(document)</script>\\\"></object>\": \"data\"}";
doPostAsync("/api/plugins/telemetry/" + device.getId() + "/SHARED_SCOPE", invalidRequestBody, String.class, status().isBadRequest());
doPostAsync("/api/plugins/telemetry/DEVICE/" + device.getId() + "/timeseries/smth", invalidRequestBody, String.class, status().isBadRequest());
}
private Device createDevice() throws Exception {
@ -50,7 +53,6 @@ public abstract class BaseTelemetryControllerTest extends AbstractControllerTest
SaveDeviceWithCredentialsRequest saveRequest = new SaveDeviceWithCredentialsRequest(device, deviceCredentials);
Device savedDevice = readResponse(doPost("/api/device-with-credentials", saveRequest).andExpect(status().isOk()), Device.class);
return savedDevice;
return readResponse(doPost("/api/device-with-credentials", saveRequest).andExpect(status().isOk()), Device.class);
}
}

4
common/data/src/main/java/org/thingsboard/server/common/data/kv/JsonDataEntry.java
View File

@ -15,10 +15,14 @@
*/
package org.thingsboard.server.common.data.kv;
import org.thingsboard.server.common.data.validation.NoXss;
import java.util.Objects;
import java.util.Optional;
public class JsonDataEntry extends BasicKvEntry {
@NoXss
private final String value;
public JsonDataEntry(String key, String value) {

3
common/data/src/main/java/org/thingsboard/server/common/data/kv/StringDataEntry.java
View File

@ -15,6 +15,8 @@
*/
package org.thingsboard.server.common.data.kv;
import org.thingsboard.server.common.data.validation.NoXss;
import java.util.Objects;
import java.util.Optional;
@ -22,6 +24,7 @@ public class StringDataEntry extends BasicKvEntry {
private static final long serialVersionUID = 1L;
@NoXss
private final String value;
public StringDataEntry(String key, String value) {

10
dao/src/main/java/org/thingsboard/server/dao/attributes/AttributeUtils.java
View File

@ -21,6 +21,8 @@ import org.thingsboard.server.dao.exception.IncorrectParameterException;
import org.thingsboard.server.dao.service.Validator;
import org.thingsboard.server.dao.util.KvUtils;
import java.util.List;
public class AttributeUtils {
public static void validate(EntityId id, String scope) {
@ -28,8 +30,12 @@ public class AttributeUtils {
Validator.validateString(scope, "Incorrect scope " + scope);
}
public static void validate(AttributeKvEntry kvEntry) {
KvUtils.validate(kvEntry);
public static void validate(List<AttributeKvEntry> kvEntries, boolean validateNoxss) {
kvEntries.forEach(kv -> validate(kv, validateNoxss));
}
public static void validate(AttributeKvEntry kvEntry, boolean validateNoxss) {
KvUtils.validate(kvEntry, validateNoxss);
if (kvEntry.getDataType() == null) {
throw new IncorrectParameterException("Incorrect kvEntry. Data type can't be null");
} else {

8
dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java
View File

@ -18,6 +18,7 @@ package org.thingsboard.server.dao.attributes;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.context.annotation.Primary;
import org.springframework.stereotype.Service;
@ -45,6 +46,9 @@ import static org.thingsboard.server.dao.attributes.AttributeUtils.validate;
public class BaseAttributesService implements AttributesService {
private final AttributesDao attributesDao;
@Value("${sql.attributes.noxss_validation_enabled:true}")
private boolean noxssValidationEnabled;
public BaseAttributesService(AttributesDao attributesDao) {
this.attributesDao = attributesDao;
}
@ -82,14 +86,14 @@ public class BaseAttributesService implements AttributesService {
@Override
public ListenableFuture<String> save(TenantId tenantId, EntityId entityId, String scope, AttributeKvEntry attribute) {
validate(entityId, scope);
AttributeUtils.validate(attribute);
AttributeUtils.validate(attribute, noxssValidationEnabled);
return attributesDao.save(tenantId, entityId, scope, attribute);
}
@Override
public ListenableFuture<List<String>> save(TenantId tenantId, EntityId entityId, String scope, List<AttributeKvEntry> attributes) {
validate(entityId, scope);
attributes.forEach(AttributeUtils::validate);
AttributeUtils.validate(attributes, noxssValidationEnabled);
List<ListenableFuture<String>> saveFutures = attributes.stream().map(attribute -> attributesDao.save(tenantId, entityId, scope, attribute)).collect(Collectors.toList());
return Futures.allAsList(saveFutures);
}

9
dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java
View File

@ -70,6 +70,9 @@ public class CachedAttributesService implements AttributesService {
@Value("${cache.type:caffeine}")
private String cacheType;
@Value("${sql.attributes.noxss_validation_enabled:true}")
private boolean noxssValidationEnabled;
public CachedAttributesService(AttributesDao attributesDao,
StatsFactory statsFactory,
CacheExecutorService cacheExecutorService,
@ -134,7 +137,7 @@ public class CachedAttributesService implements AttributesService {
@Override
public ListenableFuture<List<AttributeKvEntry>> find(TenantId tenantId, EntityId entityId, String scope, Collection<String> attributeKeys) {
validate(entityId, scope);
attributeKeys = new LinkedHashSet<>(attributeKeys); // deduplicate the attributes
attributeKeys = new LinkedHashSet<>(attributeKeys); // deduplicate the attributes
attributeKeys.forEach(attributeKey -> Validator.validateString(attributeKey, "Incorrect attribute key " + attributeKey));
Map<String, TbCacheValueWrapper<AttributeKvEntry>> wrappedCachedAttributes = findCachedAttributes(entityId, scope, attributeKeys);
@ -212,7 +215,7 @@ public class CachedAttributesService implements AttributesService {
@Override
public ListenableFuture<String> save(TenantId tenantId, EntityId entityId, String scope, AttributeKvEntry attribute) {
validate(entityId, scope);
AttributeUtils.validate(attribute);
AttributeUtils.validate(attribute, noxssValidationEnabled);
ListenableFuture<String> future = attributesDao.save(tenantId, entityId, scope, attribute);
return Futures.transform(future, key -> evict(entityId, scope, attribute, key), cacheExecutor);
}
@ -220,7 +223,7 @@ public class CachedAttributesService implements AttributesService {
@Override
public ListenableFuture<List<String>> save(TenantId tenantId, EntityId entityId, String scope, List<AttributeKvEntry> attributes) {
validate(entityId, scope);
attributes.forEach(AttributeUtils::validate);
AttributeUtils.validate(attributes, noxssValidationEnabled);
List<ListenableFuture<String>> futures = new ArrayList<>(attributes.size());
for (var attribute : attributes) {

9
dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java
View File

@ -81,6 +81,9 @@ public class BaseTimeseriesService implements TimeseriesService {
@Value("${database.ts_max_intervals}")
private long maxTsIntervals;
@Value("${sql.ts.noxss_validation_enabled:true}")
private boolean noxssValidationEnabled;
@Autowired
private TimeseriesDao timeseriesDao;
@ -156,7 +159,7 @@ public class BaseTimeseriesService implements TimeseriesService {
@Override
public ListenableFuture<Integer> save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry) {
KvUtils.validate(tsKvEntry);
KvUtils.validate(tsKvEntry, noxssValidationEnabled);
validate(entityId);
List<ListenableFuture<Integer>> futures = Lists.newArrayListWithExpectedSize(INSERTS_PER_ENTRY);
saveAndRegisterFutures(tenantId, futures, entityId, tsKvEntry, 0L);
@ -174,7 +177,7 @@ public class BaseTimeseriesService implements TimeseriesService {
}
private ListenableFuture<Integer> doSave(TenantId tenantId, EntityId entityId, List<TsKvEntry> tsKvEntries, long ttl, boolean saveLatest) {
KvUtils.validate(tsKvEntries);
KvUtils.validate(tsKvEntries, noxssValidationEnabled);
int inserts = saveLatest ? INSERTS_PER_ENTRY : INSERTS_PER_ENTRY_WITHOUT_LATEST;
List<ListenableFuture<Integer>> futures = Lists.newArrayListWithExpectedSize(tsKvEntries.size() * inserts);
for (TsKvEntry tsKvEntry : tsKvEntries) {
@ -189,7 +192,7 @@ public class BaseTimeseriesService implements TimeseriesService {
@Override
public ListenableFuture<List<Void>> saveLatest(TenantId tenantId, EntityId entityId, List<TsKvEntry> tsKvEntries) {
KvUtils.validate(tsKvEntries);
KvUtils.validate(tsKvEntries, noxssValidationEnabled);
List<ListenableFuture<Void>> futures = Lists.newArrayListWithExpectedSize(tsKvEntries.size());
for (TsKvEntry tsKvEntry : tsKvEntries) {
futures.add(timeseriesLatestDao.saveLatest(tenantId, entityId, tsKvEntry));

10
dao/src/main/java/org/thingsboard/server/dao/util/KvUtils.java
View File

@ -22,14 +22,16 @@ import org.thingsboard.server.dao.service.ConstraintValidator;
import java.util.List;
public class KvUtils {
public static void validate(List<? extends KvEntry> tsKvEntries) {
tsKvEntries.forEach(KvUtils::validate);
public static void validate(List<? extends KvEntry> tsKvEntries, boolean validateNoxss) {
tsKvEntries.forEach(kvEntry -> validate(kvEntry, validateNoxss));
}
public static void validate(KvEntry tsKvEntry) {
public static void validate(KvEntry tsKvEntry, boolean validateNoxss) {
if (tsKvEntry == null) {
throw new IncorrectParameterException("Key value entry can't be null");
}
ConstraintValidator.validateFields(tsKvEntry);
if (validateNoxss) {
ConstraintValidator.validateFields(tsKvEntry);
}
}
}

21
dao/src/test/java/org/thingsboard/server/dao/service/ConstraintValidatorTest.java
View File

@ -16,6 +16,7 @@
package org.thingsboard.server.dao.service;
import org.junit.Assert;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import org.thingsboard.server.common.data.kv.JsonDataEntry;
import org.thingsboard.server.common.data.kv.StringDataEntry;
@ -23,6 +24,9 @@ import org.thingsboard.server.dao.exception.DataValidationException;
class ConstraintValidatorTest {
private static final int MIN_IN_MS = 60000;
private static final int _1M = 1_000_000;
@Test
void validateFields() {
StringDataEntry stringDataEntryValid = new StringDataEntry("key", "value");
@ -33,8 +37,21 @@ class ConstraintValidatorTest {
JsonDataEntry jsonDataEntryInvalid = new JsonDataEntry("key", "{\"value\": <object type=\"text/html\"><script>alert(document)</script></object>}");
Assert.assertThrows(DataValidationException.class, () -> ConstraintValidator.validateFields(stringDataEntryInvalid1));
// Assert.assertThrows(DataValidationException.class, () -> ConstraintValidator.validateFields(stringDataEntryInvalid2));
// Assert.assertThrows(DataValidationException.class, () -> ConstraintValidator.validateFields(jsonDataEntryInvalid));
Assert.assertThrows(DataValidationException.class, () -> ConstraintValidator.validateFields(stringDataEntryInvalid2));
Assert.assertThrows(DataValidationException.class, () -> ConstraintValidator.validateFields(jsonDataEntryInvalid));
ConstraintValidator.validateFields(stringDataEntryValid);
}
@Test
void validatePerMinute() {
StringDataEntry stringDataEntryValid = new StringDataEntry("key", "value");
long start = System.currentTimeMillis();
for (int i = 0; i < _1M; i++) {
ConstraintValidator.validateFields(stringDataEntryValid);
}
long end = System.currentTimeMillis();
Assertions.assertTrue(MIN_IN_MS > end - start);
}
}