dendi-wang/thingsboard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #8472 from YevhenBondarenko/kv-validator

Browse Source 
kv validator improvements
This commit is contained in:
Andrew Shvayka 2023-05-03 15:19:17 +03:00 committed by GitHub
commit b3e4e2adcb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 51 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java
View File

@ -43,6 +43,7 @@ import org.thingsboard.server.common.msg.queue.TbCallback;
import org.thingsboard.server.common.stats.TbApiUsageReportClient;
import org.thingsboard.server.dao.attributes.AttributesService;
import org.thingsboard.server.dao.timeseries.TimeseriesService;
import org.thingsboard.server.dao.util.KvUtils;
import org.thingsboard.server.service.apiusage.TbApiUsageStateService;
import org.thingsboard.server.service.entitiy.entityview.TbEntityViewService;
import org.thingsboard.server.service.subscription.TbSubscriptionUtils;
@ -134,6 +135,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer
checkInternalEntity(entityId);
boolean sysTenant = TenantId.SYS_TENANT_ID.equals(tenantId) || tenantId == null;
if (sysTenant || apiUsageStateService.getApiUsageState(tenantId).isDbStorageEnabled()) {
KvUtils.validate(ts);
if (saveLatest) {
saveAndNotifyInternal(tenantId, entityId, ts, ttl, getCallback(tenantId, customerId, sysTenant, callback));
} else {

30
dao/src/main/java/org/thingsboard/server/dao/service/NoXssValidator.java
View File

@ -30,21 +30,18 @@ import java.util.Optional;
@Slf4j
public class NoXssValidator implements ConstraintValidator<NoXss, Object> {
private static final AntiSamy xssChecker = new AntiSamy();
private static Policy xssPolicy;
private static final Policy xssPolicy;
@Override
public void initialize(NoXss constraintAnnotation) {
if (xssPolicy == null) {
xssPolicy = Optional.ofNullable(getClass().getClassLoader().getResourceAsStream("xss-policy.xml"))
.map(inputStream -> {
try {
return Policy.getInstance(inputStream);
} catch (Exception e) {
throw new RuntimeException(e);
}
})
.orElseThrow(() -> new IllegalStateException("XSS policy file not found"));
}
static {
xssPolicy = Optional.ofNullable(NoXssValidator.class.getClassLoader().getResourceAsStream("xss-policy.xml"))
.map(inputStream -> {
try {
return Policy.getInstance(inputStream);
} catch (Exception e) {
throw new RuntimeException(e);
}
})
.orElseThrow(() -> new IllegalStateException("XSS policy file not found"));
}
@Override
@ -55,10 +52,13 @@ public class NoXssValidator implements ConstraintValidator<NoXss, Object> {
} else {
return true;
}
return isValid(stringValue);
}
public static boolean isValid(String stringValue) {
if (stringValue.isEmpty()) {
return true;
}
try {
return xssChecker.scan(stringValue, xssPolicy).getNumberOfErrors() == 0;
} catch (ScanException | PolicyException e) {

4
dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java
View File

@ -42,7 +42,6 @@ import org.thingsboard.server.common.data.kv.TsKvLatestRemovingResult;
import org.thingsboard.server.dao.entityview.EntityViewService;
import org.thingsboard.server.dao.exception.IncorrectParameterException;
import org.thingsboard.server.dao.service.Validator;
import org.thingsboard.server.dao.util.KvUtils;
import java.util.Collection;
import java.util.Collections;
@ -156,7 +155,6 @@ public class BaseTimeseriesService implements TimeseriesService {
@Override
public ListenableFuture<Integer> save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry) {
KvUtils.validate(tsKvEntry);
validate(entityId);
List<ListenableFuture<Integer>> futures = Lists.newArrayListWithExpectedSize(INSERTS_PER_ENTRY);
saveAndRegisterFutures(tenantId, futures, entityId, tsKvEntry, 0L);
@ -174,7 +172,6 @@ public class BaseTimeseriesService implements TimeseriesService {
}
private ListenableFuture<Integer> doSave(TenantId tenantId, EntityId entityId, List<TsKvEntry> tsKvEntries, long ttl, boolean saveLatest) {
KvUtils.validate(tsKvEntries);
int inserts = saveLatest ? INSERTS_PER_ENTRY : INSERTS_PER_ENTRY_WITHOUT_LATEST;
List<ListenableFuture<Integer>> futures = Lists.newArrayListWithExpectedSize(tsKvEntries.size() * inserts);
for (TsKvEntry tsKvEntry : tsKvEntries) {
@ -189,7 +186,6 @@ public class BaseTimeseriesService implements TimeseriesService {
@Override
public ListenableFuture<List<Void>> saveLatest(TenantId tenantId, EntityId entityId, List<TsKvEntry> tsKvEntries) {
KvUtils.validate(tsKvEntries);
List<ListenableFuture<Void>> futures = Lists.newArrayListWithExpectedSize(tsKvEntries.size());
for (TsKvEntry tsKvEntry : tsKvEntries) {
futures.add(timeseriesLatestDao.saveLatest(tenantId, entityId, tsKvEntry));

36
dao/src/main/java/org/thingsboard/server/dao/util/KvUtils.java
View File

@ -15,13 +15,27 @@
*/
package org.thingsboard.server.dao.util;
import com.github.benmanes.caffeine.cache.Cache;
import com.github.benmanes.caffeine.cache.Caffeine;
import org.thingsboard.server.common.data.kv.KvEntry;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.dao.exception.IncorrectParameterException;
import org.thingsboard.server.dao.service.ConstraintValidator;
import org.thingsboard.server.dao.service.NoXssValidator;
import java.util.List;
import java.util.concurrent.TimeUnit;
public class KvUtils {
private static final Cache<String, Boolean> validatedKeys;
static {
validatedKeys = Caffeine.newBuilder()
.weakKeys()
.expireAfterAccess(60, TimeUnit.MINUTES)
.maximumSize(100000).build();
}
public static void validate(List<? extends KvEntry> tsKvEntries) {
tsKvEntries.forEach(KvUtils::validate);
}
@ -30,6 +44,24 @@ public class KvUtils {
if (tsKvEntry == null) {
throw new IncorrectParameterException("Key value entry can't be null");
}
ConstraintValidator.validateFields(tsKvEntry);
String key = tsKvEntry.getKey();
if (key == null) {
throw new DataValidationException("Key can't be null");
}
if (validatedKeys.getIfPresent(key) != null) {
return;
}
if (!NoXssValidator.isValid(key)) {
throw new DataValidationException("Validation error: key is malformed");
}
if (key.length() > 255) {
throw new DataValidationException("Validation error: key length must be equal or less than 255");
}
validatedKeys.put(key, Boolean.TRUE);
}
}