dendi-wang/thingsboard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

upgrade logic + rate limits validation

Browse Source
This commit is contained in:
dshvaika 2025-05-22 18:08:15 +03:00
parent a242d3a43b
commit 35e9d007d0
10 changed files with 254 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
application/src/main/data/upgrade/basic/schema_update.sql
View File

@ -14,3 +14,28 @@
-- limitations under the License.
--
UPDATE tenant_profile
SET profile_data = jsonb_set(
profile_data,
'{configuration}',
(
(profile_data -> 'configuration') - 'cassandraQueryTenantRateLimitsConfiguration'
||
COALESCE(
CASE
WHEN profile_data -> 'configuration' ->
'cassandraQueryTenantRateLimitsConfiguration' IS NOT NULL THEN
jsonb_build_object(
'cassandraReadQueryTenantCoreRateLimits',
profile_data -> 'configuration' -> 'cassandraQueryTenantRateLimitsConfiguration',
'cassandraWriteQueryTenantCoreRateLimits',
profile_data -> 'configuration' -> 'cassandraQueryTenantRateLimitsConfiguration',
'cassandraReadQueryTenantRuleEngineRateLimits',
profile_data -> 'configuration' -> 'cassandraQueryTenantRateLimitsConfiguration',
'cassandraWriteQueryTenantRuleEngineRateLimits',
profile_data -> 'configuration' -> 'cassandraQueryTenantRateLimitsConfiguration'
)
END,
'{}'::jsonb)
)
);

46
application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java
View File

@ -24,6 +24,7 @@ import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Profile;
import org.springframework.stereotype.Service;
import org.thingsboard.server.common.data.TenantProfile;
import org.thingsboard.server.common.data.alarm.AlarmSeverity;
import org.thingsboard.server.common.data.id.RuleChainId;
import org.thingsboard.server.common.data.id.RuleNodeId;
@ -34,8 +35,10 @@ import org.thingsboard.server.common.data.query.FilterPredicateValue;
import org.thingsboard.server.common.data.relation.EntityRelation;
import org.thingsboard.server.common.data.relation.RelationTypeGroup;
import org.thingsboard.server.common.data.rule.RuleNode;
import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration;
import org.thingsboard.server.dao.relation.RelationService;
import org.thingsboard.server.dao.rule.RuleChainService;
import org.thingsboard.server.dao.tenant.TenantProfileService;
import org.thingsboard.server.service.component.ComponentDiscoveryService;
import org.thingsboard.server.service.component.RuleNodeClassInfo;
import org.thingsboard.server.service.install.DbUpgradeExecutorService;
@ -69,14 +72,57 @@ public class DefaultDataUpdateService implements DataUpdateService {
@Autowired
private DbUpgradeExecutorService executorService;
@Autowired
private TenantProfileService tenantProfileService;
@Override
public void updateData() throws Exception {
log.info("Updating data ...");
//TODO: should be cleaned after each release
updateInputNodes();
deduplicateRateLimitsPerSecondsConfigurations();
log.info("Data updated.");
}
private void deduplicateRateLimitsPerSecondsConfigurations() {
log.info("Starting update of tenant profiles...");
int totalProfiles = 0;
int updatedTenantProfiles = 0;
int skippedProfiles = 0;
int failedProfiles = 0;
var tenantProfiles = new PageDataIterable<>(
pageLink -> tenantProfileService.findTenantProfiles(TenantId.SYS_TENANT_ID, pageLink), 1024);
for (TenantProfile tenantProfile : tenantProfiles) {
totalProfiles++;
String profileName = tenantProfile.getName();
UUID profileId = tenantProfile.getId().getId();
try {
Optional<DefaultTenantProfileConfiguration> profileConfiguration = tenantProfile.getProfileConfiguration();
if (profileConfiguration.isEmpty()) {
log.debug("[{}][{}] Skipping tenant profile with non-default configuration.", profileId, profileName);
skippedProfiles++;
continue;
}
DefaultTenantProfileConfiguration defaultTenantProfileConfiguration = profileConfiguration.get();
defaultTenantProfileConfiguration.deduplicateRateLimitsConfigs();
tenantProfileService.saveTenantProfile(TenantId.SYS_TENANT_ID, tenantProfile);
updatedTenantProfiles++;
log.debug("[{}][{}] Successfully updated tenant profile.", profileId, profileName);
} catch (Exception e) {
log.error("[{}][{}] Failed to updated tenant profile: ", profileId, profileName, e);
failedProfiles++;
}
}
log.info("Tenant profiles update completed. Total: {}, Updated: {}, Skipped: {}, Failed: {}",
totalProfiles, updatedTenantProfiles, skippedProfiles, failedProfiles);
}
private void updateInputNodes() {
log.info("Creating relations for input nodes...");
int n = 0;

21
common/data/src/main/java/org/thingsboard/server/common/data/limit/LimitedApi.java
View File

@ -31,7 +31,7 @@ public enum LimitedApi {
REST_REQUESTS_PER_CUSTOMER(DefaultTenantProfileConfiguration::getCustomerServerRestLimitsConfiguration, "REST API requests per customer", false),
WS_UPDATES_PER_SESSION(DefaultTenantProfileConfiguration::getWsUpdatesPerSessionRateLimit, "WS updates per session", true),
CASSANDRA_WRITE_QUERIES_CORE(DefaultTenantProfileConfiguration::getCassandraReadQueryTenantCoreRateLimits, "Rest API and WS telemetry read queries", true),
CASSANDRA_READ_QUERIES_CORE(DefaultTenantProfileConfiguration::getCassandraWriteQueryTenantCoreRateLimits, "Rest API and WS telemetry write queries", true),
CASSANDRA_READ_QUERIES_CORE(DefaultTenantProfileConfiguration::getCassandraWriteQueryTenantCoreRateLimits, "Rest API write queries", true),
CASSANDRA_WRITE_QUERIES_RULE_ENGINE(DefaultTenantProfileConfiguration::getCassandraReadQueryTenantRuleEngineRateLimits, "Rule Engine telemetry read queries", true),
CASSANDRA_READ_QUERIES_RULE_ENGINE(DefaultTenantProfileConfiguration::getCassandraWriteQueryTenantRuleEngineRateLimits, "Rule Engine telemetry write queries", true),
CASSANDRA_READ_QUERIES_MONOLITH(
@ -57,28 +57,31 @@ public enum LimitedApi {
WS_SUBSCRIPTIONS("WS subscriptions", false),
CALCULATED_FIELD_DEBUG_EVENTS("calculated field debug events", true);
private Function<DefaultTenantProfileConfiguration, String> configExtractor;
private final Function<DefaultTenantProfileConfiguration, String> configExtractor;
@Getter
private final boolean perTenant;
@Getter
private boolean refillRateLimitIntervally;
private final boolean refillRateLimitIntervally;
@Getter
private String label;
private final String label;
LimitedApi(Function<DefaultTenantProfileConfiguration, String> configExtractor, String label, boolean perTenant) {
this.configExtractor = configExtractor;
this.label = label;
this.perTenant = perTenant;
this(configExtractor, label, perTenant, false);
}
LimitedApi(boolean perTenant, boolean refillRateLimitIntervally) {
this.perTenant = perTenant;
this.refillRateLimitIntervally = refillRateLimitIntervally;
this(null, null, perTenant, refillRateLimitIntervally);
}
LimitedApi(String label, boolean perTenant) {
this(null, label, perTenant, false);
}
LimitedApi(Function<DefaultTenantProfileConfiguration, String> configExtractor, String label, boolean perTenant, boolean refillRateLimitIntervally) {
this.configExtractor = configExtractor;
this.label = label;
this.perTenant = perTenant;
this.refillRateLimitIntervally = refillRateLimitIntervally;
}
public String getLimitConfig(DefaultTenantProfileConfiguration profileConfiguration) {

25
common/data/src/main/java/org/thingsboard/server/common/data/limit/LimitedApiUtil.java
View File

@ -21,8 +21,10 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.function.Function;
import java.util.stream.Collectors;
@ -64,4 +66,27 @@ public class LimitedApiUtil {
.collect(Collectors.joining(","));
}
public static boolean isValid(String configStr) {
List<LimitedApiEntry> limitedApiEntries = parseConfig(configStr);
Set<Long> distinctDurations = new HashSet<>();
for (LimitedApiEntry entry : limitedApiEntries) {
if (!distinctDurations.add(entry.durationSeconds())) {
return false;
}
}
return true;
}
@Deprecated(forRemoval = true, since = "4.0.2")
public static String deduplicateByDuration(String configStr) {
if (configStr == null) {
return null;
}
Set<Long> distinctDurations = new HashSet<>();
return parseConfig(configStr).stream()
.filter(entry -> distinctDurations.add(entry.durationSeconds()))
.map(LimitedApiEntry::toString)
.collect(Collectors.joining(","));
}
}

68
common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java
View File

@ -24,6 +24,8 @@ import lombok.NoArgsConstructor;
import org.thingsboard.server.common.data.ApiUsageRecordKey;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.TenantProfileType;
import org.thingsboard.server.common.data.limit.LimitedApiUtil;
import org.thingsboard.server.common.data.validation.RateLimit;
import java.io.Serial;
@ -49,37 +51,53 @@ public class DefaultTenantProfileConfiguration implements TenantProfileConfigura
private long maxResourceSize;
@Schema(example = "1000:1,20000:60")
@RateLimit(fieldName = "Transport tenant messages")
private String transportTenantMsgRateLimit;
@Schema(example = "1000:1,20000:60")
@RateLimit(fieldName = "Transport tenant telemetry messages")
private String transportTenantTelemetryMsgRateLimit;
@Schema(example = "1000:1,20000:60")
@RateLimit(fieldName = "Transport tenant telemetry data points")
private String transportTenantTelemetryDataPointsRateLimit;
@Schema(example = "20:1,600:60")
@RateLimit(fieldName = "Transport device messages")
private String transportDeviceMsgRateLimit;
@Schema(example = "20:1,600:60")
@RateLimit(fieldName = "Transport device telemetry messages")
private String transportDeviceTelemetryMsgRateLimit;
@Schema(example = "20:1,600:60")
@RateLimit(fieldName = "Transport device telemetry data points")
private String transportDeviceTelemetryDataPointsRateLimit;
@Schema(example = "20:1,600:60")
@RateLimit(fieldName = "Transport gateway messages")
private String transportGatewayMsgRateLimit;
@Schema(example = "20:1,600:60")
@RateLimit(fieldName = "Transport gateway telemetry messages")
private String transportGatewayTelemetryMsgRateLimit;
@Schema(example = "20:1,600:60")
@RateLimit(fieldName = "Transport gateway telemetry data points")
private String transportGatewayTelemetryDataPointsRateLimit;
@Schema(example = "20:1,600:60")
@RateLimit(fieldName = "Transport gateway device messages")
private String transportGatewayDeviceMsgRateLimit;
@Schema(example = "20:1,600:60")
@RateLimit(fieldName = "Transport gateway device telemetry messages")
private String transportGatewayDeviceTelemetryMsgRateLimit;
@Schema(example = "20:1,600:60")
@RateLimit(fieldName = "Transport gateway device telemetry data points")
private String transportGatewayDeviceTelemetryDataPointsRateLimit;
@Schema(example = "20:1,600:60")
@RateLimit(fieldName = "Entity version creation")
private String tenantEntityExportRateLimit;
@Schema(example = "20:1,600:60")
@RateLimit(fieldName = "Entity version load")
private String tenantEntityImportRateLimit;
@Schema(example = "20:1,600:60")
@RateLimit(fieldName = "Notification requests")
private String tenantNotificationRequestsRateLimit;
@Schema(example = "20:1,600:60")
@RateLimit(fieldName = "Notification requests per notification rule")
private String tenantNotificationRequestsPerRuleRateLimit;
@Schema(example = "10000000")
@ -107,7 +125,9 @@ public class DefaultTenantProfileConfiguration implements TenantProfileConfigura
@Schema(example = "1000")
private long maxCreatedAlarms;
@RateLimit(fieldName = "REST requests for tenant")
private String tenantServerRestLimitsConfiguration;
@RateLimit(fieldName = "REST requests for customer")
private String customerServerRestLimitsConfiguration;
private int maxWsSessionsPerTenant;
@ -119,17 +139,26 @@ public class DefaultTenantProfileConfiguration implements TenantProfileConfigura
private long maxWsSubscriptionsPerCustomer;
private long maxWsSubscriptionsPerRegularUser;
private long maxWsSubscriptionsPerPublicUser;
@RateLimit(fieldName = "WS updates per session")
private String wsUpdatesPerSessionRateLimit;
@RateLimit(fieldName = "Rest API and WS telemetry read queries")
private String cassandraReadQueryTenantCoreRateLimits;
@RateLimit(fieldName = "Rest API write queries")
private String cassandraWriteQueryTenantCoreRateLimits;
@RateLimit(fieldName = "Rule Engine telemetry read queries")
private String cassandraReadQueryTenantRuleEngineRateLimits;
@RateLimit(fieldName = "Rule Engine telemetry write queries")
private String cassandraWriteQueryTenantRuleEngineRateLimits;
@RateLimit(fieldName = "Edge events")
private String edgeEventRateLimits;
@RateLimit(fieldName = "Edge events per edge")
private String edgeEventRateLimitsPerEdge;
@RateLimit(fieldName = "Edge uplink messages")
private String edgeUplinkMessagesRateLimits;
@RateLimit(fieldName = "Edge uplink messages per edge")
private String edgeUplinkMessagesRateLimitsPerEdge;
private int defaultStorageTtlDays;
@ -207,4 +236,43 @@ public class DefaultTenantProfileConfiguration implements TenantProfileConfigura
return maxRuleNodeExecutionsPerMessage;
}
@Deprecated(forRemoval = true, since = "4.0.2")
public void deduplicateRateLimitsConfigs() {
this.transportTenantMsgRateLimit = LimitedApiUtil.deduplicateByDuration(transportTenantMsgRateLimit);
this.transportTenantTelemetryMsgRateLimit = LimitedApiUtil.deduplicateByDuration(transportTenantTelemetryMsgRateLimit);
this.transportTenantTelemetryDataPointsRateLimit = LimitedApiUtil.deduplicateByDuration(transportTenantTelemetryDataPointsRateLimit);
this.transportDeviceMsgRateLimit = LimitedApiUtil.deduplicateByDuration(transportDeviceMsgRateLimit);
this.transportDeviceTelemetryMsgRateLimit = LimitedApiUtil.deduplicateByDuration(transportDeviceTelemetryMsgRateLimit);
this.transportDeviceTelemetryDataPointsRateLimit = LimitedApiUtil.deduplicateByDuration(transportDeviceTelemetryDataPointsRateLimit);
this.transportGatewayMsgRateLimit = LimitedApiUtil.deduplicateByDuration(transportGatewayMsgRateLimit);
this.transportGatewayTelemetryMsgRateLimit = LimitedApiUtil.deduplicateByDuration(transportGatewayTelemetryMsgRateLimit);
this.transportGatewayTelemetryDataPointsRateLimit = LimitedApiUtil.deduplicateByDuration(transportGatewayTelemetryDataPointsRateLimit);
this.transportGatewayDeviceMsgRateLimit = LimitedApiUtil.deduplicateByDuration(transportGatewayDeviceMsgRateLimit);
this.transportGatewayDeviceTelemetryMsgRateLimit = LimitedApiUtil.deduplicateByDuration(transportGatewayDeviceTelemetryMsgRateLimit);
this.transportGatewayDeviceTelemetryDataPointsRateLimit = LimitedApiUtil.deduplicateByDuration(transportGatewayDeviceTelemetryDataPointsRateLimit);
this.tenantEntityExportRateLimit = LimitedApiUtil.deduplicateByDuration(tenantEntityExportRateLimit);
this.tenantEntityImportRateLimit = LimitedApiUtil.deduplicateByDuration(tenantEntityImportRateLimit);
this.tenantNotificationRequestsRateLimit = LimitedApiUtil.deduplicateByDuration(tenantNotificationRequestsRateLimit);
this.tenantNotificationRequestsPerRuleRateLimit = LimitedApiUtil.deduplicateByDuration(tenantNotificationRequestsPerRuleRateLimit);
this.cassandraReadQueryTenantCoreRateLimits = LimitedApiUtil.deduplicateByDuration(cassandraReadQueryTenantCoreRateLimits);
this.cassandraWriteQueryTenantCoreRateLimits = LimitedApiUtil.deduplicateByDuration(cassandraWriteQueryTenantCoreRateLimits);
this.cassandraReadQueryTenantRuleEngineRateLimits = LimitedApiUtil.deduplicateByDuration(cassandraReadQueryTenantRuleEngineRateLimits);
this.cassandraWriteQueryTenantRuleEngineRateLimits = LimitedApiUtil.deduplicateByDuration(cassandraWriteQueryTenantRuleEngineRateLimits);
this.edgeEventRateLimits = LimitedApiUtil.deduplicateByDuration(edgeEventRateLimits);
this.edgeEventRateLimitsPerEdge = LimitedApiUtil.deduplicateByDuration(edgeEventRateLimitsPerEdge);
this.edgeUplinkMessagesRateLimits = LimitedApiUtil.deduplicateByDuration(edgeUplinkMessagesRateLimits);
this.edgeUplinkMessagesRateLimitsPerEdge = LimitedApiUtil.deduplicateByDuration(edgeUplinkMessagesRateLimitsPerEdge);
this.wsUpdatesPerSessionRateLimit = LimitedApiUtil.deduplicateByDuration(wsUpdatesPerSessionRateLimit);
this.tenantServerRestLimitsConfiguration = LimitedApiUtil.deduplicateByDuration(tenantServerRestLimitsConfiguration);
this.customerServerRestLimitsConfiguration = LimitedApiUtil.deduplicateByDuration(customerServerRestLimitsConfiguration);
}
}

39
common/data/src/main/java/org/thingsboard/server/common/data/validation/RateLimit.java Normal file
View File

@ -0,0 +1,39 @@
/**
* Copyright © 2016-2025 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.data.validation;
import jakarta.validation.Constraint;
import jakarta.validation.Payload;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
@Retention(RetentionPolicy.RUNTIME)
@Target({ElementType.FIELD, ElementType.METHOD})
@Constraint(validatedBy = {})
public @interface RateLimit {
String message() default "rate limit has duplicate 'Per seconds' configuration.";
String fieldName() default "";
Class<?>[] groups() default {};
Class<? extends Payload>[] payload() default {};
}

1
common/message/src/main/java/org/thingsboard/server/common/msg/tools/TbRateLimits.java
View File

@ -18,7 +18,6 @@ package org.thingsboard.server.common.msg.tools;
import io.github.bucket4j.Bandwidth;
import io.github.bucket4j.BandwidthBuilder;
import io.github.bucket4j.Bucket;
import io.github.bucket4j.Refill;
import io.github.bucket4j.local.LocalBucket;
import io.github.bucket4j.local.LocalBucketBuilder;
import lombok.Getter;

2
dao/src/main/java/org/thingsboard/server/dao/service/ConstraintValidator.java
View File

@ -33,6 +33,7 @@ import org.springframework.context.annotation.Configuration;
import org.springframework.validation.beanvalidation.LocalValidatorFactoryBean;
import org.thingsboard.server.common.data.validation.Length;
import org.thingsboard.server.common.data.validation.NoXss;
import org.thingsboard.server.common.data.validation.RateLimit;
import org.thingsboard.server.dao.exception.DataValidationException;
import java.util.Collection;
@ -103,6 +104,7 @@ public class ConstraintValidator {
ConstraintMapping constraintMapping = new DefaultConstraintMapping(null);
constraintMapping.constraintDefinition(NoXss.class).validatedBy(NoXssValidator.class);
constraintMapping.constraintDefinition(Length.class).validatedBy(StringLengthValidator.class);
constraintMapping.constraintDefinition(RateLimit.class).validatedBy(RateLimitValidator.class);
return constraintMapping;
}

32
dao/src/main/java/org/thingsboard/server/dao/service/RateLimitValidator.java Normal file
View File

@ -0,0 +1,32 @@
/**
* Copyright © 2016-2025 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.dao.service;
import jakarta.validation.ConstraintValidator;
import jakarta.validation.ConstraintValidatorContext;
import lombok.extern.slf4j.Slf4j;
import org.thingsboard.server.common.data.limit.LimitedApiUtil;
import org.thingsboard.server.common.data.validation.RateLimit;
@Slf4j
public class RateLimitValidator implements ConstraintValidator<RateLimit, String> {
@Override
public boolean isValid(String value, ConstraintValidatorContext constraintValidatorContext) {
return value == null || LimitedApiUtil.isValid(value);
}
}

6
dao/src/main/java/org/thingsboard/server/dao/service/validator/TenantProfileDataValidator.java
View File

@ -24,6 +24,7 @@ import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.queue.ProcessingStrategy;
import org.thingsboard.server.common.data.queue.SubmitStrategy;
import org.thingsboard.server.common.data.queue.SubmitStrategyType;
import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration;
import org.thingsboard.server.common.data.tenant.profile.TenantProfileQueueConfiguration;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.dao.service.DataValidator;
@ -51,9 +52,12 @@ public class TenantProfileDataValidator extends DataValidator<TenantProfile> {
if (tenantProfile.getProfileData() == null) {
throw new DataValidationException("Tenant profile data should be specified!");
}
if (tenantProfile.getProfileData().getConfiguration() == null) {
Optional<DefaultTenantProfileConfiguration> profileConfiguration = tenantProfile.getProfileConfiguration();
if (profileConfiguration.isEmpty()) {
throw new DataValidationException("Tenant profile data configuration should be specified!");
}
if (tenantProfile.isDefault()) {
TenantProfile defaultTenantProfile = tenantProfileService.findDefaultTenantProfile(tenantId);
if (defaultTenantProfile != null && !defaultTenantProfile.getId().equals(tenantProfile.getId())) {