dendi-wang/thingsboard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

updated transport http limits, added logging for initialized filters

Browse Source
This commit is contained in:
dashevchenko 2024-08-07 19:03:05 +03:00
parent 4169da4c08
commit 936bb82334
8 changed files with 30 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java
View File

@ -57,7 +57,7 @@ import org.thingsboard.server.service.security.auth.oauth2.HttpCookieOAuth2Autho
import org.thingsboard.server.service.security.auth.rest.RestAuthenticationProvider; import org.thingsboard.server.service.security.auth.rest.RestAuthenticationProvider;
import org.thingsboard.server.service.security.auth.rest.RestLoginProcessingFilter; import org.thingsboard.server.service.security.auth.rest.RestLoginProcessingFilter;
import org.thingsboard.server.service.security.auth.rest.RestPublicLoginProcessingFilter; import org.thingsboard.server.service.security.auth.rest.RestPublicLoginProcessingFilter;
import org.thingsboard.server.transport.http.config.RequestSizeFilter; import org.thingsboard.server.transport.http.config.PayloadSizeFilter;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
@ -84,7 +84,7 @@ public class ThingsboardSecurityConfiguration {
public static final String MAIL_OAUTH2_PROCESSING_ENTRY_POINT = "/api/admin/mail/oauth2/code"; public static final String MAIL_OAUTH2_PROCESSING_ENTRY_POINT = "/api/admin/mail/oauth2/code";
public static final String DEVICE_CONNECTIVITY_CERTIFICATE_DOWNLOAD_ENTRY_POINT = "/api/device-connectivity/mqtts/certificate/download"; public static final String DEVICE_CONNECTIVITY_CERTIFICATE_DOWNLOAD_ENTRY_POINT = "/api/device-connectivity/mqtts/certificate/download";
@Value("${server.http.max_payload_size:/api/image*/**=52428800;/api/**=16777216}") @Value("${server.http.max_payload_size:/api/image*/**=52428800;/api/resource/**=52428800;/api/**=16777216}")
private String maxPayloadSizeConfig; private String maxPayloadSizeConfig;
@Autowired @Autowired
@ -130,8 +130,8 @@ public class ThingsboardSecurityConfiguration {
private RateLimitProcessingFilter rateLimitProcessingFilter; private RateLimitProcessingFilter rateLimitProcessingFilter;
@Bean @Bean
protected RequestSizeFilter requestSizeFilter() { protected PayloadSizeFilter payloadSizeFilter() {
return new RequestSizeFilter(maxPayloadSizeConfig); return new PayloadSizeFilter(maxPayloadSizeConfig);
} }
@Bean @Bean
@ -234,7 +234,7 @@ public class ThingsboardSecurityConfiguration {
.addFilterBefore(buildRestPublicLoginProcessingFilter(), UsernamePasswordAuthenticationFilter.class) .addFilterBefore(buildRestPublicLoginProcessingFilter(), UsernamePasswordAuthenticationFilter.class)
.addFilterBefore(buildJwtTokenAuthenticationProcessingFilter(), UsernamePasswordAuthenticationFilter.class) .addFilterBefore(buildJwtTokenAuthenticationProcessingFilter(), UsernamePasswordAuthenticationFilter.class)
.addFilterBefore(buildRefreshTokenProcessingFilter(), UsernamePasswordAuthenticationFilter.class) .addFilterBefore(buildRefreshTokenProcessingFilter(), UsernamePasswordAuthenticationFilter.class)
.addFilterBefore(requestSizeFilter(), UsernamePasswordAuthenticationFilter.class) .addFilterBefore(payloadSizeFilter(), UsernamePasswordAuthenticationFilter.class)
.addFilterAfter(rateLimitProcessingFilter, UsernamePasswordAuthenticationFilter.class); .addFilterAfter(rateLimitProcessingFilter, UsernamePasswordAuthenticationFilter.class);
if (oauth2Configuration != null) { if (oauth2Configuration != null) {
http.oauth2Login(login -> login http.oauth2Login(login -> login

8
application/src/main/resources/thingsboard.yml
View File

@ -52,8 +52,8 @@ server:
key_password: "${SSL_KEY_PASSWORD:thingsboard}" key_password: "${SSL_KEY_PASSWORD:thingsboard}"
# HTTP settings # HTTP settings
http: http:
# Semi-colon-separated list of urlPattern=maxPayloadSize pairs that define max http request size for specified url pattern. # Semi-colon-separated list of urlPattern=maxPayloadSize pairs that define max http request size for specified url pattern. After first match all other will be skipped
max_payload_size: "${HTTP_MAX_PAYLOAD_SIZE_LIMIT_CONFIGURATION:/api/image*/**=52428800;/api/**=16777216}" max_payload_size: "${HTTP_MAX_PAYLOAD_SIZE_LIMIT_CONFIGURATION:/api/image*/**=52428800;/api/resource/**=52428800;/api/**=16777216}"
# HTTP/2 support (takes effect only if server SSL is enabled) # HTTP/2 support (takes effect only if server SSL is enabled)
http2: http2:
# Enable/disable HTTP/2 support # Enable/disable HTTP/2 support
@ -963,8 +963,8 @@ transport:
request_timeout: "${HTTP_REQUEST_TIMEOUT:60000}" request_timeout: "${HTTP_REQUEST_TIMEOUT:60000}"
# HTTP maximum request processing timeout in milliseconds # HTTP maximum request processing timeout in milliseconds
max_request_timeout: "${HTTP_MAX_REQUEST_TIMEOUT:300000}" max_request_timeout: "${HTTP_MAX_REQUEST_TIMEOUT:300000}"
# Semi-colon-separated list of urlPattern=maxPayloadSize pairs that define max http request size for specified url pattern. # Semi-colon-separated list of urlPattern=maxPayloadSize pairs that define max http request size for specified url pattern. After first match all other will be skipped
max_payload_size: "${HTTP_MAX_PAYLOAD_SIZE_LIMIT_CONFIGURATION:/api/v1/*/attributes=52428800;/api/v1/**=65536}" max_payload_size: "${HTTP_TRANSPORT_MAX_PAYLOAD_SIZE_LIMIT_CONFIGURATION:/api/v1/*/rpc/**=65536;/api/v1/**=52428800}"
# Local MQTT transport parameters # Local MQTT transport parameters
mqtt: mqtt:
# Enable/disable mqtt transport protocol. # Enable/disable mqtt transport protocol.

18
application/src/test/java/org/thingsboard/server/controller/ImageControllerTest.java
View File

File diff suppressed because one or more lines are too long

2
application/src/test/java/org/thingsboard/server/system/BaseHttpDeviceApiTest.java
View File

@ -42,7 +42,7 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.
*/ */
@TestPropertySource(properties = { @TestPropertySource(properties = {
"transport.http.enabled=true", "transport.http.enabled=true",
"transport.http.max_payload_size=/api/v1/*/attributes=20000;/api/v1/**=10000" "transport.http.max_payload_size=/api/v1/*/rpc/**=10000;/api/v1/**=20000"
}) })
public abstract class BaseHttpDeviceApiTest extends AbstractControllerTest { public abstract class BaseHttpDeviceApiTest extends AbstractControllerTest {

2
common/message/src/main/java/org/thingsboard/server/common/msg/tools/MaxPayloadSizeExceededException.java
View File

@ -23,7 +23,7 @@ public class MaxPayloadSizeExceededException extends RuntimeException {
private final long limit; private final long limit;
public MaxPayloadSizeExceededException(long limit) { public MaxPayloadSizeExceededException(long limit) {
super("Payload size exceeds the limit " + limit); super("Payload size exceeds the limit of " + limit + " bytes");
this.limit = limit; this.limit = limit;
} }
} }

9
common/transport/http/src/main/java/org/thingsboard/server/transport/http/config/RequestSizeFilter.java → common/transport/http/src/main/java/org/thingsboard/server/transport/http/config/PayloadSizeFilter.java
View File

@ -33,12 +33,12 @@ import java.util.Map;
@Slf4j @Slf4j
@RequiredArgsConstructor @RequiredArgsConstructor
public class RequestSizeFilter extends OncePerRequestFilter { public class PayloadSizeFilter extends OncePerRequestFilter {
private final Map<String, Long> limits = new LinkedHashMap<>(); private final Map<String, Long> limits = new LinkedHashMap<>();
private final AntPathMatcher pathMatcher = new AntPathMatcher(); private final AntPathMatcher pathMatcher = new AntPathMatcher();
public RequestSizeFilter(String limitsConfiguration) { public PayloadSizeFilter(String limitsConfiguration) {
for (String limit : limitsConfiguration.split(";")) { for (String limit : limitsConfiguration.split(";")) {
try { try {
String urlPathPattern = limit.split("=")[0]; String urlPathPattern = limit.split("=")[0];
@ -48,6 +48,7 @@ public class RequestSizeFilter extends OncePerRequestFilter {
throw new IllegalArgumentException("Failed to parse size limits configuration: " + limitsConfiguration); throw new IllegalArgumentException("Failed to parse size limits configuration: " + limitsConfiguration);
} }
} }
log.info("Initialized payload size filter with configuration: {}" , limitsConfiguration);
} }
@Override @Override
@ -65,9 +66,7 @@ public class RequestSizeFilter extends OncePerRequestFilter {
private boolean checkMaxPayloadSizeExceeded(HttpServletRequest request, HttpServletResponse response, long maxPayloadSize) throws IOException { private boolean checkMaxPayloadSizeExceeded(HttpServletRequest request, HttpServletResponse response, long maxPayloadSize) throws IOException {
if (request.getContentLength() > maxPayloadSize) { if (request.getContentLength() > maxPayloadSize) {
if (log.isDebugEnabled()) { log.info("[{}] [{}] Payload size {} exceeds the limit of {} bytes", request.getRemoteAddr(), request.getRequestURL(), request.getContentLength(), maxPayloadSize);
log.debug("Too large payload size. Url: {}, client ip: {}, content length: {}", request.getRequestURL(), request.getRemoteAddr(), request.getContentLength());
}
handleMaxPayloadSizeExceededException(response, new MaxPayloadSizeExceededException(maxPayloadSize)); handleMaxPayloadSizeExceededException(response, new MaxPayloadSizeExceededException(maxPayloadSize));
return true; return true;
} }

12
common/transport/http/src/main/java/org/thingsboard/server/transport/http/config/TransportSecurityConfiguration.java
View File

@ -16,8 +16,6 @@
package org.thingsboard.server.transport.http.config; package org.thingsboard.server.transport.http.config;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
import org.springframework.boot.autoconfigure.security.SecurityProperties;
import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Configuration;
import org.springframework.core.annotation.Order; import org.springframework.core.annotation.Order;
@ -32,17 +30,15 @@ import org.springframework.security.web.util.matcher.AntPathRequestMatcher;
@Configuration @Configuration
@EnableWebSecurity @EnableWebSecurity
@EnableMethodSecurity @EnableMethodSecurity
@Order(SecurityProperties.BASIC_AUTH_ORDER)
@ConditionalOnExpression("'${service.type:null}'=='tb-transport' || ('${service.type:null}'=='monolith' && '${transport.api_enabled:true}'=='true' && '${transport.http.enabled}'=='true')")
public class TransportSecurityConfiguration { public class TransportSecurityConfiguration {
public static final String DEVICE_API_ENTRY_POINT = "/api/v1/**"; public static final String DEVICE_API_ENTRY_POINT = "/api/v1/**";
@Value("${transport.http.max_payload_size:/api/v1/*/attributes=52428800;/api/v1/**=65536}") @Value("${transport.http.max_payload_size:/api/v1/*/rpc/**=65536;/api/v1/**=52428800}")
private String maxPayloadSizeConfig; private String maxPayloadSizeConfig;
@Bean @Bean
protected RequestSizeFilter httpTransportRequestSizeFilter() { protected PayloadSizeFilter transportPayloadSizeFilter() {
return new RequestSizeFilter(maxPayloadSizeConfig); return new PayloadSizeFilter(maxPayloadSizeConfig);
} }
@Bean @Bean
@ -55,7 +51,7 @@ public class TransportSecurityConfiguration {
.csrf(AbstractHttpConfigurer::disable) .csrf(AbstractHttpConfigurer::disable)
.authorizeHttpRequests(config -> config .authorizeHttpRequests(config -> config
.requestMatchers(DEVICE_API_ENTRY_POINT).permitAll()) .requestMatchers(DEVICE_API_ENTRY_POINT).permitAll())
.addFilterBefore(httpTransportRequestSizeFilter(), UsernamePasswordAuthenticationFilter.class); .addFilterBefore(transportPayloadSizeFilter(), UsernamePasswordAuthenticationFilter.class);
return http.build(); return http.build();
} }
} }

4
transport/http/src/main/resources/tb-http-transport.yml
View File

@ -170,8 +170,8 @@ transport:
request_timeout: "${HTTP_REQUEST_TIMEOUT:60000}" request_timeout: "${HTTP_REQUEST_TIMEOUT:60000}"
# HTTP maximum request processing timeout in milliseconds # HTTP maximum request processing timeout in milliseconds
max_request_timeout: "${HTTP_MAX_REQUEST_TIMEOUT:300000}" max_request_timeout: "${HTTP_MAX_REQUEST_TIMEOUT:300000}"
# Semi-colon-separated list of urlPattern=maxPayloadSize pairs that define max http request size for specified url pattern. # Semi-colon-separated list of urlPattern=maxPayloadSize pairs that define max http request size for specified url pattern. After first match all other will be skipped
max_payload_size: "${HTTP_MAX_PAYLOAD_SIZE_LIMIT_CONFIGURATION:/api/v1/*/attributes=52428800;/api/v1/**=65536}" max_payload_size: "${HTTP_TRANSPORT_MAX_PAYLOAD_SIZE_LIMIT_CONFIGURATION:/api/v1/*/rpc/**=65536;/api/v1/**=52428800}"
sessions: sessions:
# Session inactivity timeout is a global configuration parameter that defines how long the device transport session will be opened after the last message arrives from the device. # Session inactivity timeout is a global configuration parameter that defines how long the device transport session will be opened after the last message arrives from the device.
# The parameter value is in milliseconds. # The parameter value is in milliseconds.