Merge pull request #11330 from dashevchenko/http_transport_large_request_fix

Payload size filter for all requests
This commit is contained in:
Andrew Shvayka 2024-08-30 18:05:44 +03:00 committed by GitHub
commit 04e4116f30
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
12 changed files with 209 additions and 124 deletions

View File

@ -1,78 +0,0 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.config;
import jakarta.servlet.FilterChain;
import jakarta.servlet.ServletException;
import jakarta.servlet.http.HttpServletRequest;
import jakarta.servlet.http.HttpServletResponse;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.springframework.util.AntPathMatcher;
import org.springframework.web.filter.OncePerRequestFilter;
import org.thingsboard.server.common.msg.tools.MaxPayloadSizeExceededException;
import org.thingsboard.server.exception.ThingsboardErrorResponseHandler;
import java.io.IOException;
import java.util.List;
@Slf4j
@Component
@RequiredArgsConstructor
public class RequestSizeFilter extends OncePerRequestFilter {
private final List<String> urls = List.of("/api/plugins/rpc/**", "/api/rpc/**");
private final AntPathMatcher pathMatcher = new AntPathMatcher();
private final ThingsboardErrorResponseHandler errorResponseHandler;
@Value("${transport.http.max_payload_size:65536}")
private int maxPayloadSize;
@Override
public void doFilterInternal(HttpServletRequest request, HttpServletResponse response, FilterChain chain) throws IOException, ServletException {
if (request.getContentLength() > maxPayloadSize) {
if (log.isDebugEnabled()) {
log.debug("Too large payload size. Url: {}, client ip: {}, content length: {}", request.getRequestURL(),
request.getRemoteAddr(), request.getContentLength());
}
errorResponseHandler.handle(new MaxPayloadSizeExceededException(), response);
return;
}
chain.doFilter(request, response);
}
@Override
protected boolean shouldNotFilter(HttpServletRequest request) {
for (String url : urls) {
if (pathMatcher.match(url, request.getRequestURI())) {
return false;
}
}
return true;
}
@Override
protected boolean shouldNotFilterAsyncDispatch() {
return false;
}
@Override
protected boolean shouldNotFilterErrorDispatch() {
return false;
}
}

View File

@ -17,6 +17,7 @@ package org.thingsboard.server.config;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
import org.springframework.boot.autoconfigure.security.SecurityProperties;
import org.springframework.boot.web.servlet.FilterRegistrationBean;
@ -56,6 +57,7 @@ import org.thingsboard.server.service.security.auth.oauth2.HttpCookieOAuth2Autho
import org.thingsboard.server.service.security.auth.rest.RestAuthenticationProvider;
import org.thingsboard.server.service.security.auth.rest.RestLoginProcessingFilter;
import org.thingsboard.server.service.security.auth.rest.RestPublicLoginProcessingFilter;
import org.thingsboard.server.transport.http.config.PayloadSizeFilter;
import java.util.ArrayList;
import java.util.Arrays;
@ -82,6 +84,9 @@ public class ThingsboardSecurityConfiguration {
public static final String MAIL_OAUTH2_PROCESSING_ENTRY_POINT = "/api/admin/mail/oauth2/code";
public static final String DEVICE_CONNECTIVITY_CERTIFICATE_DOWNLOAD_ENTRY_POINT = "/api/device-connectivity/mqtts/certificate/download";
@Value("${server.http.max_payload_size:/api/image*/**=52428800;/api/resource/**=52428800;/api/**=16777216}")
private String maxPayloadSizeConfig;
@Autowired
private ThingsboardErrorResponseHandler restAccessDeniedHandler;
@ -124,8 +129,10 @@ public class ThingsboardSecurityConfiguration {
@Autowired
private RateLimitProcessingFilter rateLimitProcessingFilter;
@Autowired
private RequestSizeFilter requestSizeFilter;
@Bean
protected PayloadSizeFilter payloadSizeFilter() {
return new PayloadSizeFilter(maxPayloadSizeConfig);
}
@Bean
protected FilterRegistrationBean<ShallowEtagHeaderFilter> buildEtagFilter() throws Exception {
@ -214,7 +221,6 @@ public class ThingsboardSecurityConfiguration {
.authorizeHttpRequests(config -> config
.requestMatchers(NON_TOKEN_BASED_AUTH_ENTRY_POINTS).permitAll() // static resources, user activation and password reset end-points (webjars included)
.requestMatchers(
DEVICE_API_ENTRY_POINT, // Device HTTP Transport API
FORM_BASED_LOGIN_ENTRY_POINT, // Login end-point
PUBLIC_LOGIN_ENTRY_POINT, // Public login end-point
TOKEN_REFRESH_ENTRY_POINT, // Token refresh end-point
@ -228,7 +234,7 @@ public class ThingsboardSecurityConfiguration {
.addFilterBefore(buildRestPublicLoginProcessingFilter(), UsernamePasswordAuthenticationFilter.class)
.addFilterBefore(buildJwtTokenAuthenticationProcessingFilter(), UsernamePasswordAuthenticationFilter.class)
.addFilterBefore(buildRefreshTokenProcessingFilter(), UsernamePasswordAuthenticationFilter.class)
.addFilterBefore(requestSizeFilter, UsernamePasswordAuthenticationFilter.class)
.addFilterBefore(payloadSizeFilter(), UsernamePasswordAuthenticationFilter.class)
.addFilterAfter(rateLimitProcessingFilter, UsernamePasswordAuthenticationFilter.class);
if (oauth2Configuration != null) {
http.oauth2Login(login -> login

View File

@ -50,6 +50,10 @@ server:
key_alias: "${SSL_KEY_ALIAS:tomcat}"
# Password used to access the key
key_password: "${SSL_KEY_PASSWORD:thingsboard}"
# HTTP settings
http:
# Semi-colon-separated list of urlPattern=maxPayloadSize pairs that define max http request size for specified url pattern. After first match all other will be skipped
max_payload_size: "${HTTP_MAX_PAYLOAD_SIZE_LIMIT_CONFIGURATION:/api/image*/**=52428800;/api/resource/**=52428800;/api/**=16777216}"
# HTTP/2 support (takes effect only if server SSL is enabled)
http2:
# Enable/disable HTTP/2 support
@ -995,8 +999,8 @@ transport:
request_timeout: "${HTTP_REQUEST_TIMEOUT:60000}"
# HTTP maximum request processing timeout in milliseconds
max_request_timeout: "${HTTP_MAX_REQUEST_TIMEOUT:300000}"
# Maximum request size
max_payload_size: "${HTTP_MAX_PAYLOAD_SIZE:65536}" # max payload size in bytes
# Semi-colon-separated list of urlPattern=maxPayloadSize pairs that define max http request size for specified url pattern. After first match all other will be skipped
max_payload_size: "${HTTP_TRANSPORT_MAX_PAYLOAD_SIZE_LIMIT_CONFIGURATION:/api/v1/*/rpc/**=65536;/api/v1/**=52428800}"
# Local MQTT transport parameters
mqtt:
# Enable/disable mqtt transport protocol.

File diff suppressed because one or more lines are too long

View File

@ -40,7 +40,7 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.
@DaoSqlTest
@TestPropertySource(properties = {
"transport.http.max_payload_size=10000"
"server.http.max_payload_size=/api/image*/**=10;/api/**=10000"
})
public class RpcControllerTest extends AbstractControllerTest {

View File

@ -42,7 +42,7 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.
*/
@TestPropertySource(properties = {
"transport.http.enabled=true",
"transport.http.max_payload_size=10000"
"transport.http.max_payload_size=/api/v1/*/rpc/**=10000;/api/v1/**=20000"
})
public abstract class BaseHttpDeviceApiTest extends AbstractControllerTest {
@ -80,13 +80,13 @@ public abstract class BaseHttpDeviceApiTest extends AbstractControllerTest {
@Test
public void testReplyToCommandWithLargeResponse() throws Exception {
String errorResponse = doPost("/api/v1/" + deviceCredentials.getCredentialsId() + "/rpc/5",
JacksonUtil.toString(createRpcResponsePayload(10001)),
JacksonUtil.toString(createJsonPayloadOfSize(10001)),
String.class,
status().isPayloadTooLarge());
assertThat(errorResponse).contains("Payload size exceeds the limit");
doPost("/api/v1/" + deviceCredentials.getCredentialsId() + "/rpc/5",
JacksonUtil.toString(createRpcResponsePayload(10000)),
JacksonUtil.toString(createJsonPayloadOfSize(10000)),
String.class,
status().isOk());
}
@ -105,7 +105,21 @@ public abstract class BaseHttpDeviceApiTest extends AbstractControllerTest {
status().isOk());
}
private String createRpcResponsePayload(int size) {
@Test
public void testPostLargeAttribute() throws Exception {
String errorResponse = doPost("/api/v1/" + deviceCredentials.getCredentialsId() + "/attributes",
JacksonUtil.toString(createJsonPayloadOfSize(20001)),
String.class,
status().isPayloadTooLarge());
assertThat(errorResponse).contains("Payload size exceeds the limit");
doPost("/api/v1/" + deviceCredentials.getCredentialsId() + "/attributes",
JacksonUtil.toString(createJsonPayloadOfSize(20000)),
String.class,
status().isOk());
}
private String createJsonPayloadOfSize(int size) {
String value = "a".repeat(size - 19);
return "{\"result\":\"" + value + "\"}";
}

View File

@ -15,10 +15,16 @@
*/
package org.thingsboard.server.common.msg.tools;
import lombok.Getter;
public class MaxPayloadSizeExceededException extends RuntimeException {
public MaxPayloadSizeExceededException() {
super("Payload size exceeds the limit");
@Getter
private final long limit;
public MaxPayloadSizeExceededException(long limit) {
super("Payload size exceeds the limit of " + limit + " bytes");
this.limit = limit;
}
}

View File

@ -45,6 +45,10 @@
<artifactId>spring-boot-starter-web</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-security</artifactId>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>

View File

@ -24,11 +24,9 @@ import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.responses.ApiResponse;
import io.swagger.v3.oas.annotations.responses.ApiResponses;
import jakarta.servlet.http.HttpServletRequest;
import jakarta.servlet.http.HttpServletResponse;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
import org.springframework.core.io.ByteArrayResource;
import org.springframework.http.HttpHeaders;
@ -36,7 +34,6 @@ import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.http.converter.HttpMessageNotReadableException;
import org.springframework.web.bind.annotation.ExceptionHandler;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
@ -44,7 +41,6 @@ import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.context.request.async.DeferredResult;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.common.adaptor.JsonConverter;
import org.thingsboard.server.common.data.DataConstants;
import org.thingsboard.server.common.data.DeviceTransportType;
@ -53,7 +49,6 @@ import org.thingsboard.server.common.data.TbTransportService;
import org.thingsboard.server.common.data.id.DeviceId;
import org.thingsboard.server.common.data.ota.OtaPackageType;
import org.thingsboard.server.common.data.rpc.RpcStatus;
import org.thingsboard.server.common.msg.tools.MaxPayloadSizeExceededException;
import org.thingsboard.server.common.transport.SessionMsgListener;
import org.thingsboard.server.common.transport.TransportContext;
import org.thingsboard.server.common.transport.TransportService;
@ -76,7 +71,6 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToServerRpcRequestMs
import org.thingsboard.server.gen.transport.TransportProtos.ToServerRpcResponseMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceTokenRequestMsg;
import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import java.util.UUID;
@ -136,9 +130,6 @@ public class DeviceApiController implements TbTransportService {
private static final String ACCESS_TOKEN_PARAM_DESCRIPTION = "Your device access token.";
@Value("${transport.http.max_payload_size:65536}")
private int maxPayloadSize;
@Autowired
private HttpTransportContext transportContext;
@ -289,7 +280,6 @@ public class DeviceApiController implements TbTransportService {
@PathVariable("requestId") Integer requestId,
@io.swagger.v3.oas.annotations.parameters.RequestBody(description = "Reply to the RPC request, JSON. For example: {\"status\":\"success\"}", required = true)
@RequestBody String json, HttpServletRequest httpServletRequest) {
checkPayloadSize(httpServletRequest);
DeferredResult<ResponseEntity> responseWriter = new DeferredResult<ResponseEntity>();
transportContext.getTransportService().process(DeviceTransportType.DEFAULT, ValidateDeviceTokenRequestMsg.newBuilder().setToken(deviceToken).build(),
new DeviceAuthCallback(transportContext, responseWriter, sessionInfo -> {
@ -320,7 +310,6 @@ public class DeviceApiController implements TbTransportService {
@PathVariable("deviceToken") String deviceToken,
@io.swagger.v3.oas.annotations.parameters.RequestBody(description = "The RPC request JSON", required = true)
@RequestBody String json, HttpServletRequest httpServletRequest) {
checkPayloadSize(httpServletRequest);
DeferredResult<ResponseEntity> responseWriter = new DeferredResult<ResponseEntity>();
transportContext.getTransportService().process(DeviceTransportType.DEFAULT, ValidateDeviceTokenRequestMsg.newBuilder().setToken(deviceToken).build(),
new DeviceAuthCallback(transportContext, responseWriter, sessionInfo -> {
@ -443,12 +432,6 @@ public class DeviceApiController implements TbTransportService {
return responseWriter;
}
private void checkPayloadSize(HttpServletRequest httpServletRequest) {
if (httpServletRequest.getContentLength() > maxPayloadSize) {
throw new MaxPayloadSizeExceededException();
}
}
private DeferredResult<ResponseEntity> getOtaPackageCallback(String deviceToken, String title, String version, int size, int chunk, OtaPackageType firmwareType) {
DeferredResult<ResponseEntity> responseWriter = new DeferredResult<>();
transportContext.getTransportService().process(DeviceTransportType.DEFAULT, ValidateDeviceTokenRequestMsg.newBuilder().setToken(deviceToken).build(),
@ -637,20 +620,6 @@ public class DeviceApiController implements TbTransportService {
}
@ExceptionHandler(MaxPayloadSizeExceededException.class)
public void handle(MaxPayloadSizeExceededException exception, HttpServletRequest request, HttpServletResponse response) {
log.debug("Too large payload size. Url: {}, client ip: {}, content length: {}", request.getRequestURL(),
request.getRemoteAddr(), request.getContentLength());
if (!response.isCommitted()) {
try {
response.setStatus(HttpStatus.PAYLOAD_TOO_LARGE.value());
JacksonUtil.writeValue(response.getWriter(), exception.getMessage());
} catch (IOException e) {
log.error("Can't handle exception", e);
}
}
}
private static MediaType parseMediaType(String contentType) {
try {
return MediaType.parseMediaType(contentType);

View File

@ -0,0 +1,90 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.transport.http.config;
import jakarta.servlet.FilterChain;
import jakarta.servlet.ServletException;
import jakarta.servlet.http.HttpServletRequest;
import jakarta.servlet.http.HttpServletResponse;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.http.HttpStatus;
import org.springframework.util.AntPathMatcher;
import org.springframework.web.filter.OncePerRequestFilter;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.common.msg.tools.MaxPayloadSizeExceededException;
import java.io.IOException;
import java.util.LinkedHashMap;
import java.util.Map;
@Slf4j
@RequiredArgsConstructor
public class PayloadSizeFilter extends OncePerRequestFilter {
private final Map<String, Long> limits = new LinkedHashMap<>();
private final AntPathMatcher pathMatcher = new AntPathMatcher();
public PayloadSizeFilter(String limitsConfiguration) {
for (String limit : limitsConfiguration.split(";")) {
try {
String urlPathPattern = limit.split("=")[0];
long maxPayloadSize = Long.parseLong(limit.split("=")[1]);
limits.put(urlPathPattern, maxPayloadSize);
} catch (Exception e) {
throw new IllegalArgumentException("Failed to parse size limits configuration: " + limitsConfiguration);
}
}
log.info("Initialized payload size filter with configuration: {}" , limitsConfiguration);
}
@Override
public void doFilterInternal(HttpServletRequest request, HttpServletResponse response, FilterChain chain) throws IOException, ServletException {
for (String url : limits.keySet()) {
if (pathMatcher.match(url, request.getRequestURI())) {
if (checkMaxPayloadSizeExceeded(request, response, limits.get(url))) {
return;
}
break;
}
}
chain.doFilter(request, response);
}
private boolean checkMaxPayloadSizeExceeded(HttpServletRequest request, HttpServletResponse response, long maxPayloadSize) throws IOException {
if (request.getContentLength() > maxPayloadSize) {
log.info("[{}] [{}] Payload size {} exceeds the limit of {} bytes", request.getRemoteAddr(), request.getRequestURL(), request.getContentLength(), maxPayloadSize);
handleMaxPayloadSizeExceededException(response, new MaxPayloadSizeExceededException(maxPayloadSize));
return true;
}
return false;
}
@Override
protected boolean shouldNotFilterAsyncDispatch() {
return false;
}
@Override
protected boolean shouldNotFilterErrorDispatch() {
return false;
}
private void handleMaxPayloadSizeExceededException(HttpServletResponse response, MaxPayloadSizeExceededException exception) throws IOException {
response.setStatus(HttpStatus.PAYLOAD_TOO_LARGE.value());
JacksonUtil.writeValue(response.getWriter(), exception.getMessage());
}
}

View File

@ -0,0 +1,57 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.transport.http.config;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.annotation.Order;
import org.springframework.security.config.annotation.method.configuration.EnableMethodSecurity;
import org.springframework.security.config.annotation.web.builders.HttpSecurity;
import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity;
import org.springframework.security.config.annotation.web.configurers.AbstractHttpConfigurer;
import org.springframework.security.web.SecurityFilterChain;
import org.springframework.security.web.authentication.UsernamePasswordAuthenticationFilter;
import org.springframework.security.web.util.matcher.AntPathRequestMatcher;
@Configuration
@EnableWebSecurity
@EnableMethodSecurity
public class TransportSecurityConfiguration {
public static final String DEVICE_API_ENTRY_POINT = "/api/v1/**";
@Value("${transport.http.max_payload_size:/api/v1/*/rpc/**=65536;/api/v1/**=52428800}")
private String maxPayloadSizeConfig;
@Bean
protected PayloadSizeFilter transportPayloadSizeFilter() {
return new PayloadSizeFilter(maxPayloadSizeConfig);
}
@Bean
@Order(1)
SecurityFilterChain httpTransportFilterChain(HttpSecurity http) throws Exception {
http
.securityMatcher(AntPathRequestMatcher.antMatcher(DEVICE_API_ENTRY_POINT))
.cors(cors -> {
})
.csrf(AbstractHttpConfigurer::disable)
.authorizeHttpRequests(config -> config
.requestMatchers(DEVICE_API_ENTRY_POINT).permitAll())
.addFilterBefore(transportPayloadSizeFilter(), UsernamePasswordAuthenticationFilter.class);
return http.build();
}
}

View File

@ -170,8 +170,8 @@ transport:
request_timeout: "${HTTP_REQUEST_TIMEOUT:60000}"
# HTTP maximum request processing timeout in milliseconds
max_request_timeout: "${HTTP_MAX_REQUEST_TIMEOUT:300000}"
# Maximum request size
max_payload_size: "${HTTP_MAX_PAYLOAD_SIZE:65536}" # max payload size in bytes
# Semi-colon-separated list of urlPattern=maxPayloadSize pairs that define max http request size for specified url pattern. After first match all other will be skipped
max_payload_size: "${HTTP_TRANSPORT_MAX_PAYLOAD_SIZE_LIMIT_CONFIGURATION:/api/v1/*/rpc/**=65536;/api/v1/**=52428800}"
sessions:
# Session inactivity timeout is a global configuration parameter that defines how long the device transport session will be opened after the last message arrives from the device.
# The parameter value is in milliseconds.