dendi-wang/thingsboard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

configurable max payload size by url pattern

Browse Source
This commit is contained in:
dashevchenko 2024-08-01 16:10:39 +03:00
parent 02f7001102
commit 4169da4c08
10 changed files with 81 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java
View File

@ -84,10 +84,8 @@ public class ThingsboardSecurityConfiguration {
public static final String MAIL_OAUTH2_PROCESSING_ENTRY_POINT = "/api/admin/mail/oauth2/code";
public static final String DEVICE_CONNECTIVITY_CERTIFICATE_DOWNLOAD_ENTRY_POINT = "/api/device-connectivity/mqtts/certificate/download";
@Value("${server.http.max_payload_size:16777216}")
private int maxPayloadSize;
@Value("${transport.http.max_payload_size:65536}")
private int httpTransportMaxPayloadSize;
@Value("${server.http.max_payload_size:/api/image*/**=52428800;/api/**=16777216}")
private String maxPayloadSizeConfig;
@Autowired
private ThingsboardErrorResponseHandler restAccessDeniedHandler;
@ -131,14 +129,9 @@ public class ThingsboardSecurityConfiguration {
@Autowired
private RateLimitProcessingFilter rateLimitProcessingFilter;
@Bean
protected RequestSizeFilter httpTransportRequestSizeFilter() {
return new RequestSizeFilter(httpTransportMaxPayloadSize);
}
@Bean
protected RequestSizeFilter requestSizeFilter() {
return new RequestSizeFilter(maxPayloadSize);
return new RequestSizeFilter(maxPayloadSizeConfig);
}
@Bean
@ -217,20 +210,6 @@ public class ThingsboardSecurityConfiguration {
}
@Bean
@Order(1)
SecurityFilterChain httpTransportFilterChain(HttpSecurity http) throws Exception {
http
.securityMatchers(matchers -> matchers.requestMatchers(DEVICE_API_ENTRY_POINT))
.cors(cors -> {})
.csrf(AbstractHttpConfigurer::disable)
.authorizeHttpRequests(config -> config
.requestMatchers(DEVICE_API_ENTRY_POINT).permitAll())
.addFilterBefore(httpTransportRequestSizeFilter(), UsernamePasswordAuthenticationFilter.class);
return http.build();
}
@Bean
@Order(2)
SecurityFilterChain filterChain(HttpSecurity http) throws Exception {
http.headers(headers -> headers
.cacheControl(config -> {})

8
application/src/main/resources/thingsboard.yml
View File

@ -52,8 +52,8 @@ server:
key_password: "${SSL_KEY_PASSWORD:thingsboard}"
# HTTP settings
http:
# Maximum request size
max_payload_size: "${HTTP_MAX_PAYLOAD_SIZE:16777216}" # max payload size in bytes
# Semi-colon-separated list of urlPattern=maxPayloadSize pairs that define max http request size for specified url pattern.
max_payload_size: "${HTTP_MAX_PAYLOAD_SIZE_LIMIT_CONFIGURATION:/api/image*/**=52428800;/api/**=16777216}"
# HTTP/2 support (takes effect only if server SSL is enabled)
http2:
# Enable/disable HTTP/2 support
@ -963,8 +963,8 @@ transport:
request_timeout: "${HTTP_REQUEST_TIMEOUT:60000}"
# HTTP maximum request processing timeout in milliseconds
max_request_timeout: "${HTTP_MAX_REQUEST_TIMEOUT:300000}"
# HTTP maximum request size
max_payload_size: "${HTTP_MAX_PAYLOAD_SIZE:65536}" # max payload size in bytes
# Semi-colon-separated list of urlPattern=maxPayloadSize pairs that define max http request size for specified url pattern.
max_payload_size: "${HTTP_MAX_PAYLOAD_SIZE_LIMIT_CONFIGURATION:/api/v1/*/attributes=52428800;/api/v1/**=65536}"
# Local MQTT transport parameters
mqtt:
# Enable/disable mqtt transport protocol.

13
application/src/test/java/org/thingsboard/server/controller/ImageControllerTest.java
View File

File diff suppressed because one or more lines are too long

2
application/src/test/java/org/thingsboard/server/controller/RpcControllerTest.java
View File

@ -40,7 +40,7 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.
@DaoSqlTest
@TestPropertySource(properties = {
"server.http.max_payload_size=10000"
"server.http.max_payload_size=/api/image*/**=10;/api/**=10000"
})
public class RpcControllerTest extends AbstractControllerTest {

22
application/src/test/java/org/thingsboard/server/system/BaseHttpDeviceApiTest.java
View File

@ -42,7 +42,7 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.
*/
@TestPropertySource(properties = {
"transport.http.enabled=true",
"transport.http.max_payload_size=10000"
"transport.http.max_payload_size=/api/v1/*/attributes=20000;/api/v1/**=10000"
})
public abstract class BaseHttpDeviceApiTest extends AbstractControllerTest {
@ -80,13 +80,13 @@ public abstract class BaseHttpDeviceApiTest extends AbstractControllerTest {
@Test
public void testReplyToCommandWithLargeResponse() throws Exception {
String errorResponse = doPost("/api/v1/" + deviceCredentials.getCredentialsId() + "/rpc/5",
JacksonUtil.toString(createRpcResponsePayload(10001)),
JacksonUtil.toString(createJsonPayloadOfSize(10001)),
String.class,
status().isPayloadTooLarge());
assertThat(errorResponse).contains("Payload size exceeds the limit");
doPost("/api/v1/" + deviceCredentials.getCredentialsId() + "/rpc/5",
JacksonUtil.toString(createRpcResponsePayload(10000)),
JacksonUtil.toString(createJsonPayloadOfSize(10000)),
String.class,
status().isOk());
}
@ -105,7 +105,21 @@ public abstract class BaseHttpDeviceApiTest extends AbstractControllerTest {
status().isOk());
}
private String createRpcResponsePayload(int size) {
@Test
public void testPostLargeAttribute() throws Exception {
String errorResponse = doPost("/api/v1/" + deviceCredentials.getCredentialsId() + "/attributes",
JacksonUtil.toString(createJsonPayloadOfSize(20001)),
String.class,
status().isPayloadTooLarge());
assertThat(errorResponse).contains("Payload size exceeds the limit");
doPost("/api/v1/" + deviceCredentials.getCredentialsId() + "/attributes",
JacksonUtil.toString(createJsonPayloadOfSize(20000)),
String.class,
status().isOk());
}
private String createJsonPayloadOfSize(int size) {
String value = "a".repeat(size - 19);
return "{\"result\":\"" + value + "\"}";
}

4
common/message/src/main/java/org/thingsboard/server/common/msg/tools/MaxPayloadSizeExceededException.java
View File

@ -20,9 +20,9 @@ import lombok.Getter;
public class MaxPayloadSizeExceededException extends RuntimeException {
@Getter
private final int limit;
private final long limit;
public MaxPayloadSizeExceededException(int limit) {
public MaxPayloadSizeExceededException(long limit) {
super("Payload size exceeds the limit " + limit);
this.limit = limit;
}

20
common/transport/http/src/main/java/org/thingsboard/server/transport/http/DeviceApiController.java
View File

@ -24,11 +24,9 @@ import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.responses.ApiResponse;
import io.swagger.v3.oas.annotations.responses.ApiResponses;
import jakarta.servlet.http.HttpServletRequest;
import jakarta.servlet.http.HttpServletResponse;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
import org.springframework.core.io.ByteArrayResource;
import org.springframework.http.HttpHeaders;
@ -36,7 +34,6 @@ import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.http.converter.HttpMessageNotReadableException;
import org.springframework.web.bind.annotation.ExceptionHandler;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
@ -44,7 +41,6 @@ import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.context.request.async.DeferredResult;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.common.adaptor.JsonConverter;
import org.thingsboard.server.common.data.DataConstants;
import org.thingsboard.server.common.data.DeviceTransportType;
@ -53,7 +49,6 @@ import org.thingsboard.server.common.data.TbTransportService;
import org.thingsboard.server.common.data.id.DeviceId;
import org.thingsboard.server.common.data.ota.OtaPackageType;
import org.thingsboard.server.common.data.rpc.RpcStatus;
import org.thingsboard.server.common.msg.tools.MaxPayloadSizeExceededException;
import org.thingsboard.server.common.transport.SessionMsgListener;
import org.thingsboard.server.common.transport.TransportContext;
import org.thingsboard.server.common.transport.TransportService;
@ -76,7 +71,6 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToServerRpcRequestMs
import org.thingsboard.server.gen.transport.TransportProtos.ToServerRpcResponseMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceTokenRequestMsg;
import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import java.util.UUID;
@ -626,20 +620,6 @@ public class DeviceApiController implements TbTransportService {
}
@ExceptionHandler(MaxPayloadSizeExceededException.class)
public void handle(MaxPayloadSizeExceededException exception, HttpServletRequest request, HttpServletResponse response) {
log.debug("Too large payload size. Url: {}, client ip: {}, content length: {}", request.getRequestURL(),
request.getRemoteAddr(), request.getContentLength());
if (!response.isCommitted()) {
try {
response.setStatus(HttpStatus.PAYLOAD_TOO_LARGE.value());
JacksonUtil.writeValue(response.getWriter(), exception.getMessage());
} catch (IOException e) {
log.error("Can't handle exception", e);
}
}
}
private static MediaType parseMediaType(String contentType) {
try {
return MediaType.parseMediaType(contentType);

34
common/transport/http/src/main/java/org/thingsboard/server/transport/http/config/RequestSizeFilter.java
View File

@ -22,28 +22,56 @@ import jakarta.servlet.http.HttpServletResponse;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.http.HttpStatus;
import org.springframework.util.AntPathMatcher;
import org.springframework.web.filter.OncePerRequestFilter;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.common.msg.tools.MaxPayloadSizeExceededException;
import java.io.IOException;
import java.util.LinkedHashMap;
import java.util.Map;
@Slf4j
@RequiredArgsConstructor
public class RequestSizeFilter extends OncePerRequestFilter {
private final int maxPayloadSize;
private final Map<String, Long> limits = new LinkedHashMap<>();
private final AntPathMatcher pathMatcher = new AntPathMatcher();
public RequestSizeFilter(String limitsConfiguration) {
for (String limit : limitsConfiguration.split(";")) {
try {
String urlPathPattern = limit.split("=")[0];
long maxPayloadSize = Long.parseLong(limit.split("=")[1]);
limits.put(urlPathPattern, maxPayloadSize);
} catch (Exception e) {
throw new IllegalArgumentException("Failed to parse size limits configuration: " + limitsConfiguration);
}
}
}
@Override
public void doFilterInternal(HttpServletRequest request, HttpServletResponse response, FilterChain chain) throws IOException, ServletException {
for (String url : limits.keySet()) {
if (pathMatcher.match(url, request.getRequestURI())) {
if (checkMaxPayloadSizeExceeded(request, response, limits.get(url))) {
return;
}
break;
}
}
chain.doFilter(request, response);
}
private boolean checkMaxPayloadSizeExceeded(HttpServletRequest request, HttpServletResponse response, long maxPayloadSize) throws IOException {
if (request.getContentLength() > maxPayloadSize) {
if (log.isDebugEnabled()) {
log.debug("Too large payload size. Url: {}, client ip: {}, content length: {}", request.getRequestURL(), request.getRemoteAddr(), request.getContentLength());
}
handleMaxPayloadSizeExceededException(response, new MaxPayloadSizeExceededException(maxPayloadSize));
return;
return true;
}
chain.doFilter(request, response);
return false;
}
@Override

12
common/transport/http/src/main/java/org/thingsboard/server/transport/http/config/TransportSecurityConfiguration.java
View File

@ -27,27 +27,29 @@ import org.springframework.security.config.annotation.web.configuration.EnableWe
import org.springframework.security.config.annotation.web.configurers.AbstractHttpConfigurer;
import org.springframework.security.web.SecurityFilterChain;
import org.springframework.security.web.authentication.UsernamePasswordAuthenticationFilter;
import org.springframework.security.web.util.matcher.AntPathRequestMatcher;
@Configuration
@EnableWebSecurity
@EnableMethodSecurity
@Order(SecurityProperties.BASIC_AUTH_ORDER)
@ConditionalOnExpression("('${service.type:null}'=='tb-transport')")
@ConditionalOnExpression("'${service.type:null}'=='tb-transport' || ('${service.type:null}'=='monolith' && '${transport.api_enabled:true}'=='true' && '${transport.http.enabled}'=='true')")
public class TransportSecurityConfiguration {
public static final String DEVICE_API_ENTRY_POINT = "/api/v1/**";
@Value("${transport.http.max_payload_size:65536}")
private int maxPayloadSize;
@Value("${transport.http.max_payload_size:/api/v1/*/attributes=52428800;/api/v1/**=65536}")
private String maxPayloadSizeConfig;
@Bean
protected RequestSizeFilter httpTransportRequestSizeFilter() {
return new RequestSizeFilter(maxPayloadSize);
return new RequestSizeFilter(maxPayloadSizeConfig);
}
@Bean
@Order(1)
SecurityFilterChain httpTransportFilterChain(HttpSecurity http) throws Exception {
http
.securityMatchers(matchers -> matchers.requestMatchers(DEVICE_API_ENTRY_POINT))
.securityMatcher(AntPathRequestMatcher.antMatcher(DEVICE_API_ENTRY_POINT))
.cors(cors -> {
})
.csrf(AbstractHttpConfigurer::disable)

4
transport/http/src/main/resources/tb-http-transport.yml
View File

@ -170,8 +170,8 @@ transport:
request_timeout: "${HTTP_REQUEST_TIMEOUT:60000}"
# HTTP maximum request processing timeout in milliseconds
max_request_timeout: "${HTTP_MAX_REQUEST_TIMEOUT:300000}"
# HTTP maximum request size
max_payload_size: "${HTTP_MAX_PAYLOAD_SIZE:65536}" # max payload size in bytes
# Semi-colon-separated list of urlPattern=maxPayloadSize pairs that define max http request size for specified url pattern.
max_payload_size: "${HTTP_MAX_PAYLOAD_SIZE_LIMIT_CONFIGURATION:/api/v1/*/attributes=52428800;/api/v1/**=65536}"
sessions:
# Session inactivity timeout is a global configuration parameter that defines how long the device transport session will be opened after the last message arrives from the device.
# The parameter value is in milliseconds.