configurable max payload size by url pattern

application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java

@@ -84,10 +84,8 @@ public class ThingsboardSecurityConfiguration {
     public static final String MAIL_OAUTH2_PROCESSING_ENTRY_POINT = "/api/admin/mail/oauth2/code";
     public static final String DEVICE_CONNECTIVITY_CERTIFICATE_DOWNLOAD_ENTRY_POINT = "/api/device-connectivity/mqtts/certificate/download";
 
-    @Value("${server.http.max_payload_size:16777216}")
+    @Value("${server.http.max_payload_size:/api/image*/**=52428800;/api/**=16777216}")
-    private int maxPayloadSize;
+    private String maxPayloadSizeConfig;
-    @Value("${transport.http.max_payload_size:65536}")
-    private int httpTransportMaxPayloadSize;
 
     @Autowired
     private ThingsboardErrorResponseHandler restAccessDeniedHandler;
@@ -131,14 +129,9 @@ public class ThingsboardSecurityConfiguration {
     @Autowired
     private RateLimitProcessingFilter rateLimitProcessingFilter;
 
-    @Bean
-    protected RequestSizeFilter httpTransportRequestSizeFilter() {
-        return new RequestSizeFilter(httpTransportMaxPayloadSize);
-    }
-
     @Bean
     protected RequestSizeFilter requestSizeFilter() {
-        return new RequestSizeFilter(maxPayloadSize);
+        return new RequestSizeFilter(maxPayloadSizeConfig);
     }
 
     @Bean
@@ -217,20 +210,6 @@ public class ThingsboardSecurityConfiguration {
     }
 
     @Bean
-    @Order(1)
-    SecurityFilterChain httpTransportFilterChain(HttpSecurity http) throws Exception {
-        http
-                .securityMatchers(matchers -> matchers.requestMatchers(DEVICE_API_ENTRY_POINT))
-                .cors(cors -> {})
-                .csrf(AbstractHttpConfigurer::disable)
-                .authorizeHttpRequests(config -> config
-                        .requestMatchers(DEVICE_API_ENTRY_POINT).permitAll())
-                .addFilterBefore(httpTransportRequestSizeFilter(), UsernamePasswordAuthenticationFilter.class);
-        return http.build();
-    }
-
-    @Bean
-    @Order(2)
     SecurityFilterChain filterChain(HttpSecurity http) throws Exception {
         http.headers(headers -> headers
                         .cacheControl(config -> {})

application/src/main/resources/thingsboard.yml

@@ -52,8 +52,8 @@ server:
         key_password: "${SSL_KEY_PASSWORD:thingsboard}"
   # HTTP settings
   http:
-    # Maximum request size
+    # Semi-colon-separated list of urlPattern=maxPayloadSize pairs that define max http request size for specified url pattern.
-    max_payload_size: "${HTTP_MAX_PAYLOAD_SIZE:16777216}" # max payload size in bytes
+    max_payload_size: "${HTTP_MAX_PAYLOAD_SIZE_LIMIT_CONFIGURATION:/api/image*/**=52428800;/api/**=16777216}"
   # HTTP/2 support (takes effect only if server SSL is enabled)
   http2:
     # Enable/disable HTTP/2 support
@@ -963,8 +963,8 @@ transport:
     request_timeout: "${HTTP_REQUEST_TIMEOUT:60000}"
     # HTTP maximum request processing timeout in milliseconds
     max_request_timeout: "${HTTP_MAX_REQUEST_TIMEOUT:300000}"
-    # HTTP maximum request size
+    # Semi-colon-separated list of urlPattern=maxPayloadSize pairs that define max http request size for specified url pattern.
-    max_payload_size: "${HTTP_MAX_PAYLOAD_SIZE:65536}" # max payload size in bytes
+    max_payload_size: "${HTTP_MAX_PAYLOAD_SIZE_LIMIT_CONFIGURATION:/api/v1/*/attributes=52428800;/api/v1/**=65536}"
   # Local MQTT transport parameters
   mqtt:
     # Enable/disable mqtt transport protocol.

application/src/test/java/org/thingsboard/server/controller/RpcControllerTest.java

@@ -40,7 +40,7 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.
 
 @DaoSqlTest
 @TestPropertySource(properties = {
-        "server.http.max_payload_size=10000"
+        "server.http.max_payload_size=/api/image*/**=10;/api/**=10000"
 })
 public class RpcControllerTest extends AbstractControllerTest {
 

application/src/test/java/org/thingsboard/server/system/BaseHttpDeviceApiTest.java

@@ -42,7 +42,7 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.
  */
 @TestPropertySource(properties = {
         "transport.http.enabled=true",
-        "transport.http.max_payload_size=10000"
+        "transport.http.max_payload_size=/api/v1/*/attributes=20000;/api/v1/**=10000"
 })
 public abstract class BaseHttpDeviceApiTest extends AbstractControllerTest {
 
@@ -80,13 +80,13 @@ public abstract class BaseHttpDeviceApiTest extends AbstractControllerTest {
     @Test
     public void testReplyToCommandWithLargeResponse() throws Exception {
         String errorResponse = doPost("/api/v1/" + deviceCredentials.getCredentialsId() + "/rpc/5",
-                JacksonUtil.toString(createRpcResponsePayload(10001)),
+                JacksonUtil.toString(createJsonPayloadOfSize(10001)),
                 String.class,
                 status().isPayloadTooLarge());
         assertThat(errorResponse).contains("Payload size exceeds the limit");
 
         doPost("/api/v1/" + deviceCredentials.getCredentialsId() + "/rpc/5",
-                JacksonUtil.toString(createRpcResponsePayload(10000)),
+                JacksonUtil.toString(createJsonPayloadOfSize(10000)),
                 String.class,
                 status().isOk());
     }
@@ -105,7 +105,21 @@ public abstract class BaseHttpDeviceApiTest extends AbstractControllerTest {
                 status().isOk());
     }
 
-    private String createRpcResponsePayload(int size) {
+    @Test
+    public void testPostLargeAttribute() throws Exception {
+        String errorResponse = doPost("/api/v1/" + deviceCredentials.getCredentialsId() + "/attributes",
+                JacksonUtil.toString(createJsonPayloadOfSize(20001)),
+                String.class,
+                status().isPayloadTooLarge());
+        assertThat(errorResponse).contains("Payload size exceeds the limit");
+
+        doPost("/api/v1/" + deviceCredentials.getCredentialsId() + "/attributes",
+                JacksonUtil.toString(createJsonPayloadOfSize(20000)),
+                String.class,
+                status().isOk());
+    }
+
+    private String createJsonPayloadOfSize(int size) {
         String value = "a".repeat(size - 19);
         return "{\"result\":\"" + value + "\"}";
     }

common/message/src/main/java/org/thingsboard/server/common/msg/tools/MaxPayloadSizeExceededException.java

@@ -20,9 +20,9 @@ import lombok.Getter;
 public class MaxPayloadSizeExceededException extends RuntimeException {
 
     @Getter
-    private final int limit;
+    private final long limit;
 
-    public MaxPayloadSizeExceededException(int limit) {
+    public MaxPayloadSizeExceededException(long limit) {
         super("Payload size exceeds the limit " + limit);
         this.limit = limit;
     }

common/transport/http/src/main/java/org/thingsboard/server/transport/http/DeviceApiController.java

@@ -24,11 +24,9 @@ import io.swagger.v3.oas.annotations.media.Schema;
 import io.swagger.v3.oas.annotations.responses.ApiResponse;
 import io.swagger.v3.oas.annotations.responses.ApiResponses;
 import jakarta.servlet.http.HttpServletRequest;
-import jakarta.servlet.http.HttpServletResponse;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Value;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
 import org.springframework.core.io.ByteArrayResource;
 import org.springframework.http.HttpHeaders;
@@ -36,7 +34,6 @@ import org.springframework.http.HttpStatus;
 import org.springframework.http.MediaType;
 import org.springframework.http.ResponseEntity;
 import org.springframework.http.converter.HttpMessageNotReadableException;
-import org.springframework.web.bind.annotation.ExceptionHandler;
 import org.springframework.web.bind.annotation.PathVariable;
 import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RequestMapping;
@@ -44,7 +41,6 @@ import org.springframework.web.bind.annotation.RequestMethod;
 import org.springframework.web.bind.annotation.RequestParam;
 import org.springframework.web.bind.annotation.RestController;
 import org.springframework.web.context.request.async.DeferredResult;
-import org.thingsboard.common.util.JacksonUtil;
 import org.thingsboard.server.common.adaptor.JsonConverter;
 import org.thingsboard.server.common.data.DataConstants;
 import org.thingsboard.server.common.data.DeviceTransportType;
@@ -53,7 +49,6 @@ import org.thingsboard.server.common.data.TbTransportService;
 import org.thingsboard.server.common.data.id.DeviceId;
 import org.thingsboard.server.common.data.ota.OtaPackageType;
 import org.thingsboard.server.common.data.rpc.RpcStatus;
-import org.thingsboard.server.common.msg.tools.MaxPayloadSizeExceededException;
 import org.thingsboard.server.common.transport.SessionMsgListener;
 import org.thingsboard.server.common.transport.TransportContext;
 import org.thingsboard.server.common.transport.TransportService;
@@ -76,7 +71,6 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToServerRpcRequestMs
 import org.thingsboard.server.gen.transport.TransportProtos.ToServerRpcResponseMsg;
 import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceTokenRequestMsg;
 
-import java.io.IOException;
 import java.util.Arrays;
 import java.util.List;
 import java.util.UUID;
@@ -626,20 +620,6 @@ public class DeviceApiController implements TbTransportService {
 
     }
 
-    @ExceptionHandler(MaxPayloadSizeExceededException.class)
-    public void handle(MaxPayloadSizeExceededException exception, HttpServletRequest request, HttpServletResponse response) {
-        log.debug("Too large payload size. Url: {}, client ip: {}, content length: {}", request.getRequestURL(),
-                request.getRemoteAddr(), request.getContentLength());
-        if (!response.isCommitted()) {
-            try {
-                response.setStatus(HttpStatus.PAYLOAD_TOO_LARGE.value());
-                JacksonUtil.writeValue(response.getWriter(), exception.getMessage());
-            } catch (IOException e) {
-                log.error("Can't handle exception", e);
-            }
-        }
-    }
-
     private static MediaType parseMediaType(String contentType) {
         try {
             return MediaType.parseMediaType(contentType);

common/transport/http/src/main/java/org/thingsboard/server/transport/http/config/RequestSizeFilter.java

@@ -22,28 +22,56 @@ import jakarta.servlet.http.HttpServletResponse;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.http.HttpStatus;
+import org.springframework.util.AntPathMatcher;
 import org.springframework.web.filter.OncePerRequestFilter;
 import org.thingsboard.common.util.JacksonUtil;
 import org.thingsboard.server.common.msg.tools.MaxPayloadSizeExceededException;
 
 import java.io.IOException;
+import java.util.LinkedHashMap;
+import java.util.Map;
 
 @Slf4j
 @RequiredArgsConstructor
 public class RequestSizeFilter extends OncePerRequestFilter {
 
-    private final int maxPayloadSize;
+    private final Map<String, Long> limits = new LinkedHashMap<>();
+    private final AntPathMatcher pathMatcher = new AntPathMatcher();
+
+    public RequestSizeFilter(String limitsConfiguration) {
+        for (String limit : limitsConfiguration.split(";")) {
+            try {
+                String urlPathPattern = limit.split("=")[0];
+                long maxPayloadSize = Long.parseLong(limit.split("=")[1]);
+                limits.put(urlPathPattern, maxPayloadSize);
+            } catch (Exception e) {
+                throw new IllegalArgumentException("Failed to parse size limits configuration: " + limitsConfiguration);
+            }
+        }
+    }
 
     @Override
     public void doFilterInternal(HttpServletRequest request, HttpServletResponse response, FilterChain chain) throws IOException, ServletException {
+        for (String url : limits.keySet()) {
+            if (pathMatcher.match(url, request.getRequestURI())) {
+                if (checkMaxPayloadSizeExceeded(request, response, limits.get(url))) {
+                    return;
+                }
+                break;
+            }
+        }
+        chain.doFilter(request, response);
+    }
+
+    private boolean checkMaxPayloadSizeExceeded(HttpServletRequest request, HttpServletResponse response, long maxPayloadSize) throws IOException {
         if (request.getContentLength() > maxPayloadSize) {
             if (log.isDebugEnabled()) {
                 log.debug("Too large payload size. Url: {}, client ip: {}, content length: {}", request.getRequestURL(), request.getRemoteAddr(), request.getContentLength());
             }
             handleMaxPayloadSizeExceededException(response, new MaxPayloadSizeExceededException(maxPayloadSize));
-            return;
+            return true;
         }
-        chain.doFilter(request, response);
+        return false;
     }
 
     @Override

common/transport/http/src/main/java/org/thingsboard/server/transport/http/config/TransportSecurityConfiguration.java

@@ -27,27 +27,29 @@ import org.springframework.security.config.annotation.web.configuration.EnableWe
 import org.springframework.security.config.annotation.web.configurers.AbstractHttpConfigurer;
 import org.springframework.security.web.SecurityFilterChain;
 import org.springframework.security.web.authentication.UsernamePasswordAuthenticationFilter;
+import org.springframework.security.web.util.matcher.AntPathRequestMatcher;
 
 @Configuration
 @EnableWebSecurity
 @EnableMethodSecurity
 @Order(SecurityProperties.BASIC_AUTH_ORDER)
-@ConditionalOnExpression("('${service.type:null}'=='tb-transport')")
+@ConditionalOnExpression("'${service.type:null}'=='tb-transport' || ('${service.type:null}'=='monolith' && '${transport.api_enabled:true}'=='true' && '${transport.http.enabled}'=='true')")
 public class TransportSecurityConfiguration {
     public static final String DEVICE_API_ENTRY_POINT = "/api/v1/**";
 
-    @Value("${transport.http.max_payload_size:65536}")
+    @Value("${transport.http.max_payload_size:/api/v1/*/attributes=52428800;/api/v1/**=65536}")
-    private int maxPayloadSize;
+    private String maxPayloadSizeConfig;
 
     @Bean
     protected RequestSizeFilter httpTransportRequestSizeFilter() {
-        return new RequestSizeFilter(maxPayloadSize);
+        return new RequestSizeFilter(maxPayloadSizeConfig);
     }
 
     @Bean
+    @Order(1)
     SecurityFilterChain httpTransportFilterChain(HttpSecurity http) throws Exception {
         http
-                .securityMatchers(matchers -> matchers.requestMatchers(DEVICE_API_ENTRY_POINT))
+                .securityMatcher(AntPathRequestMatcher.antMatcher(DEVICE_API_ENTRY_POINT))
                 .cors(cors -> {
                 })
                 .csrf(AbstractHttpConfigurer::disable)

transport/http/src/main/resources/tb-http-transport.yml

@@ -170,8 +170,8 @@ transport:
     request_timeout: "${HTTP_REQUEST_TIMEOUT:60000}"
     # HTTP maximum request processing timeout in milliseconds
     max_request_timeout: "${HTTP_MAX_REQUEST_TIMEOUT:300000}"
-    # HTTP maximum request size
+    # Semi-colon-separated list of urlPattern=maxPayloadSize pairs that define max http request size for specified url pattern.
-    max_payload_size: "${HTTP_MAX_PAYLOAD_SIZE:65536}" # max payload size in bytes
+    max_payload_size: "${HTTP_MAX_PAYLOAD_SIZE_LIMIT_CONFIGURATION:/api/v1/*/attributes=52428800;/api/v1/**=65536}"
   sessions:
     # Session inactivity timeout is a global configuration parameter that defines how long the device transport session will be opened after the last message arrives from the device.
     # The parameter value is in milliseconds.