HTTP API: validate payload size (#11265)

* added check for large ToDeviceRpcResponseMsg, ToServerRpcRequestMsg messages on default transport level

* moved http request size check to controller

* minor refactoring

* test fixed

* updated error messages, updated url patterns to single one, added yml property to http transport

* updated swagger docs, added to rpc filter "/api/rpc/**" and "/api/plugins/rpc/**" endpoints

* remove redundant set

* fixed large request check for DeviceApiController in microservice architecture

* renamed yml parameter and filter
This commit is contained in:
Daria Shevchenko 2024-07-30 14:04:53 +03:00 committed by GitHub
parent f6f1aaf538
commit 15cb12e48f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
10 changed files with 241 additions and 5 deletions

View File

@ -0,0 +1,78 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.config;
import jakarta.servlet.FilterChain;
import jakarta.servlet.ServletException;
import jakarta.servlet.http.HttpServletRequest;
import jakarta.servlet.http.HttpServletResponse;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.springframework.util.AntPathMatcher;
import org.springframework.web.filter.OncePerRequestFilter;
import org.thingsboard.server.common.msg.tools.MaxPayloadSizeExceededException;
import org.thingsboard.server.exception.ThingsboardErrorResponseHandler;
import java.io.IOException;
import java.util.List;
@Slf4j
@Component
@RequiredArgsConstructor
public class RequestSizeFilter extends OncePerRequestFilter {
private final List<String> urls = List.of("/api/plugins/rpc/**", "/api/rpc/**");
private final AntPathMatcher pathMatcher = new AntPathMatcher();
private final ThingsboardErrorResponseHandler errorResponseHandler;
@Value("${transport.http.max_payload_size:65536}")
private int maxPayloadSize;
@Override
public void doFilterInternal(HttpServletRequest request, HttpServletResponse response, FilterChain chain) throws IOException, ServletException {
if (request.getContentLength() > maxPayloadSize) {
if (log.isDebugEnabled()) {
log.debug("Too large payload size. Url: {}, client ip: {}, content length: {}", request.getRequestURL(),
request.getRemoteAddr(), request.getContentLength());
}
errorResponseHandler.handle(new MaxPayloadSizeExceededException(), response);
return;
}
chain.doFilter(request, response);
}
@Override
protected boolean shouldNotFilter(HttpServletRequest request) {
for (String url : urls) {
if (pathMatcher.match(url, request.getRequestURI())) {
return false;
}
}
return true;
}
@Override
protected boolean shouldNotFilterAsyncDispatch() {
return false;
}
@Override
protected boolean shouldNotFilterErrorDispatch() {
return false;
}
}

View File

@ -124,6 +124,9 @@ public class ThingsboardSecurityConfiguration {
@Autowired
private RateLimitProcessingFilter rateLimitProcessingFilter;
@Autowired
private RequestSizeFilter requestSizeFilter;
@Bean
protected FilterRegistrationBean<ShallowEtagHeaderFilter> buildEtagFilter() throws Exception {
ShallowEtagHeaderFilter etagFilter = new ShallowEtagHeaderFilter();
@ -225,6 +228,7 @@ public class ThingsboardSecurityConfiguration {
.addFilterBefore(buildRestPublicLoginProcessingFilter(), UsernamePasswordAuthenticationFilter.class)
.addFilterBefore(buildJwtTokenAuthenticationProcessingFilter(), UsernamePasswordAuthenticationFilter.class)
.addFilterBefore(buildRefreshTokenProcessingFilter(), UsernamePasswordAuthenticationFilter.class)
.addFilterBefore(requestSizeFilter, UsernamePasswordAuthenticationFilter.class)
.addFilterAfter(rateLimitProcessingFilter, UsernamePasswordAuthenticationFilter.class);
if (oauth2Configuration != null) {
http.oauth2Login(login -> login

View File

@ -118,6 +118,7 @@ public class RpcV2Controller extends AbstractRpcController {
@ApiResponse(responseCode = "200", description = "Persistent RPC request was saved to the database or lightweight RPC request was sent to the device."),
@ApiResponse(responseCode = "400", description = "Invalid structure of the request."),
@ApiResponse(responseCode = "401", description = "User is not authorized to send the RPC request. Most likely, User belongs to different Customer or Tenant."),
@ApiResponse(responseCode = "413", description = "Request payload is too large"),
@ApiResponse(responseCode = "504", description = "Timeout to process the RPC call. Most likely, device is offline."),
})
@PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN', 'CUSTOMER_USER')")
@ -136,6 +137,7 @@ public class RpcV2Controller extends AbstractRpcController {
@ApiResponse(responseCode = "200", description = "Persistent RPC request was saved to the database or lightweight RPC response received."),
@ApiResponse(responseCode = "400", description = "Invalid structure of the request."),
@ApiResponse(responseCode = "401", description = "User is not authorized to send the RPC request. Most likely, User belongs to different Customer or Tenant."),
@ApiResponse(responseCode = "413", description = "Request payload is too large"),
@ApiResponse(responseCode = "504", description = "Timeout to process the RPC call. Most likely, device is offline."),
})
@PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN', 'CUSTOMER_USER')")

View File

@ -46,6 +46,7 @@ import org.springframework.web.util.WebUtils;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.common.data.exception.ThingsboardErrorCode;
import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.msg.tools.MaxPayloadSizeExceededException;
import org.thingsboard.server.common.msg.tools.TbRateLimitsException;
import org.thingsboard.server.service.security.exception.AuthMethodNotSupportedException;
import org.thingsboard.server.service.security.exception.JwtExpiredTokenException;
@ -146,6 +147,8 @@ public class ThingsboardErrorResponseHandler extends ResponseEntityExceptionHand
handleAccessDeniedException(response);
} else if (exception instanceof AuthenticationException) {
handleAuthenticationException((AuthenticationException) exception, response);
} else if (exception instanceof MaxPayloadSizeExceededException) {
handleMaxPayloadSizeExceededException(response, (MaxPayloadSizeExceededException) exception);
} else {
response.setStatus(HttpStatus.INTERNAL_SERVER_ERROR.value());
JacksonUtil.writeValue(response.getWriter(), ThingsboardErrorResponse.of(exception.getMessage(),
@ -184,6 +187,13 @@ public class ThingsboardErrorResponseHandler extends ResponseEntityExceptionHand
ThingsboardErrorCode.TOO_MANY_REQUESTS, HttpStatus.TOO_MANY_REQUESTS));
}
private void handleMaxPayloadSizeExceededException(HttpServletResponse response, MaxPayloadSizeExceededException exception) throws IOException {
response.setStatus(HttpStatus.PAYLOAD_TOO_LARGE.value());
JacksonUtil.writeValue(response.getWriter(),
ThingsboardErrorResponse.of(exception.getMessage(),
ThingsboardErrorCode.BAD_REQUEST_PARAMS, HttpStatus.PAYLOAD_TOO_LARGE));
}
private void handleSubscriptionException(ThingsboardException subscriptionException, HttpServletResponse response) throws IOException {
response.setStatus(HttpStatus.FORBIDDEN.value());
JacksonUtil.writeValue(response.getWriter(),

View File

@ -959,6 +959,8 @@ transport:
request_timeout: "${HTTP_REQUEST_TIMEOUT:60000}"
# HTTP maximum request processing timeout in milliseconds
max_request_timeout: "${HTTP_MAX_REQUEST_TIMEOUT:300000}"
# Maximum request size
max_payload_size: "${HTTP_MAX_PAYLOAD_SIZE:65536}" # max payload size in bytes
# Local MQTT transport parameters
mqtt:
# Enable/disable mqtt transport protocol.

View File

@ -22,6 +22,7 @@ import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.springframework.test.context.TestPropertySource;
import org.springframework.test.web.servlet.MvcResult;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.common.data.Device;
@ -38,6 +39,9 @@ import java.util.List;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
@DaoSqlTest
@TestPropertySource(properties = {
"transport.http.max_payload_size=10000"
})
public class RpcControllerTest extends AbstractControllerTest {
private Tenant savedTenant;
@ -78,13 +82,18 @@ public class RpcControllerTest extends AbstractControllerTest {
}
private ObjectNode createDefaultRpc() {
return createDefaultRpc(100);
}
private ObjectNode createDefaultRpc(int size) {
ObjectNode rpc = JacksonUtil.newObjectNode();
rpc.put("method", "setGpio");
ObjectNode params = JacksonUtil.newObjectNode();
params.put("pin", 7);
params.put("value", 1);
String value = "a".repeat(size - 83);
params.put("value", value);
rpc.set("params", params);
rpc.put("persistent", true);
@ -122,6 +131,28 @@ public class RpcControllerTest extends AbstractControllerTest {
Assert.assertEquals(savedDevice.getId(), savedRpc.getDeviceId());
}
@Test
public void testSaveLargeRpc() throws Exception {
Device device = createDefaultDevice();
Device savedDevice = doPost("/api/device", device, Device.class);
ObjectNode rpc = createDefaultRpc(10001);
doPost(
"/api/rpc/oneway/" + savedDevice.getId().getId().toString(),
JacksonUtil.toString(rpc),
String.class,
status().isPayloadTooLarge()
);
ObjectNode validRpc = createDefaultRpc(10000);
doPost(
"/api/rpc/oneway/" + savedDevice.getId().getId().toString(),
JacksonUtil.toString(validRpc),
String.class,
status().isOk()
);
}
@Test
public void testDeleteRpc() throws Exception {
Device device = createDefaultDevice();

View File

@ -20,6 +20,7 @@ import org.junit.Test;
import org.springframework.test.context.TestPropertySource;
import org.springframework.test.web.servlet.ResultActions;
import org.springframework.test.web.servlet.request.MockHttpServletRequestBuilder;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.common.data.Device;
import org.thingsboard.server.common.data.security.DeviceCredentials;
import org.thingsboard.server.controller.AbstractControllerTest;
@ -29,6 +30,7 @@ import java.util.Map;
import java.util.Random;
import java.util.concurrent.atomic.AtomicInteger;
import static org.assertj.core.api.Assertions.assertThat;
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.asyncDispatch;
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get;
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post;
@ -40,6 +42,7 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.
*/
@TestPropertySource(properties = {
"transport.http.enabled=true",
"transport.http.max_payload_size=10000"
})
public abstract class BaseHttpDeviceApiTest extends AbstractControllerTest {
@ -74,6 +77,44 @@ public abstract class BaseHttpDeviceApiTest extends AbstractControllerTest {
doGetAsync("/api/v1/" + deviceCredentials.getCredentialsId() + "/attributes?clientKeys=keyA,keyB,keyC").andExpect(status().isOk());
}
@Test
public void testReplyToCommandWithLargeResponse() throws Exception {
String errorResponse = doPost("/api/v1/" + deviceCredentials.getCredentialsId() + "/rpc/5",
JacksonUtil.toString(createRpcResponsePayload(10001)),
String.class,
status().isPayloadTooLarge());
assertThat(errorResponse).contains("Payload size exceeds the limit");
doPost("/api/v1/" + deviceCredentials.getCredentialsId() + "/rpc/5",
JacksonUtil.toString(createRpcResponsePayload(10000)),
String.class,
status().isOk());
}
@Test
public void testPostRpcRequestWithLargeResponse() throws Exception {
String errorResponse = doPost("/api/v1/" + deviceCredentials.getCredentialsId() + "/rpc",
JacksonUtil.toString(createRpcRequestPayload(10001)),
String.class,
status().isPayloadTooLarge());
assertThat(errorResponse).contains("Payload size exceeds the limit");
doPost("/api/v1/" + deviceCredentials.getCredentialsId() + "/rpc",
JacksonUtil.toString(createRpcRequestPayload(10000)),
String.class,
status().isOk());
}
private String createRpcResponsePayload(int size) {
String value = "a".repeat(size - 19);
return "{\"result\":\"" + value + "\"}";
}
private String createRpcRequestPayload(int size) {
String value = "a".repeat(size - 50);
return "{\"method\":\"get\",\"params\":{\"value\":\"" + value + "\"}}";
}
protected ResultActions doGetAsync(String urlTemplate, Object... urlVariables) throws Exception {
MockHttpServletRequestBuilder getRequest;
getRequest = get(urlTemplate, urlVariables);

View File

@ -0,0 +1,23 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.msg.tools;
public class MaxPayloadSizeExceededException extends RuntimeException {
public MaxPayloadSizeExceededException() {
super("Payload size exceeds the limit");
}
}

View File

@ -21,9 +21,14 @@ import com.google.gson.JsonParser;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.responses.ApiResponse;
import io.swagger.v3.oas.annotations.responses.ApiResponses;
import jakarta.servlet.http.HttpServletRequest;
import jakarta.servlet.http.HttpServletResponse;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
import org.springframework.core.io.ByteArrayResource;
import org.springframework.http.HttpHeaders;
@ -31,6 +36,7 @@ import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.http.converter.HttpMessageNotReadableException;
import org.springframework.web.bind.annotation.ExceptionHandler;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
@ -38,6 +44,8 @@ import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.context.request.async.DeferredResult;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.common.adaptor.JsonConverter;
import org.thingsboard.server.common.data.DataConstants;
import org.thingsboard.server.common.data.DeviceTransportType;
import org.thingsboard.server.common.data.StringUtils;
@ -45,11 +53,11 @@ import org.thingsboard.server.common.data.TbTransportService;
import org.thingsboard.server.common.data.id.DeviceId;
import org.thingsboard.server.common.data.ota.OtaPackageType;
import org.thingsboard.server.common.data.rpc.RpcStatus;
import org.thingsboard.server.common.msg.tools.MaxPayloadSizeExceededException;
import org.thingsboard.server.common.transport.SessionMsgListener;
import org.thingsboard.server.common.transport.TransportContext;
import org.thingsboard.server.common.transport.TransportService;
import org.thingsboard.server.common.transport.TransportServiceCallback;
import org.thingsboard.server.common.adaptor.JsonConverter;
import org.thingsboard.server.common.transport.auth.SessionInfoCreator;
import org.thingsboard.server.common.transport.auth.ValidateDeviceCredentialsResponse;
import org.thingsboard.server.gen.transport.TransportProtos;
@ -68,7 +76,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToServerRpcRequestMs
import org.thingsboard.server.gen.transport.TransportProtos.ToServerRpcResponseMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceTokenRequestMsg;
import jakarta.servlet.http.HttpServletRequest;
import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import java.util.UUID;
@ -128,6 +136,9 @@ public class DeviceApiController implements TbTransportService {
private static final String ACCESS_TOKEN_PARAM_DESCRIPTION = "Your device access token.";
@Value("${transport.http.max_payload_size:65536}")
private int maxPayloadSize;
@Autowired
private HttpTransportContext transportContext;
@ -265,6 +276,11 @@ public class DeviceApiController implements TbTransportService {
@Operation(summary = "Reply to RPC commands (replyToCommand)",
description = "Replies to server originated RPC command identified by 'requestId' parameter. The response is arbitrary JSON.\n\n" +
REQUIRE_ACCESS_TOKEN)
@ApiResponses(value = {
@ApiResponse(responseCode = "200", description = "RPC reply to command request was sent to Core."),
@ApiResponse(responseCode = "400", description = "Invalid structure of the request."),
@ApiResponse(responseCode = "413", description = "Request payload is too large."),
})
@RequestMapping(value = "/{deviceToken}/rpc/{requestId}", method = RequestMethod.POST)
public DeferredResult<ResponseEntity> replyToCommand(
@Parameter(description = ACCESS_TOKEN_PARAM_DESCRIPTION, required = true , schema = @Schema(defaultValue = "YOUR_DEVICE_ACCESS_TOKEN"))
@ -272,7 +288,8 @@ public class DeviceApiController implements TbTransportService {
@Parameter(description = "RPC request id from the incoming RPC request", required = true , schema = @Schema(defaultValue = "123"))
@PathVariable("requestId") Integer requestId,
@io.swagger.v3.oas.annotations.parameters.RequestBody(description = "Reply to the RPC request, JSON. For example: {\"status\":\"success\"}", required = true)
@RequestBody String json) {
@RequestBody String json, HttpServletRequest httpServletRequest) {
checkPayloadSize(httpServletRequest);
DeferredResult<ResponseEntity> responseWriter = new DeferredResult<ResponseEntity>();
transportContext.getTransportService().process(DeviceTransportType.DEFAULT, ValidateDeviceTokenRequestMsg.newBuilder().setToken(deviceToken).build(),
new DeviceAuthCallback(transportContext, responseWriter, sessionInfo -> {
@ -292,12 +309,18 @@ public class DeviceApiController implements TbTransportService {
"{\"result\": 4}" +
MARKDOWN_CODE_BLOCK_END +
REQUIRE_ACCESS_TOKEN)
@ApiResponses(value = {
@ApiResponse(responseCode = "200", description = "RPC request to server was sent to Rule Engine."),
@ApiResponse(responseCode = "400", description = "Invalid structure of the request."),
@ApiResponse(responseCode = "413", description = "Request payload too large."),
})
@RequestMapping(value = "/{deviceToken}/rpc", method = RequestMethod.POST)
public DeferredResult<ResponseEntity> postRpcRequest(
@Parameter(description = ACCESS_TOKEN_PARAM_DESCRIPTION, required = true , schema = @Schema(defaultValue = "YOUR_DEVICE_ACCESS_TOKEN"))
@PathVariable("deviceToken") String deviceToken,
@io.swagger.v3.oas.annotations.parameters.RequestBody(description = "The RPC request JSON", required = true)
@RequestBody String json) {
@RequestBody String json, HttpServletRequest httpServletRequest) {
checkPayloadSize(httpServletRequest);
DeferredResult<ResponseEntity> responseWriter = new DeferredResult<ResponseEntity>();
transportContext.getTransportService().process(DeviceTransportType.DEFAULT, ValidateDeviceTokenRequestMsg.newBuilder().setToken(deviceToken).build(),
new DeviceAuthCallback(transportContext, responseWriter, sessionInfo -> {
@ -420,6 +443,12 @@ public class DeviceApiController implements TbTransportService {
return responseWriter;
}
private void checkPayloadSize(HttpServletRequest httpServletRequest) {
if (httpServletRequest.getContentLength() > maxPayloadSize) {
throw new MaxPayloadSizeExceededException();
}
}
private DeferredResult<ResponseEntity> getOtaPackageCallback(String deviceToken, String title, String version, int size, int chunk, OtaPackageType firmwareType) {
DeferredResult<ResponseEntity> responseWriter = new DeferredResult<>();
transportContext.getTransportService().process(DeviceTransportType.DEFAULT, ValidateDeviceTokenRequestMsg.newBuilder().setToken(deviceToken).build(),
@ -608,6 +637,20 @@ public class DeviceApiController implements TbTransportService {
}
@ExceptionHandler(MaxPayloadSizeExceededException.class)
public void handle(MaxPayloadSizeExceededException exception, HttpServletRequest request, HttpServletResponse response) {
log.debug("Too large payload size. Url: {}, client ip: {}, content length: {}", request.getRequestURL(),
request.getRemoteAddr(), request.getContentLength());
if (!response.isCommitted()) {
try {
response.setStatus(HttpStatus.PAYLOAD_TOO_LARGE.value());
JacksonUtil.writeValue(response.getWriter(), exception.getMessage());
} catch (IOException e) {
log.error("Can't handle exception", e);
}
}
}
private static MediaType parseMediaType(String contentType) {
try {
return MediaType.parseMediaType(contentType);

View File

@ -170,6 +170,8 @@ transport:
request_timeout: "${HTTP_REQUEST_TIMEOUT:60000}"
# HTTP maximum request processing timeout in milliseconds
max_request_timeout: "${HTTP_MAX_REQUEST_TIMEOUT:300000}"
# Maximum request size
max_payload_size: "${HTTP_MAX_PAYLOAD_SIZE:65536}" # max payload size in bytes
sessions:
# Session inactivity timeout is a global configuration parameter that defines how long the device transport session will be opened after the last message arrives from the device.
# The parameter value is in milliseconds.