dendi-wang/thingsboard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #8598 from dashevchenko/xxe_fixed

Browse Source 
Fixed xxe vulnerability
This commit is contained in:
Andrew Shvayka 2023-05-22 17:46:51 +03:00 committed by GitHub
commit 07cba2b16c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 347 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
application/src/main/java/org/thingsboard/server/service/resource/DefaultTbResourceService.java
View File

@ -16,8 +16,6 @@
package org.thingsboard.server.service.resource;
import lombok.extern.slf4j.Slf4j;
import org.eclipse.leshan.core.model.DDFFileParser;
import org.eclipse.leshan.core.model.DefaultDDFFileValidator;
import org.springframework.stereotype.Service;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.ResourceType;
@ -53,11 +51,9 @@ import static org.thingsboard.server.utils.LwM2mObjectModelUtils.toLwm2mResource
public class DefaultTbResourceService extends AbstractTbEntityService implements TbResourceService {
private final ResourceService resourceService;
private final DDFFileParser ddfFileParser;
public DefaultTbResourceService(ResourceService resourceService) {
this.resourceService = resourceService;
this.ddfFileParser = new DDFFileParser(new DefaultDDFFileValidator());
}
@Override

3
application/src/main/java/org/thingsboard/server/utils/LwM2mObjectModelUtils.java
View File

@ -27,6 +27,7 @@ import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.lwm2m.LwM2mInstance;
import org.thingsboard.server.common.data.lwm2m.LwM2mObject;
import org.thingsboard.server.common.data.lwm2m.LwM2mResourceObserve;
import org.thingsboard.server.common.data.util.TbDDFFileParser;
import org.thingsboard.server.dao.exception.DataValidationException;
import java.io.ByteArrayInputStream;
@ -41,7 +42,7 @@ import static org.thingsboard.server.common.data.lwm2m.LwM2mConstants.LWM2M_SEPA
@Slf4j
public class LwM2mObjectModelUtils {
private static final DDFFileParser ddfFileParser = new DDFFileParser(new DefaultDDFFileValidator());
private static final TbDDFFileParser ddfFileParser = new TbDDFFileParser();
public static void toLwm2mResource (TbResource resource) throws ThingsboardException {
try {

85
application/src/test/java/org/thingsboard/server/service/resource/sql/BaseTbResourceServiceTest.java
View File

@ -45,6 +45,8 @@ import java.util.Collections;
import java.util.List;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertThrows;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
@DaoSqlTest
@ -75,6 +77,32 @@ public class BaseTbResourceServiceTest extends AbstractControllerTest {
"</Object>\n" +
"</LWM2M>";
private static final String LWM2M_TEST_MODEL_WITH_XXE = "<!DOCTYPE replace [<!ENTITY ObjectVersion SYSTEM \"file:///etc/hostname\"> ]>" +
"<LWM2M xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:noNamespaceSchemaLocation=\"http://www.openmobilealliance.org/tech/profiles/LWM2M-v1_1.xsd\">\n" +
"<Object ObjectType=\"MODefinition\">\n" +
"<Name>My first resource</Name>\n" +
"<Description1></Description1>\n" +
"<ObjectID>0</ObjectID>\n" +
"<ObjectURN></ObjectURN>\n" +
"<ObjectVersion>&ObjectVersion;</ObjectVersion>\n" +
"<MultipleInstances>Multiple</MultipleInstances>\n" +
"<Mandatory>Mandatory</Mandatory>\n" +
"<Resources>\n" +
"<Item ID=\"0\">\n" +
"<Name>LWM2M</Name>\n" +
"<Operations>RW</Operations>\n" +
"<MultipleInstances>Single</MultipleInstances>\n" +
"<Mandatory>Mandatory</Mandatory>\n" +
"<Type>String</Type>\n" +
"<RangeEnumeration>0..255</RangeEnumeration>\n" +
"<Units></Units>\n" +
"<Description></Description>\n" +
"</Item>\n" +
"</Resources>\n" +
"<Description2></Description2>\n" +
"</Object>\n" +
"</LWM2M>";
private static final String DEFAULT_FILE_NAME = "test.jks";
private IdComparator<TbResourceInfo> idComparator = new IdComparator<>();
@ -126,11 +154,11 @@ public class BaseTbResourceServiceTest extends AbstractControllerTest {
loginTenantAdmin();
Assert.assertEquals(0, resourceService.sumDataSizeByTenantId(tenantId));
assertEquals(0, resourceService.sumDataSizeByTenantId(tenantId));
createResource("test", DEFAULT_FILE_NAME);
Assert.assertEquals(1, resourceService.sumDataSizeByTenantId(tenantId));
assertEquals(1, resourceService.sumDataSizeByTenantId(tenantId));
try {
assertThatThrownBy(() -> createResource("test1", 1 + DEFAULT_FILE_NAME))
@ -145,19 +173,19 @@ public class BaseTbResourceServiceTest extends AbstractControllerTest {
@Test
public void sumDataSizeByTenantId() throws Exception {
Assert.assertEquals(0, resourceService.sumDataSizeByTenantId(tenantId));
assertEquals(0, resourceService.sumDataSizeByTenantId(tenantId));
createResource("test", DEFAULT_FILE_NAME);
Assert.assertEquals(1, resourceService.sumDataSizeByTenantId(tenantId));
assertEquals(1, resourceService.sumDataSizeByTenantId(tenantId));
int maxSumDataSize = 8;
for (int i = 2; i <= maxSumDataSize; i++) {
createResource("test" + i, i + DEFAULT_FILE_NAME);
Assert.assertEquals(i, resourceService.sumDataSizeByTenantId(tenantId));
assertEquals(i, resourceService.sumDataSizeByTenantId(tenantId));
}
Assert.assertEquals(maxSumDataSize, resourceService.sumDataSizeByTenantId(tenantId));
assertEquals(maxSumDataSize, resourceService.sumDataSizeByTenantId(tenantId));
}
private TbResource createResource(String title, String filename) throws Exception {
@ -184,16 +212,16 @@ public class BaseTbResourceServiceTest extends AbstractControllerTest {
Assert.assertNotNull(savedResource);
Assert.assertNotNull(savedResource.getId());
Assert.assertTrue(savedResource.getCreatedTime() > 0);
Assert.assertEquals(resource.getTenantId(), savedResource.getTenantId());
Assert.assertEquals(resource.getTitle(), savedResource.getTitle());
Assert.assertEquals(resource.getResourceKey(), savedResource.getResourceKey());
Assert.assertEquals(resource.getData(), savedResource.getData());
assertEquals(resource.getTenantId(), savedResource.getTenantId());
assertEquals(resource.getTitle(), savedResource.getTitle());
assertEquals(resource.getResourceKey(), savedResource.getResourceKey());
assertEquals(resource.getData(), savedResource.getData());
savedResource.setTitle("My new resource");
resourceService.save(savedResource);
TbResource foundResource = resourceService.findResourceById(tenantId, savedResource.getId());
Assert.assertEquals(foundResource.getTitle(), savedResource.getTitle());
assertEquals(foundResource.getTitle(), savedResource.getTitle());
resourceService.delete(savedResource, null);
}
@ -211,10 +239,10 @@ public class BaseTbResourceServiceTest extends AbstractControllerTest {
Assert.assertNotNull(savedResource);
Assert.assertNotNull(savedResource.getId());
Assert.assertTrue(savedResource.getCreatedTime() > 0);
Assert.assertEquals(resource.getTenantId(), savedResource.getTenantId());
Assert.assertEquals("My first resource id=0 v1.0", savedResource.getTitle());
Assert.assertEquals("0_1.0", savedResource.getResourceKey());
Assert.assertEquals(resource.getData(), savedResource.getData());
assertEquals(resource.getTenantId(), savedResource.getTenantId());
assertEquals("My first resource id=0 v1.0", savedResource.getTitle());
assertEquals("0_1.0", savedResource.getResourceKey());
assertEquals(resource.getData(), savedResource.getData());
resourceService.delete(savedResource, null);
}
@ -228,7 +256,7 @@ public class BaseTbResourceServiceTest extends AbstractControllerTest {
resource.setData("Test Data");
TbResource savedResource = resourceService.save(resource);
Assert.assertEquals(TenantId.SYS_TENANT_ID, savedResource.getTenantId());
assertEquals(TenantId.SYS_TENANT_ID, savedResource.getTenantId());
resourceService.delete(savedResource, null);
}
@ -285,6 +313,21 @@ public class BaseTbResourceServiceTest extends AbstractControllerTest {
});
}
@Test
public void testSaveLwm2mTbResourceWithXXE() {
TbResource resource = new TbResource();
resource.setTenantId(tenantId);
resource.setResourceType(ResourceType.LWM2M_MODEL);
resource.setFileName("xxe_test_model.xml");
resource.setData(Base64.getEncoder().encodeToString(LWM2M_TEST_MODEL_WITH_XXE.getBytes()));
DataValidationException thrown = assertThrows(DataValidationException.class, () -> {
resourceService.save(resource);
});
assertEquals("Failed to parse file xxe_test_model.xml", thrown.getMessage());
}
@Test
public void testFindResourceById() throws Exception {
TbResource resource = new TbResource();
@ -296,7 +339,7 @@ public class BaseTbResourceServiceTest extends AbstractControllerTest {
TbResource foundResource = resourceService.findResourceById(tenantId, savedResource.getId());
Assert.assertNotNull(foundResource);
Assert.assertEquals(savedResource, foundResource);
assertEquals(savedResource, foundResource);
resourceService.delete(savedResource, null);
}
@ -312,7 +355,7 @@ public class BaseTbResourceServiceTest extends AbstractControllerTest {
TbResource foundResource = resourceService.getResource(tenantId, savedResource.getResourceType(), savedResource.getResourceKey());
Assert.assertNotNull(foundResource);
Assert.assertEquals(savedResource, foundResource);
assertEquals(savedResource, foundResource);
resourceService.delete(savedResource, null);
}
@ -366,7 +409,7 @@ public class BaseTbResourceServiceTest extends AbstractControllerTest {
Collections.sort(resources, idComparator);
Collections.sort(loadedResources, idComparator);
Assert.assertEquals(resources, loadedResources);
assertEquals(resources, loadedResources);
resourceService.deleteResourcesByTenantId(tenantId);
@ -427,14 +470,14 @@ public class BaseTbResourceServiceTest extends AbstractControllerTest {
Collections.sort(resources, idComparator);
Collections.sort(loadedResources, idComparator);
Assert.assertEquals(resources, loadedResources);
assertEquals(resources, loadedResources);
resourceService.deleteResourcesByTenantId(tenantId);
pageLink = new PageLink(100);
pageData = resourceService.findAllTenantResourcesByTenantId(tenantId, pageLink);
Assert.assertFalse(pageData.hasNext());
Assert.assertEquals(pageData.getData().size(), 100);
assertEquals(pageData.getData().size(), 100);
resourceService.deleteResourcesByTenantId(TenantId.SYS_TENANT_ID);

5
common/data/pom.xml
View File

@ -116,6 +116,11 @@
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java-util</artifactId>
</dependency>
<dependency>
<groupId>org.eclipse.leshan</groupId>
<artifactId>leshan-core</artifactId>
<scope>compile</scope>
</dependency>
</dependencies>
<build>

272
common/data/src/main/java/org/thingsboard/server/common/data/util/TbDDFFileParser.java Normal file
View File

@ -0,0 +1,272 @@
/**
* Copyright © 2016-2023 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.data.util;
import lombok.extern.slf4j.Slf4j;
import org.eclipse.leshan.core.LwM2m;
import org.eclipse.leshan.core.model.DDFFileValidator;
import org.eclipse.leshan.core.model.DefaultDDFFileValidator;
import org.eclipse.leshan.core.model.InvalidDDFFileException;
import org.eclipse.leshan.core.model.ObjectModel;
import org.eclipse.leshan.core.model.ResourceModel;
import org.eclipse.leshan.core.util.StringUtils;
import org.w3c.dom.DOMException;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.SAXException;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@Slf4j
public class TbDDFFileParser {
private static final DDFFileValidator ddfFileValidator = new DefaultDDFFileValidator();
public List<ObjectModel> parse(InputStream inputStream, String streamName)
throws InvalidDDFFileException, IOException {
streamName = streamName == null ? "" : streamName;
log.debug("Parsing DDF file {}", streamName);
try {
// Parse XML file
DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
factory.setNamespaceAware(true);
factory.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true);
DocumentBuilder builder = factory.newDocumentBuilder();
Document document = builder.parse(inputStream);
// Get DDF file validator
LwM2m.LwM2mVersion lwm2mVersion = null;
ddfFileValidator.validate(document);
// Build list of ObjectModel
ArrayList<ObjectModel> objects = new ArrayList<>();
NodeList nodeList = document.getDocumentElement().getElementsByTagName("Object");
for (int i = 0; i < nodeList.getLength(); i++) {
objects.add(parseObject(nodeList.item(i), streamName, lwm2mVersion, true));
}
return objects;
} catch (InvalidDDFFileException | SAXException e) {
throw new InvalidDDFFileException(e, "Invalid DDF file %s", streamName);
}
catch (ParserConfigurationException e) {
throw new IllegalStateException("Unable to create Document Builder", e);
}
}
private ObjectModel parseObject(Node object, String streamName, LwM2m.LwM2mVersion schemaVersion, boolean validate)
throws InvalidDDFFileException {
Node objectType = object.getAttributes().getNamedItem("ObjectType");
if (validate && (objectType == null || !"MODefinition".equals(objectType.getTextContent()))) {
throw new InvalidDDFFileException(
"Object element in %s MUST have a ObjectType attribute equals to 'MODefinition'.", streamName);
}
Integer id = null;
String name = null;
String description = null;
String version = ObjectModel.DEFAULT_VERSION;
Boolean multiple = null;
Boolean mandatory = null;
Map<Integer, ResourceModel> resources = new HashMap<>();
String urn = null;
String description2 = null;
String lwm2mVersion = ObjectModel.DEFAULT_VERSION;
for (int i = 0; i < object.getChildNodes().getLength(); i++) {
Node field = object.getChildNodes().item(i);
if (field.getNodeType() != Node.ELEMENT_NODE)
continue;
switch (field.getNodeName()) {
case "ObjectID":
id = Integer.valueOf(field.getTextContent());
break;
case "Name":
name = field.getTextContent();
break;
case "Description1":
description = field.getTextContent();
break;
case "ObjectVersion":
if (!StringUtils.isEmpty(field.getTextContent())) {
version = field.getTextContent();
}
break;
case "MultipleInstances":
if ("Multiple".equals(field.getTextContent())) {
multiple = true;
} else if ("Single".equals(field.getTextContent())) {
multiple = false;
}
break;
case "Mandatory":
if ("Mandatory".equals(field.getTextContent())) {
mandatory = true;
} else if ("Optional".equals(field.getTextContent())) {
mandatory = false;
}
break;
case "Resources":
for (int j = 0; j < field.getChildNodes().getLength(); j++) {
Node item = field.getChildNodes().item(j);
if (item.getNodeType() != Node.ELEMENT_NODE)
continue;
if (item.getNodeName().equals("Item")) {
ResourceModel resource = parseResource(item, streamName);
if (validate && resources.containsKey(resource.id)) {
throw new InvalidDDFFileException(
"Object %s in %s contains at least 2 resources with same id %s.",
id != null ? id : "", streamName, resource.id);
} else {
resources.put(resource.id, resource);
}
}
}
break;
case "ObjectURN":
urn = field.getTextContent();
break;
case "LWM2MVersion":
if (!StringUtils.isEmpty(field.getTextContent())) {
lwm2mVersion = field.getTextContent();
if (schemaVersion != null && !schemaVersion.toString().equals(lwm2mVersion)) {
throw new InvalidDDFFileException(
"LWM2MVersion is not consistent with xml shema(xsi:noNamespaceSchemaLocation) in %s : %s expected but was %s.",
streamName, schemaVersion, lwm2mVersion);
}
}
break;
case "Description2":
description2 = field.getTextContent();
break;
default:
break;
}
}
return new ObjectModel(id, name, description, version, multiple, mandatory, resources.values(), urn,
lwm2mVersion, description2);
}
private ResourceModel parseResource(Node item, String streamName) throws DOMException, InvalidDDFFileException {
Integer id = Integer.valueOf(item.getAttributes().getNamedItem("ID").getTextContent());
String name = null;
ResourceModel.Operations operations = null;
Boolean multiple = false;
Boolean mandatory = false;
ResourceModel.Type type = null;
String rangeEnumeration = null;
String units = null;
String description = null;
for (int i = 0; i < item.getChildNodes().getLength(); i++) {
Node field = item.getChildNodes().item(i);
if (field.getNodeType() != Node.ELEMENT_NODE)
continue;
switch (field.getNodeName()) {
case "Name":
name = field.getTextContent();
break;
case "Operations":
String strOp = field.getTextContent();
if (strOp != null && !strOp.isEmpty()) {
operations = ResourceModel.Operations.valueOf(strOp);
} else {
operations = ResourceModel.Operations.NONE;
}
break;
case "MultipleInstances":
if ("Multiple".equals(field.getTextContent())) {
multiple = true;
} else if ("Single".equals(field.getTextContent())) {
multiple = false;
}
break;
case "Mandatory":
if ("Mandatory".equals(field.getTextContent())) {
mandatory = true;
} else if ("Optional".equals(field.getTextContent())) {
mandatory = false;
}
break;
case "Type":
switch (field.getTextContent()) {
case "String":
type = ResourceModel.Type.STRING;
break;
case "Integer":
type = ResourceModel.Type.INTEGER;
break;
case "Float":
type = ResourceModel.Type.FLOAT;
break;
case "Boolean":
type = ResourceModel.Type.BOOLEAN;
break;
case "Opaque":
type = ResourceModel.Type.OPAQUE;
break;
case "Time":
type = ResourceModel.Type.TIME;
break;
case "Objlnk":
type = ResourceModel.Type.OBJLNK;
break;
case "Unsigned Integer":
type = ResourceModel.Type.UNSIGNED_INTEGER;
break;
case "Corelnk":
type = ResourceModel.Type.CORELINK;
break;
case "":
type = ResourceModel.Type.NONE;
break;
default:
break;
}
break;
case "RangeEnumeration":
rangeEnumeration = field.getTextContent();
break;
case "Units":
units = field.getTextContent();
break;
case "Description":
description = field.getTextContent();
break;
default:
break;
}
}
return new ResourceModel(id, name, operations, multiple, mandatory, type, rangeEnumeration, units, description);
}
}

5
common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mTransportServerHelper.java
View File

@ -25,6 +25,7 @@ import org.eclipse.leshan.core.model.ObjectModel;
import org.eclipse.leshan.core.model.ResourceModel;
import org.eclipse.leshan.core.node.codec.CodecException;
import org.springframework.stereotype.Component;
import org.thingsboard.server.common.data.util.TbDDFFileParser;
import org.thingsboard.server.common.transport.TransportServiceCallback;
import org.thingsboard.server.common.transport.auth.ValidateDeviceCredentialsResponse;
import org.thingsboard.server.gen.transport.TransportProtos;
@ -142,9 +143,9 @@ public class LwM2mTransportServerHelper {
.build();
}
public ObjectModel parseFromXmlToObjectModel(byte[] xmlByte, String streamName, DefaultDDFFileValidator ddfValidator) {
public ObjectModel parseFromXmlToObjectModel(byte[] xmlByte, String streamName) {
try {
DDFFileParser ddfFileParser = new DDFFileParser(ddfValidator);
TbDDFFileParser ddfFileParser = new TbDDFFileParser();
return ddfFileParser.parse(new ByteArrayInputStream(xmlByte), streamName).get(0);
} catch (IOException | InvalidDDFFileException e) {
log.error("Could not parse the XML file [{}]", streamName, e);

3
common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mVersionedModelProvider.java
View File

@ -154,8 +154,7 @@ public class LwM2mVersionedModelProvider implements LwM2mModelProvider {
Optional<TbResource> tbResource = context.getTransportResourceCache().get(this.tenantId, LWM2M_MODEL, key);
return tbResource.map(resource -> helper.parseFromXmlToObjectModel(
Base64.getDecoder().decode(resource.getData()),
key + ".xml",
new DefaultDDFFileValidator())).orElse(null);
key + ".xml")).orElse(null);
}
}
}