Improvements to FirmwareInfo

This commit is contained in:
Andrii Shvaika 2021-04-23 14:35:47 +03:00
parent a9e83b484f
commit b0a27954ea
16 changed files with 147 additions and 65 deletions

View File

@ -70,6 +70,7 @@ CREATE TABLE IF NOT EXISTS firmware (
checksum_algorithm varchar(32),
checksum varchar(1020),
data bytea,
data_size bigint,
additional_info varchar,
search_text varchar(255),
CONSTRAINT firmware_tenant_title_version_unq_key UNIQUE (tenant_id, title, version)

View File

@ -129,16 +129,18 @@ public class FirmwareController extends BaseController {
firmware.setVersion(info.getVersion());
firmware.setAdditionalInfo(info.getAdditionalInfo());
byte[] data = file.getBytes();
if (StringUtils.isEmpty(checksumAlgorithm)) {
checksumAlgorithm = "sha256";
checksum = Hashing.sha256().hashBytes(file.getBytes()).toString();
checksum = Hashing.sha256().hashBytes(data).toString();
}
firmware.setChecksumAlgorithm(checksumAlgorithm);
firmware.setChecksum(checksum);
firmware.setFileName(file.getOriginalFilename());
firmware.setContentType(file.getContentType());
firmware.setData(ByteBuffer.wrap(file.getBytes()));
firmware.setData(ByteBuffer.wrap(data));
firmware.setDataSize((long) data.length);
return firmwareService.saveFirmware(firmware);
} catch (Exception e) {
throw handleException(e);

View File

@ -23,6 +23,7 @@ import org.thingsboard.server.common.data.DataConstants;
import org.thingsboard.server.common.data.Device;
import org.thingsboard.server.common.data.DeviceProfile;
import org.thingsboard.server.common.data.Firmware;
import org.thingsboard.server.common.data.FirmwareInfo;
import org.thingsboard.server.common.data.id.DeviceId;
import org.thingsboard.server.common.data.id.FirmwareId;
import org.thingsboard.server.common.data.id.TenantId;
@ -174,12 +175,21 @@ public class DefaultFirmwareStateService implements FirmwareStateService {
.setTs(ts)
.build();
FirmwareInfo firmware = firmwareService.findFirmwareInfoById(tenantId, firmwareId);
if (firmware == null) {
log.warn("[{}] Failed to send firmware update because firmware was already deleted", firmwareId);
return;
}
TopicPartitionInfo tpi = new TopicPartitionInfo(fwStateMsgProducer.getDefaultTopic(), null, null, false);
fwStateMsgProducer.send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), msg), null);
BasicTsKvEntry status = new BasicTsKvEntry(ts, new StringDataEntry(DataConstants.FIRMWARE_STATE, FirmwareUpdateStatus.QUEUED.name()));
List<TsKvEntry> telemetry = new ArrayList<>();
telemetry.add(new BasicTsKvEntry(ts, new StringDataEntry(DataConstants.TARGET_FIRMWARE_TITLE, firmware.getTitle())));
telemetry.add(new BasicTsKvEntry(ts, new StringDataEntry(DataConstants.TARGET_FIRMWARE_VERSION, firmware.getVersion())));
telemetry.add(new BasicTsKvEntry(ts, new StringDataEntry(DataConstants.FIRMWARE_STATE, FirmwareUpdateStatus.QUEUED.name())));
telemetryService.saveAndNotify(tenantId, deviceId, Collections.singletonList(status), new FutureCallback<>() {
telemetryService.saveAndNotify(tenantId, deviceId, telemetry, new FutureCallback<>() {
@Override
public void onSuccess(@Nullable Void tmp) {
log.trace("[{}] Success save firmware status!", deviceId);
@ -193,16 +203,13 @@ public class DefaultFirmwareStateService implements FirmwareStateService {
}
private void update(Device device, Firmware firmware, long ts) {
private void update(Device device, FirmwareInfo firmware, long ts) {
TenantId tenantId = device.getTenantId();
DeviceId deviceId = device.getId();
List<TsKvEntry> telemetry = new ArrayList<>();
telemetry.add(new BasicTsKvEntry(ts, new StringDataEntry(DataConstants.TARGET_FIRMWARE_TITLE, firmware.getTitle())));
telemetry.add(new BasicTsKvEntry(ts, new StringDataEntry(DataConstants.TARGET_FIRMWARE_VERSION, firmware.getVersion())));
telemetry.add(new BasicTsKvEntry(ts, new StringDataEntry(DataConstants.FIRMWARE_STATE, FirmwareUpdateStatus.INITIATED.name())));
BasicTsKvEntry status = new BasicTsKvEntry(System.currentTimeMillis(), new StringDataEntry(DataConstants.FIRMWARE_STATE, FirmwareUpdateStatus.INITIATED.name()));
telemetryService.saveAndNotify(tenantId, deviceId, telemetry, new FutureCallback<>() {
telemetryService.saveAndNotify(tenantId, deviceId, Collections.singletonList(status), new FutureCallback<>() {
@Override
public void onSuccess(@Nullable Void tmp) {
log.trace("[{}] Success save telemetry with target firmware for device!", deviceId);
@ -219,7 +226,7 @@ public class DefaultFirmwareStateService implements FirmwareStateService {
attributes.add(new BaseAttributeKvEntry(ts, new StringDataEntry(DataConstants.FIRMWARE_TITLE, firmware.getTitle())));
attributes.add(new BaseAttributeKvEntry(ts, new StringDataEntry(DataConstants.FIRMWARE_VERSION, firmware.getVersion())));
attributes.add(new BaseAttributeKvEntry(ts, new LongDataEntry(FIRMWARE_SIZE, (long) firmware.getData().array().length)));
attributes.add(new BaseAttributeKvEntry(ts, new LongDataEntry(FIRMWARE_SIZE, firmware.getDataSize())));
attributes.add(new BaseAttributeKvEntry(ts, new StringDataEntry(DataConstants.FIRMWARE_CHECKSUM_ALGORITHM, firmware.getChecksumAlgorithm())));
attributes.add(new BaseAttributeKvEntry(ts, new StringDataEntry(DataConstants.FIRMWARE_CHECKSUM, firmware.getChecksum())));
telemetryService.saveAndNotify(tenantId, deviceId, DataConstants.SHARED_SCOPE, attributes, new FutureCallback<>() {

View File

@ -1,3 +1,18 @@
/**
* Copyright © 2016-2021 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.firmware;
public enum FirmwareUpdateStatus {

View File

@ -151,8 +151,8 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
@PostConstruct
public void init() {
super.init("tb-core-consumer", "tb-core-notifications-consumer");
this.usageStatsExecutor = Executors.newCachedThreadPool(ThingsBoardThreadFactory.forName("tb-core-usage-stats-consumer"));
this.firmwareStatesExecutor = Executors.newCachedThreadPool(ThingsBoardThreadFactory.forName("tb-core-firmware-notifications-consumer"));
this.usageStatsExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("tb-core-usage-stats-consumer"));
this.firmwareStatesExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("tb-core-firmware-notifications-consumer"));
}
@PreDestroy
@ -363,7 +363,6 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
try {
List<TbProtoQueueMsg<ToFirmwareStateServiceMsg>> msgs = firmwareStatesConsumer.poll(getNotificationPollDuration());
if (msgs.isEmpty()) {
Thread.sleep(maxProcessingTimeoutPerRecord);
continue;
}
long timeToSleep = maxProcessingTimeoutPerRecord;
@ -374,10 +373,13 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
long endTime = System.currentTimeMillis();
long spentTime = endTime - startTime;
timeToSleep = timeToSleep - spentTime;
if (isSuccessUpdate && timeToSleep > 0) {
if (isSuccessUpdate) {
if (timeToSleep > 0) {
log.debug("Spent time per record is: [{}]!", spentTime);
Thread.sleep(timeToSleep);
timeToSleep = maxProcessingTimeoutPerRecord;
timeToSleep = 0;
}
timeToSleep += maxProcessingTimeoutPerRecord;
}
} catch (Throwable e) {
log.warn("Failed to process firmware update msg: {}", msg, e);

View File

@ -27,14 +27,6 @@ public class Firmware extends FirmwareInfo {
private static final long serialVersionUID = 3091601761339422546L;
private String fileName;
private String contentType;
private String checksumAlgorithm;
private String checksum;
private transient ByteBuffer data;
public Firmware() {
@ -47,10 +39,6 @@ public class Firmware extends FirmwareInfo {
public Firmware(Firmware firmware) {
super(firmware);
this.fileName = firmware.getFileName();
this.contentType = firmware.getContentType();
this.data = firmware.getData();
this.checksumAlgorithm = firmware.getChecksumAlgorithm();
this.checksum = firmware.getChecksum();
}
}

View File

@ -32,6 +32,12 @@ public class FirmwareInfo extends SearchTextBasedWithAdditionalInfo<FirmwareId>
private String title;
private String version;
private boolean hasData;
private String fileName;
private String contentType;
private String checksumAlgorithm;
private String checksum;
private Long dataSize;
public FirmwareInfo() {
super();
@ -47,6 +53,11 @@ public class FirmwareInfo extends SearchTextBasedWithAdditionalInfo<FirmwareId>
this.title = firmwareInfo.getTitle();
this.version = firmwareInfo.getVersion();
this.hasData = firmwareInfo.isHasData();
this.fileName = firmwareInfo.getFileName();
this.contentType = firmwareInfo.getContentType();
this.checksumAlgorithm = firmwareInfo.getChecksumAlgorithm();
this.checksum = firmwareInfo.getChecksum();
this.dataSize = firmwareInfo.getDataSize();
}
@Override

View File

@ -184,13 +184,7 @@ public class BaseFirmwareService implements FirmwareService {
protected void validateUpdate(TenantId tenantId, FirmwareInfo firmware) {
FirmwareInfo firmwareOld = firmwareInfoDao.findById(tenantId, firmware.getUuidId());
if (!firmwareOld.getTitle().equals(firmware.getTitle())) {
throw new DataValidationException("Updating firmware title is prohibited!");
}
if (!firmwareOld.getVersion().equals(firmware.getVersion())) {
throw new DataValidationException("Updating firmware version is prohibited!");
}
BaseFirmwareService.validateUpdate(firmware, firmwareOld);
}
};
@ -261,6 +255,15 @@ public class BaseFirmwareService implements FirmwareService {
protected void validateUpdate(TenantId tenantId, Firmware firmware) {
Firmware firmwareOld = firmwareDao.findById(tenantId, firmware.getUuidId());
BaseFirmwareService.validateUpdate(firmware, firmwareOld);
if (firmwareOld.getData() != null && !firmwareOld.getData().equals(firmware.getData())) {
throw new DataValidationException("Updating firmware data is prohibited!");
}
}
};
private static void validateUpdate(FirmwareInfo firmware, FirmwareInfo firmwareOld) {
if (!firmwareOld.getTitle().equals(firmware.getTitle())) {
throw new DataValidationException("Updating firmware title is prohibited!");
}
@ -285,11 +288,10 @@ public class BaseFirmwareService implements FirmwareService {
throw new DataValidationException("Updating firmware content type is prohibited!");
}
if (firmwareOld.getData() != null && !firmwareOld.getData().equals(firmware.getData())) {
throw new DataValidationException("Updating firmware data is prohibited!");
if (firmwareOld.getDataSize() != null && !firmwareOld.getDataSize().equals(firmware.getDataSize())) {
throw new DataValidationException("Updating firmware data size is prohibited!");
}
}
};
private PaginatedRemover<TenantId, FirmwareInfo> tenantFirmwareRemover =
new PaginatedRemover<>() {

View File

@ -483,6 +483,7 @@ public class ModelConstants {
public static final String FIRMWARE_CHECKSUM_ALGORITHM_COLUMN = "checksum_algorithm";
public static final String FIRMWARE_CHECKSUM_COLUMN = "checksum";
public static final String FIRMWARE_DATA_COLUMN = "data";
public static final String FIRMWARE_DATA_SIZE_COLUMN = "data_size";
public static final String FIRMWARE_ADDITIONAL_INFO_COLUMN = ADDITIONAL_INFO_PROPERTY;
public static final String FIRMWARE_HAS_DATA_PROPERTY = "has_data";

View File

@ -38,6 +38,7 @@ import static org.thingsboard.server.dao.model.ModelConstants.FIRMWARE_CHECKSUM_
import static org.thingsboard.server.dao.model.ModelConstants.FIRMWARE_CHECKSUM_COLUMN;
import static org.thingsboard.server.dao.model.ModelConstants.FIRMWARE_CONTENT_TYPE_COLUMN;
import static org.thingsboard.server.dao.model.ModelConstants.FIRMWARE_DATA_COLUMN;
import static org.thingsboard.server.dao.model.ModelConstants.FIRMWARE_DATA_SIZE_COLUMN;
import static org.thingsboard.server.dao.model.ModelConstants.FIRMWARE_FILE_NAME_COLUMN;
import static org.thingsboard.server.dao.model.ModelConstants.FIRMWARE_TABLE_NAME;
import static org.thingsboard.server.dao.model.ModelConstants.FIRMWARE_TENANT_ID_COLUMN;
@ -76,6 +77,9 @@ public class FirmwareEntity extends BaseSqlEntity<Firmware> implements SearchTex
@Column(name = FIRMWARE_DATA_COLUMN, columnDefinition = "BINARY")
private byte[] data;
@Column(name = FIRMWARE_DATA_SIZE_COLUMN)
private Long dataSize;
@Type(type = "json")
@Column(name = ModelConstants.FIRMWARE_ADDITIONAL_INFO_COLUMN)
private JsonNode additionalInfo;
@ -98,6 +102,7 @@ public class FirmwareEntity extends BaseSqlEntity<Firmware> implements SearchTex
this.checksumAlgorithm = firmware.getChecksumAlgorithm();
this.checksum = firmware.getChecksum();
this.data = firmware.getData().array();
this.dataSize = firmware.getDataSize();
this.additionalInfo = firmware.getAdditionalInfo();
}
@ -122,6 +127,7 @@ public class FirmwareEntity extends BaseSqlEntity<Firmware> implements SearchTex
firmware.setContentType(contentType);
firmware.setChecksumAlgorithm(checksumAlgorithm);
firmware.setChecksum(checksum);
firmware.setDataSize(dataSize);
if (data != null) {
firmware.setData(ByteBuffer.wrap(data));
firmware.setHasData(true);

View File

@ -35,6 +35,12 @@ import javax.persistence.Table;
import javax.persistence.Transient;
import java.util.UUID;
import static org.thingsboard.server.dao.model.ModelConstants.FIRMWARE_CHECKSUM_ALGORITHM_COLUMN;
import static org.thingsboard.server.dao.model.ModelConstants.FIRMWARE_CHECKSUM_COLUMN;
import static org.thingsboard.server.dao.model.ModelConstants.FIRMWARE_CONTENT_TYPE_COLUMN;
import static org.thingsboard.server.dao.model.ModelConstants.FIRMWARE_DATA_COLUMN;
import static org.thingsboard.server.dao.model.ModelConstants.FIRMWARE_DATA_SIZE_COLUMN;
import static org.thingsboard.server.dao.model.ModelConstants.FIRMWARE_FILE_NAME_COLUMN;
import static org.thingsboard.server.dao.model.ModelConstants.FIRMWARE_HAS_DATA_PROPERTY;
import static org.thingsboard.server.dao.model.ModelConstants.FIRMWARE_TABLE_NAME;
import static org.thingsboard.server.dao.model.ModelConstants.FIRMWARE_TENANT_ID_COLUMN;
@ -58,6 +64,21 @@ public class FirmwareInfoEntity extends BaseSqlEntity<FirmwareInfo> implements S
@Column(name = FIRMWARE_VERSION_COLUMN)
private String version;
@Column(name = FIRMWARE_FILE_NAME_COLUMN)
private String fileName;
@Column(name = FIRMWARE_CONTENT_TYPE_COLUMN)
private String contentType;
@Column(name = FIRMWARE_CHECKSUM_ALGORITHM_COLUMN)
private String checksumAlgorithm;
@Column(name = FIRMWARE_CHECKSUM_COLUMN)
private String checksum;
@Column(name = FIRMWARE_DATA_SIZE_COLUMN)
private Long dataSize;
@Type(type = "json")
@Column(name = ModelConstants.FIRMWARE_ADDITIONAL_INFO_COLUMN)
private JsonNode additionalInfo;
@ -65,7 +86,6 @@ public class FirmwareInfoEntity extends BaseSqlEntity<FirmwareInfo> implements S
@Column(name = SEARCH_TEXT_PROPERTY)
private String searchText;
// @Column(name = FIRMWARE_HAS_DATA_PROPERTY, insertable = false, updatable = false)
@Transient
private boolean hasData;
@ -79,15 +99,27 @@ public class FirmwareInfoEntity extends BaseSqlEntity<FirmwareInfo> implements S
this.tenantId = firmware.getTenantId().getId();
this.title = firmware.getTitle();
this.version = firmware.getVersion();
this.fileName = firmware.getFileName();
this.contentType = firmware.getContentType();
this.checksumAlgorithm = firmware.getChecksumAlgorithm();
this.checksum = firmware.getChecksum();
this.dataSize = firmware.getDataSize();
this.additionalInfo = firmware.getAdditionalInfo();
}
public FirmwareInfoEntity(UUID id, long createdTime, UUID tenantId, String title, String version, Object additionalInfo, boolean hasData) {
public FirmwareInfoEntity(UUID id, long createdTime, UUID tenantId, String title, String version,
String fileName, String contentType, String checksumAlgorithm, String checksum, Long dataSize,
Object additionalInfo, boolean hasData) {
this.id = id;
this.createdTime = createdTime;
this.tenantId = tenantId;
this.title = title;
this.version = version;
this.fileName = fileName;
this.contentType = contentType;
this.checksumAlgorithm = checksumAlgorithm;
this.checksum = checksum;
this.dataSize = dataSize;
this.hasData = hasData;
this.additionalInfo = JacksonUtil.convertValue(additionalInfo, JsonNode.class);
}
@ -109,6 +141,11 @@ public class FirmwareInfoEntity extends BaseSqlEntity<FirmwareInfo> implements S
firmware.setTenantId(new TenantId(tenantId));
firmware.setTitle(title);
firmware.setVersion(version);
firmware.setFileName(fileName);
firmware.setContentType(contentType);
firmware.setChecksumAlgorithm(checksumAlgorithm);
firmware.setChecksum(checksum);
firmware.setDataSize(dataSize);
firmware.setAdditionalInfo(additionalInfo);
firmware.setHasData(hasData);
return firmware;

View File

@ -25,14 +25,14 @@ import org.thingsboard.server.dao.model.sql.FirmwareInfoEntity;
import java.util.UUID;
public interface FirmwareInfoRepository extends CrudRepository<FirmwareInfoEntity, UUID> {
@Query("SELECT new FirmwareInfoEntity(f.id, f.createdTime, f.tenantId, f.title, f.version, f.additionalInfo, f.data IS NOT NULL) FROM FirmwareEntity f WHERE " +
@Query("SELECT new FirmwareInfoEntity(f.id, f.createdTime, f.tenantId, f.title, f.version, f.fileName, f.contentType, f.checksumAlgorithm, f.checksum, f.dataSize, f.additionalInfo, f.data IS NOT NULL) FROM FirmwareEntity f WHERE " +
"f.tenantId = :tenantId " +
"AND LOWER(f.searchText) LIKE LOWER(CONCAT(:searchText, '%'))")
Page<FirmwareInfoEntity> findAllByTenantId(@Param("tenantId") UUID tenantId,
@Param("searchText") String searchText,
Pageable pageable);
@Query("SELECT new FirmwareInfoEntity(f.id, f.createdTime, f.tenantId, f.title, f.version, f.additionalInfo, f.data IS NOT NULL) FROM FirmwareEntity f WHERE " +
@Query("SELECT new FirmwareInfoEntity(f.id, f.createdTime, f.tenantId, f.title, f.version, f.fileName, f.contentType, f.checksumAlgorithm, f.checksum, f.dataSize, f.additionalInfo, f.data IS NOT NULL) FROM FirmwareEntity f WHERE " +
"f.tenantId = :tenantId " +
"AND ((f.data IS NOT NULL AND :hasData = true) OR (f.data IS NULL AND :hasData = false ))" +
"AND LOWER(f.searchText) LIKE LOWER(CONCAT(:searchText, '%'))")
@ -41,6 +41,6 @@ public interface FirmwareInfoRepository extends CrudRepository<FirmwareInfoEntit
@Param("searchText") String searchText,
Pageable pageable);
@Query("SELECT new FirmwareInfoEntity(f.id, f.createdTime, f.tenantId, f.title, f.version, f.additionalInfo, f.data IS NOT NULL) FROM FirmwareEntity f WHERE f.id = :id")
@Query("SELECT new FirmwareInfoEntity(f.id, f.createdTime, f.tenantId, f.title, f.version, f.fileName, f.contentType, f.checksumAlgorithm, f.checksum, f.dataSize, f.additionalInfo, f.data IS NOT NULL) FROM FirmwareEntity f WHERE f.id = :id")
FirmwareInfoEntity findFirmwareInfoById(@Param("id") UUID id);
}

View File

@ -169,6 +169,7 @@ CREATE TABLE IF NOT EXISTS firmware (
checksum_algorithm varchar(32),
checksum varchar(1020),
data binary,
data_size bigint,
additional_info varchar,
search_text varchar(255),
CONSTRAINT firmware_tenant_title_version_unq_key UNIQUE (tenant_id, title, version)

View File

@ -187,6 +187,7 @@ CREATE TABLE IF NOT EXISTS firmware (
checksum_algorithm varchar(32),
checksum varchar(1020),
data bytea,
data_size bigint,
additional_info varchar,
search_text varchar(255),
CONSTRAINT firmware_tenant_title_version_unq_key UNIQUE (tenant_id, title, version)

View File

@ -28,6 +28,7 @@ import java.util.Arrays;
"org.thingsboard.server.dao.service.attributes.sql.*SqlTest",
"org.thingsboard.server.dao.service.event.sql.*SqlTest",
"org.thingsboard.server.dao.service.timeseries.sql.*SqlTest"
})
public class SqlDaoServiceTestSuite {

View File

@ -122,6 +122,7 @@ public abstract class BaseFirmwareServiceTest extends AbstractServiceTest {
firmwareService.saveFirmware(firmware);
savedFirmwareInfo = firmwareService.findFirmwareInfoById(tenantId, savedFirmwareInfo.getId());
savedFirmwareInfo.setAdditionalInfo(JacksonUtil.newObjectNode());
firmwareService.saveFirmwareInfo(savedFirmwareInfo);
@ -421,6 +422,11 @@ public abstract class BaseFirmwareServiceTest extends AbstractServiceTest {
firmwareInfo.setTenantId(tenantId);
firmwareInfo.setTitle(TITLE);
firmwareInfo.setVersion(VERSION + i);
firmwareInfo.setFileName(FILE_NAME);
firmwareInfo.setContentType(CONTENT_TYPE);
firmwareInfo.setChecksumAlgorithm(CHECKSUM_ALGORITHM);
firmwareInfo.setChecksum(CHECKSUM);
firmwareInfo.setDataSize((long) DATA.array().length);
firmwares.add(firmwareService.saveFirmwareInfo(firmwareInfo));
}
@ -451,6 +457,7 @@ public abstract class BaseFirmwareServiceTest extends AbstractServiceTest {
firmware.setChecksumAlgorithm(CHECKSUM_ALGORITHM);
firmware.setChecksum(CHECKSUM);
firmware.setData(DATA);
firmware.setDataSize((long) DATA.array().length);
firmwareService.saveFirmware(firmware);
f.setHasData(true);
});