dendi-wang/thingsboard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #3238 from YevhenBondarenko/ts-latest-migration-improvements

Browse Source 
Cassandra ts latest migration improvements
This commit is contained in:
Igor Kulikov 2020-08-05 11:27:43 +03:00 committed by GitHub
commit 8edefd4975
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 18 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
application/src/main/java/org/thingsboard/server/service/install/migrate/CassandraTsLatestToSqlMigrateService.java
View File

@ -29,7 +29,8 @@ import org.thingsboard.server.dao.model.sqlts.dictionary.TsKvDictionaryComposite
import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestEntity; import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestEntity;
import org.thingsboard.server.dao.sqlts.dictionary.TsKvDictionaryRepository; import org.thingsboard.server.dao.sqlts.dictionary.TsKvDictionaryRepository;
import org.thingsboard.server.dao.sqlts.insert.latest.InsertLatestTsRepository; import org.thingsboard.server.dao.sqlts.insert.latest.InsertLatestTsRepository;
import org.thingsboard.server.dao.util.NoSqlAnyDao; import org.thingsboard.server.dao.util.NoSqlTsDao;
import org.thingsboard.server.dao.util.SqlTsLatestDao;
import org.thingsboard.server.service.install.EntityDatabaseSchemaService; import org.thingsboard.server.service.install.EntityDatabaseSchemaService;
import java.sql.Connection; import java.sql.Connection;
@ -51,11 +52,13 @@ import static org.thingsboard.server.service.install.migrate.CassandraToSqlColum
@Service @Service
@Profile("install") @Profile("install")
@NoSqlAnyDao @NoSqlTsDao
@SqlTsLatestDao
@Slf4j @Slf4j
public class CassandraTsLatestToSqlMigrateService implements TsLatestMigrateService { public class CassandraTsLatestToSqlMigrateService implements TsLatestMigrateService {
private static final int LATEST_KEY_LENGTH = 255; private static final int MAX_KEY_LENGTH = 255;
private static final int MAX_STR_V_LENGTH = 10000000;
@Autowired @Autowired
private EntityDatabaseSchemaService entityDatabaseSchemaService; private EntityDatabaseSchemaService entityDatabaseSchemaService;
@ -95,11 +98,10 @@ public class CassandraTsLatestToSqlMigrateService implements TsLatestMigrateServ
log.error("Unexpected error during ThingsBoard entities data migration!", e); log.error("Unexpected error during ThingsBoard entities data migration!", e);
throw e; throw e;
} }
entityDatabaseSchemaService.createDatabaseIndexes();
} }
private List<CassandraToSqlTable> tables = Arrays.asList( private List<CassandraToSqlTable> tables = Arrays.asList(
new CassandraToSqlTable("ts_kv_latest_cf", new CassandraToSqlTable("ts_kv_latest_cf", "ts_kv_latest",
idColumn("entity_id"), idColumn("entity_id"),
stringColumn("key"), stringColumn("key"),
bigintColumn("ts"), bigintColumn("ts"),
@ -129,6 +131,12 @@ public class CassandraTsLatestToSqlMigrateService implements TsLatestMigrateServ
String strV = data[4].getValue(); String strV = data[4].getValue();
if (strV != null) { if (strV != null) {
if (strV.length() > MAX_STR_V_LENGTH) {
log.warn("[ts_kv_latest] Value size [{}] exceeds maximum size [{}] of column [str_v] and will be truncated!",
strV.length(), MAX_STR_V_LENGTH);
log.warn("Affected data:\n{}", strV);
strV = strV.substring(0, MAX_STR_V_LENGTH);
}
latestEntity.setStrValue(strV); latestEntity.setStrValue(strV);
} else { } else {
Long longV = null; Long longV = null;
@ -170,9 +178,11 @@ public class CassandraTsLatestToSqlMigrateService implements TsLatestMigrateServ
} }
protected Integer getOrSaveKeyId(String strKey) { protected Integer getOrSaveKeyId(String strKey) {
if (strKey.length() > LATEST_KEY_LENGTH) { if (strKey.length() > MAX_KEY_LENGTH) {
log.warn("Key is long. Max key length is 255\n{}", strKey); log.warn("[ts_kv_latest] Value size [{}] exceeds maximum size [{}] of column [key] and will be truncated!",
strKey = strKey.substring(0, LATEST_KEY_LENGTH); strKey.length(), MAX_KEY_LENGTH);
log.warn("Affected data:\n{}", strKey);
strKey = strKey.substring(0, MAX_KEY_LENGTH);
} }
Integer keyId = tsKvDictionaryMap.get(strKey); Integer keyId = tsKvDictionaryMap.get(strKey);