Improvement of solution: README.md cleaned, build plugin fixed, some code cleaning
This commit is contained in:
AndrewVolosytnykhThingsboard
Andrew Shvayka
parent
d5658d5b32
commit
f7f06a2297
@ -70,7 +70,7 @@
|
||||
<plugins>
|
||||
<plugin>
|
||||
<artifactId>maven-assembly-plugin</artifactId>
|
||||
<configuration>
|
||||
<configuration combine.self="override">
|
||||
<archive>
|
||||
<manifest>
|
||||
<mainClass>org.thingsboard.client.tools.migrator.MigratorTool</mainClass>
|
||||
|
||||
@ -43,7 +43,6 @@ public class PgCaMigrator {
|
||||
private final long LOG_BATCH = 1000000;
|
||||
private final long rowPerFile = 1000000;
|
||||
|
||||
private long linesProcessed = 0;
|
||||
private long linesTsMigrated = 0;
|
||||
private long linesLatestMigrated = 0;
|
||||
private long castErrors = 0;
|
||||
@ -99,7 +98,7 @@ public class PgCaMigrator {
|
||||
System.out.println("START TO MIGRATE LATEST");
|
||||
long start = System.currentTimeMillis();
|
||||
processBlock(iterator, currentTsLatestWriter, outTsLatestDir, this::toValuesLatest);
|
||||
System.out.println("FORMING OF SSL FOR LATEST TS FINISHED WITH TIME: " + (System.currentTimeMillis() - start) + " ms.");
|
||||
System.out.println("TOTAL LINES MIGRATED: " + linesLatestMigrated + ", FORMING OF SSL FOR LATEST TS FINISHED WITH TIME: " + (System.currentTimeMillis() - start) + " ms.");
|
||||
isLatestDone = true;
|
||||
}
|
||||
|
||||
@ -107,7 +106,7 @@ public class PgCaMigrator {
|
||||
System.out.println("START TO MIGRATE TS");
|
||||
long start = System.currentTimeMillis();
|
||||
processBlock(iterator, currentTsWriter, outTsDir, this::toValuesTs);
|
||||
System.out.println("FORMING OF SSL FOR TS FINISHED WITH TIME: " + (System.currentTimeMillis() - start) + " ms.");
|
||||
System.out.println("TOTAL LINES MIGRATED: " + linesTsMigrated + ", FORMING OF SSL FOR TS FINISHED WITH TIME: " + (System.currentTimeMillis() - start) + " ms.");
|
||||
isTsDone = true;
|
||||
}
|
||||
}
|
||||
@ -212,7 +211,7 @@ public class PgCaMigrator {
|
||||
|
||||
private void processBlock(LineIterator iterator, CQLSSTableWriter writer, File outDir, Function<List<String>, List<Object>> function) {
|
||||
String currentLine;
|
||||
linesProcessed = 0;
|
||||
long linesProcessed = 0;
|
||||
while(iterator.hasNext()) {
|
||||
logLinesProcessed(linesProcessed++);
|
||||
currentLine = iterator.nextLine();
|
||||
|
||||
@ -23,7 +23,7 @@ It will generate single jar file with all required dependencies inside `target d
|
||||
|
||||
1. Dump related tables that need to correct save telemetry
|
||||
|
||||
`pg_dump -h localhost -U postgres -d thingsboard -t tenant -t customer -t user -t dashboard -t asset -t device -t alarm -t rule_chain -t rule_node -t entity_view -t widgets_bundle -t widget_type -t tenant_profile -t device_profile -t api_usage_state -t tb_user > related_entities.dmp`
|
||||
`pg_dump -h localhost -U postgres -d thingsboard -T admin_settings -T attribute_kv -T audit_log -T component_discriptor -T device_credentials -T event -T oauth2_client_registration -T oauth2_client_registration_info -T oauth2_client_registration_template -T relation -T rule_node_state tb_schema_settings -T user_credentials > related_entities.dmp`
|
||||
|
||||
2. Dump `ts_kv` and child:
|
||||
|
||||
@ -47,12 +47,12 @@ Create 3 empty directories. For example:
|
||||
**Note: if you run this tool on remote instance - don't forget to execute this command in `screen` to avoid unexpected termination*
|
||||
|
||||
```
|
||||
java -jar ./tools-2.4.1-SNAPSHOT-jar-with-dependencies.jar
|
||||
java -jar ./tools-3.2.2-SNAPSHOT-jar-with-dependencies.jar
|
||||
-telemetryFrom /home/user/dump/ts_kv_all.dmp
|
||||
-relatedEntities /home/user/dump/relatedEntities.dmp
|
||||
-latestOut /home/ubunut/migration/ts_latest
|
||||
-tsOut /home/ubunut/migration/ts
|
||||
-partitionsOut /home/ubunut/migration/ts_partition
|
||||
-relatedEntities /home/user/dump/related_entities.dmp
|
||||
-latestOut /home/user/migration/ts_latest
|
||||
-tsOut /home/user/migration/ts
|
||||
-partitionsOut /home/user/migration/ts_partition
|
||||
-castEnable false
|
||||
```
|
||||
*Use your paths for program arguments*
|
||||
@ -63,7 +63,7 @@ Tool execution time depends on DB size, CPU resources and Disk throughput
|
||||
* Note that this this part works only for single node Cassandra Cluster. If you have more nodes - it is better to use `sstableloader` tool.
|
||||
|
||||
1. [Optional] install Cassandra on the instance
|
||||
2. [Optional] Using `cqlsh` create `thingsboard` keyspace and requred tables from this file `schema-ts.cql`
|
||||
2. [Optional] Using `cqlsh` create `thingsboard` keyspace and requred tables from this files `schema-ts.cql` and `schema-ts-latest.cql` using `source` command
|
||||
3. Stop Cassandra
|
||||
4. Look at `/var/lib/cassandra/data/thingsboard` and check for names of data folders
|
||||
5. Copy generated SSTable files into cassandra data dir using next command:
|
||||
|
||||
@ -27,6 +27,24 @@ import java.util.Map;
|
||||
|
||||
public class RelatedEntitiesParser {
|
||||
private final Map<String, String> allEntityIdsAndTypes = new HashMap<>();
|
||||
|
||||
private final Map<String, EntityType> tableNameAndEntityType = Map.ofEntries(
|
||||
Map.entry("COPY public.alarm ", EntityType.ALARM),
|
||||
Map.entry("COPY public.asset ", EntityType.ASSET),
|
||||
Map.entry("COPY public.customer ", EntityType.CUSTOMER),
|
||||
Map.entry("COPY public.dashboard ", EntityType.DASHBOARD),
|
||||
Map.entry("COPY public.device ", EntityType.DEVICE),
|
||||
Map.entry("COPY public.rule_chain ", EntityType.RULE_CHAIN),
|
||||
Map.entry("COPY public.rule_node ", EntityType.RULE_NODE),
|
||||
Map.entry("COPY public.tenant ", EntityType.TENANT),
|
||||
Map.entry("COPY public.tb_user ", EntityType.USER),
|
||||
Map.entry("COPY public.entity_view ", EntityType.ENTITY_VIEW),
|
||||
Map.entry("COPY public.widgets_bundle ", EntityType.WIDGETS_BUNDLE),
|
||||
Map.entry("COPY public.widget_type ", EntityType.WIDGET_TYPE),
|
||||
Map.entry("COPY public.tenant_profile ", EntityType.TENANT_PROFILE),
|
||||
Map.entry("COPY public.device_profile ", EntityType.DEVICE_PROFILE),
|
||||
Map.entry("COPY public.api_usage_state ", EntityType.API_USAGE_STATE)
|
||||
);
|
||||
|
||||
public RelatedEntitiesParser(File source) throws IOException {
|
||||
processAllTables(FileUtils.lineIterator(source));
|
||||
@ -45,36 +63,10 @@ public class RelatedEntitiesParser {
|
||||
try {
|
||||
while (lineIterator.hasNext()) {
|
||||
currentLine = lineIterator.nextLine();
|
||||
if (currentLine.startsWith("COPY public.alarm")) {
|
||||
processBlock(lineIterator, EntityType.ALARM);
|
||||
} else if (currentLine.startsWith("COPY public.asset")) {
|
||||
processBlock(lineIterator, EntityType.ASSET);
|
||||
} else if (currentLine.startsWith("COPY public.customer")) {
|
||||
processBlock(lineIterator, EntityType.CUSTOMER);
|
||||
} else if (currentLine.startsWith("COPY public.dashboard")) {
|
||||
processBlock(lineIterator, EntityType.DASHBOARD);
|
||||
} else if (currentLine.startsWith("COPY public.device")) {
|
||||
processBlock(lineIterator, EntityType.DEVICE);
|
||||
} else if (currentLine.startsWith("COPY public.rule_chain")) {
|
||||
processBlock(lineIterator, EntityType.RULE_CHAIN);
|
||||
} else if (currentLine.startsWith("COPY public.rule_node")) {
|
||||
processBlock(lineIterator, EntityType.RULE_NODE);
|
||||
} else if (currentLine.startsWith("COPY public.tenant")) {
|
||||
processBlock(lineIterator, EntityType.TENANT);
|
||||
} else if (currentLine.startsWith("COPY public.tb_user")) {
|
||||
processBlock(lineIterator, EntityType.USER);
|
||||
} else if (currentLine.startsWith("COPY public.entity_view")) {
|
||||
processBlock(lineIterator, EntityType.ENTITY_VIEW);
|
||||
} else if (currentLine.startsWith("COPY public.widgets_bundle")) {
|
||||
processBlock(lineIterator, EntityType.WIDGETS_BUNDLE);
|
||||
} else if (currentLine.startsWith("COPY public.widget_type")) {
|
||||
processBlock(lineIterator, EntityType.WIDGET_TYPE);
|
||||
} else if (currentLine.startsWith("COPY public.tenant_profile")) {
|
||||
processBlock(lineIterator, EntityType.TENANT_PROFILE);
|
||||
} else if (currentLine.startsWith("COPY public.device_profile")) {
|
||||
processBlock(lineIterator, EntityType.DEVICE_PROFILE);
|
||||
} else if (currentLine.startsWith("COPY public.api_usage_state")) {
|
||||
processBlock(lineIterator, EntityType.API_USAGE_STATE);
|
||||
for(Map.Entry<String, EntityType> entry : tableNameAndEntityType.entrySet()) {
|
||||
if(currentLine.startsWith(entry.getKey())) {
|
||||
processBlock(lineIterator, entry.getValue());
|
||||
}
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user