diff --git a/tools/pom.xml b/tools/pom.xml index 99f966b283..4877914b62 100644 --- a/tools/pom.xml +++ b/tools/pom.xml @@ -70,7 +70,7 @@ maven-assembly-plugin - + org.thingsboard.client.tools.migrator.MigratorTool diff --git a/tools/src/main/java/org/thingsboard/client/tools/migrator/PgCaMigrator.java b/tools/src/main/java/org/thingsboard/client/tools/migrator/PgCaMigrator.java index 7aec7ba5f3..b611fec6f0 100644 --- a/tools/src/main/java/org/thingsboard/client/tools/migrator/PgCaMigrator.java +++ b/tools/src/main/java/org/thingsboard/client/tools/migrator/PgCaMigrator.java @@ -43,7 +43,6 @@ public class PgCaMigrator { private final long LOG_BATCH = 1000000; private final long rowPerFile = 1000000; - private long linesProcessed = 0; private long linesTsMigrated = 0; private long linesLatestMigrated = 0; private long castErrors = 0; @@ -99,7 +98,7 @@ public class PgCaMigrator { System.out.println("START TO MIGRATE LATEST"); long start = System.currentTimeMillis(); processBlock(iterator, currentTsLatestWriter, outTsLatestDir, this::toValuesLatest); - System.out.println("FORMING OF SSL FOR LATEST TS FINISHED WITH TIME: " + (System.currentTimeMillis() - start) + " ms."); + System.out.println("TOTAL LINES MIGRATED: " + linesLatestMigrated + ", FORMING OF SSL FOR LATEST TS FINISHED WITH TIME: " + (System.currentTimeMillis() - start) + " ms."); isLatestDone = true; } @@ -107,7 +106,7 @@ public class PgCaMigrator { System.out.println("START TO MIGRATE TS"); long start = System.currentTimeMillis(); processBlock(iterator, currentTsWriter, outTsDir, this::toValuesTs); - System.out.println("FORMING OF SSL FOR TS FINISHED WITH TIME: " + (System.currentTimeMillis() - start) + " ms."); + System.out.println("TOTAL LINES MIGRATED: " + linesTsMigrated + ", FORMING OF SSL FOR TS FINISHED WITH TIME: " + (System.currentTimeMillis() - start) + " ms."); isTsDone = true; } } @@ -212,7 +211,7 @@ public class PgCaMigrator { private void processBlock(LineIterator iterator, CQLSSTableWriter writer, File outDir, Function, List> function) { String currentLine; - linesProcessed = 0; + long linesProcessed = 0; while(iterator.hasNext()) { logLinesProcessed(linesProcessed++); currentLine = iterator.nextLine(); diff --git a/tools/src/main/java/org/thingsboard/client/tools/migrator/README.md b/tools/src/main/java/org/thingsboard/client/tools/migrator/README.md index cfe1675eb2..a468cbd3af 100644 --- a/tools/src/main/java/org/thingsboard/client/tools/migrator/README.md +++ b/tools/src/main/java/org/thingsboard/client/tools/migrator/README.md @@ -23,7 +23,7 @@ It will generate single jar file with all required dependencies inside `target d 1. Dump related tables that need to correct save telemetry - `pg_dump -h localhost -U postgres -d thingsboard -t tenant -t customer -t user -t dashboard -t asset -t device -t alarm -t rule_chain -t rule_node -t entity_view -t widgets_bundle -t widget_type -t tenant_profile -t device_profile -t api_usage_state -t tb_user > related_entities.dmp` + `pg_dump -h localhost -U postgres -d thingsboard -T admin_settings -T attribute_kv -T audit_log -T component_discriptor -T device_credentials -T event -T oauth2_client_registration -T oauth2_client_registration_info -T oauth2_client_registration_template -T relation -T rule_node_state tb_schema_settings -T user_credentials > related_entities.dmp` 2. Dump `ts_kv` and child: @@ -47,12 +47,12 @@ Create 3 empty directories. For example: **Note: if you run this tool on remote instance - don't forget to execute this command in `screen` to avoid unexpected termination* ``` -java -jar ./tools-2.4.1-SNAPSHOT-jar-with-dependencies.jar +java -jar ./tools-3.2.2-SNAPSHOT-jar-with-dependencies.jar -telemetryFrom /home/user/dump/ts_kv_all.dmp - -relatedEntities /home/user/dump/relatedEntities.dmp - -latestOut /home/ubunut/migration/ts_latest - -tsOut /home/ubunut/migration/ts - -partitionsOut /home/ubunut/migration/ts_partition + -relatedEntities /home/user/dump/related_entities.dmp + -latestOut /home/user/migration/ts_latest + -tsOut /home/user/migration/ts + -partitionsOut /home/user/migration/ts_partition -castEnable false ``` *Use your paths for program arguments* @@ -63,7 +63,7 @@ Tool execution time depends on DB size, CPU resources and Disk throughput * Note that this this part works only for single node Cassandra Cluster. If you have more nodes - it is better to use `sstableloader` tool. 1. [Optional] install Cassandra on the instance -2. [Optional] Using `cqlsh` create `thingsboard` keyspace and requred tables from this file `schema-ts.cql` +2. [Optional] Using `cqlsh` create `thingsboard` keyspace and requred tables from this files `schema-ts.cql` and `schema-ts-latest.cql` using `source` command 3. Stop Cassandra 4. Look at `/var/lib/cassandra/data/thingsboard` and check for names of data folders 5. Copy generated SSTable files into cassandra data dir using next command: diff --git a/tools/src/main/java/org/thingsboard/client/tools/migrator/RelatedEntitiesParser.java b/tools/src/main/java/org/thingsboard/client/tools/migrator/RelatedEntitiesParser.java index fac2609797..7c1b29f2af 100644 --- a/tools/src/main/java/org/thingsboard/client/tools/migrator/RelatedEntitiesParser.java +++ b/tools/src/main/java/org/thingsboard/client/tools/migrator/RelatedEntitiesParser.java @@ -27,6 +27,24 @@ import java.util.Map; public class RelatedEntitiesParser { private final Map allEntityIdsAndTypes = new HashMap<>(); + + private final Map tableNameAndEntityType = Map.ofEntries( + Map.entry("COPY public.alarm ", EntityType.ALARM), + Map.entry("COPY public.asset ", EntityType.ASSET), + Map.entry("COPY public.customer ", EntityType.CUSTOMER), + Map.entry("COPY public.dashboard ", EntityType.DASHBOARD), + Map.entry("COPY public.device ", EntityType.DEVICE), + Map.entry("COPY public.rule_chain ", EntityType.RULE_CHAIN), + Map.entry("COPY public.rule_node ", EntityType.RULE_NODE), + Map.entry("COPY public.tenant ", EntityType.TENANT), + Map.entry("COPY public.tb_user ", EntityType.USER), + Map.entry("COPY public.entity_view ", EntityType.ENTITY_VIEW), + Map.entry("COPY public.widgets_bundle ", EntityType.WIDGETS_BUNDLE), + Map.entry("COPY public.widget_type ", EntityType.WIDGET_TYPE), + Map.entry("COPY public.tenant_profile ", EntityType.TENANT_PROFILE), + Map.entry("COPY public.device_profile ", EntityType.DEVICE_PROFILE), + Map.entry("COPY public.api_usage_state ", EntityType.API_USAGE_STATE) + ); public RelatedEntitiesParser(File source) throws IOException { processAllTables(FileUtils.lineIterator(source)); @@ -45,36 +63,10 @@ public class RelatedEntitiesParser { try { while (lineIterator.hasNext()) { currentLine = lineIterator.nextLine(); - if (currentLine.startsWith("COPY public.alarm")) { - processBlock(lineIterator, EntityType.ALARM); - } else if (currentLine.startsWith("COPY public.asset")) { - processBlock(lineIterator, EntityType.ASSET); - } else if (currentLine.startsWith("COPY public.customer")) { - processBlock(lineIterator, EntityType.CUSTOMER); - } else if (currentLine.startsWith("COPY public.dashboard")) { - processBlock(lineIterator, EntityType.DASHBOARD); - } else if (currentLine.startsWith("COPY public.device")) { - processBlock(lineIterator, EntityType.DEVICE); - } else if (currentLine.startsWith("COPY public.rule_chain")) { - processBlock(lineIterator, EntityType.RULE_CHAIN); - } else if (currentLine.startsWith("COPY public.rule_node")) { - processBlock(lineIterator, EntityType.RULE_NODE); - } else if (currentLine.startsWith("COPY public.tenant")) { - processBlock(lineIterator, EntityType.TENANT); - } else if (currentLine.startsWith("COPY public.tb_user")) { - processBlock(lineIterator, EntityType.USER); - } else if (currentLine.startsWith("COPY public.entity_view")) { - processBlock(lineIterator, EntityType.ENTITY_VIEW); - } else if (currentLine.startsWith("COPY public.widgets_bundle")) { - processBlock(lineIterator, EntityType.WIDGETS_BUNDLE); - } else if (currentLine.startsWith("COPY public.widget_type")) { - processBlock(lineIterator, EntityType.WIDGET_TYPE); - } else if (currentLine.startsWith("COPY public.tenant_profile")) { - processBlock(lineIterator, EntityType.TENANT_PROFILE); - } else if (currentLine.startsWith("COPY public.device_profile")) { - processBlock(lineIterator, EntityType.DEVICE_PROFILE); - } else if (currentLine.startsWith("COPY public.api_usage_state")) { - processBlock(lineIterator, EntityType.API_USAGE_STATE); + for(Map.Entry entry : tableNameAndEntityType.entrySet()) { + if(currentLine.startsWith(entry.getKey())) { + processBlock(lineIterator, entry.getValue()); + } } } } finally {