From f8f7de79642ecd662a07ca8689e3fb962aa0be17 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Tue, 4 Jul 2017 09:43:53 +0300 Subject: [PATCH] TB-65: Implement upgrade procedure. --- application/pom.xml | 10 +- .../main/data/upgrade/1.3.0/schema_update.cql | 178 ++++++++++++++++++ .../install/ThingsboardInstallService.java | 92 ++++++--- .../CassandraDatabaseSchemaService.java | 2 +- .../CassandraDatabaseUpgradeService.java | 132 +++++++++++++ .../install/DatabaseUpgradeService.java | 23 +++ .../DefaultSystemDataLoaderService.java | 9 + .../install/SqlDatabaseUpgradeService.java | 37 ++++ .../install/SystemDataLoaderService.java | 2 + .../install/cql/CQLStatementsParser.java | 2 +- .../install/cql/CassandraDbHelper.java | 169 +++++++++++++++++ .../cassandra/AbstractCassandraCluster.java | 4 + pom.xml | 12 ++ 13 files changed, 644 insertions(+), 28 deletions(-) create mode 100644 application/src/main/data/upgrade/1.3.0/schema_update.cql create mode 100644 application/src/main/java/org/thingsboard/server/service/install/CassandraDatabaseUpgradeService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/install/DatabaseUpgradeService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java rename application/src/main/java/org/thingsboard/server/{ => service}/install/cql/CQLStatementsParser.java (98%) create mode 100644 application/src/main/java/org/thingsboard/server/service/install/cql/CassandraDbHelper.java diff --git a/application/pom.xml b/application/pom.xml index abd3615c95..d57222e5f5 100644 --- a/application/pom.xml +++ b/application/pom.xml @@ -27,7 +27,7 @@ application jar - Thingsboard Server Application + ThingsBoard Server Application https://thingsboard.io Open-source IoT Platform - Device management, data collection, processing and visualization @@ -137,6 +137,14 @@ org.apache.velocity velocity-tools + + commons-io + commons-io + + + org.apache.commons + commons-csv + org.springframework spring-context-support diff --git a/application/src/main/data/upgrade/1.3.0/schema_update.cql b/application/src/main/data/upgrade/1.3.0/schema_update.cql new file mode 100644 index 0000000000..4e595772a2 --- /dev/null +++ b/application/src/main/data/upgrade/1.3.0/schema_update.cql @@ -0,0 +1,178 @@ +DROP MATERIALIZED VIEW IF EXISTS thingsboard.device_by_tenant_and_name; +DROP MATERIALIZED VIEW IF EXISTS thingsboard.device_by_tenant_and_search_text; +DROP MATERIALIZED VIEW IF EXISTS thingsboard.device_by_tenant_by_type_and_search_text; +DROP MATERIALIZED VIEW IF EXISTS thingsboard.device_by_customer_and_search_text; +DROP MATERIALIZED VIEW IF EXISTS thingsboard.device_by_customer_by_type_and_search_text; +DROP MATERIALIZED VIEW IF EXISTS thingsboard.device_types_by_tenant; + +DROP TABLE IF EXISTS thingsboard.device; + +CREATE TABLE IF NOT EXISTS thingsboard.device ( + id timeuuid, + tenant_id timeuuid, + customer_id timeuuid, + name text, + type text, + search_text text, + additional_info text, + PRIMARY KEY (id, tenant_id, customer_id, type) +); + +CREATE MATERIALIZED VIEW IF NOT EXISTS thingsboard.device_by_tenant_and_name AS + SELECT * + from thingsboard.device + WHERE tenant_id IS NOT NULL AND customer_id IS NOT NULL AND type IS NOT NULL AND name IS NOT NULL AND id IS NOT NULL + PRIMARY KEY ( tenant_id, name, id, customer_id, type) + WITH CLUSTERING ORDER BY ( name ASC, id DESC, customer_id DESC); + +CREATE MATERIALIZED VIEW IF NOT EXISTS thingsboard.device_by_tenant_and_search_text AS + SELECT * + from thingsboard.device + WHERE tenant_id IS NOT NULL AND customer_id IS NOT NULL AND type IS NOT NULL AND search_text IS NOT NULL AND id IS NOT NULL + PRIMARY KEY ( tenant_id, search_text, id, customer_id, type) + WITH CLUSTERING ORDER BY ( search_text ASC, id DESC, customer_id DESC); + +CREATE MATERIALIZED VIEW IF NOT EXISTS thingsboard.device_by_tenant_by_type_and_search_text AS + SELECT * + from thingsboard.device + WHERE tenant_id IS NOT NULL AND customer_id IS NOT NULL AND type IS NOT NULL AND search_text IS NOT NULL AND id IS NOT NULL + PRIMARY KEY ( tenant_id, type, search_text, id, customer_id) + WITH CLUSTERING ORDER BY ( type ASC, search_text ASC, id DESC, customer_id DESC); + +CREATE MATERIALIZED VIEW IF NOT EXISTS thingsboard.device_by_customer_and_search_text AS + SELECT * + from thingsboard.device + WHERE tenant_id IS NOT NULL AND customer_id IS NOT NULL AND type IS NOT NULL AND search_text IS NOT NULL AND id IS NOT NULL + PRIMARY KEY ( customer_id, tenant_id, search_text, id, type ) + WITH CLUSTERING ORDER BY ( tenant_id DESC, search_text ASC, id DESC ); + +CREATE MATERIALIZED VIEW IF NOT EXISTS thingsboard.device_by_customer_by_type_and_search_text AS + SELECT * + from thingsboard.device + WHERE tenant_id IS NOT NULL AND customer_id IS NOT NULL AND type IS NOT NULL AND search_text IS NOT NULL AND id IS NOT NULL + PRIMARY KEY ( customer_id, tenant_id, type, search_text, id ) + WITH CLUSTERING ORDER BY ( tenant_id DESC, type ASC, search_text ASC, id DESC ); + +CREATE MATERIALIZED VIEW IF NOT EXISTS thingsboard.device_types_by_tenant AS + SELECT * + from thingsboard.device + WHERE tenant_id IS NOT NULL AND customer_id IS NOT NULL AND type IS NOT NULL AND id IS NOT NULL + PRIMARY KEY ( (type, tenant_id), id, customer_id) + WITH CLUSTERING ORDER BY ( id ASC, customer_id DESC); + +DROP MATERIALIZED VIEW IF EXISTS thingsboard.asset_by_tenant_and_name; +DROP MATERIALIZED VIEW IF EXISTS thingsboard.asset_by_tenant_and_search_text; +DROP MATERIALIZED VIEW IF EXISTS thingsboard.asset_by_tenant_by_type_and_search_text; +DROP MATERIALIZED VIEW IF EXISTS thingsboard.asset_by_customer_and_search_text; +DROP MATERIALIZED VIEW IF EXISTS thingsboard.asset_by_customer_by_type_and_search_text; +DROP MATERIALIZED VIEW IF EXISTS thingsboard.asset_types_by_tenant; + +DROP TABLE IF EXISTS thingsboard.asset; + +CREATE TABLE IF NOT EXISTS thingsboard.asset ( + id timeuuid, + tenant_id timeuuid, + customer_id timeuuid, + name text, + type text, + search_text text, + additional_info text, + PRIMARY KEY (id, tenant_id, customer_id, type) +); + +CREATE MATERIALIZED VIEW IF NOT EXISTS thingsboard.asset_by_tenant_and_name AS + SELECT * + from thingsboard.asset + WHERE tenant_id IS NOT NULL AND customer_id IS NOT NULL AND type IS NOT NULL AND name IS NOT NULL AND id IS NOT NULL + PRIMARY KEY ( tenant_id, name, id, customer_id, type) + WITH CLUSTERING ORDER BY ( name ASC, id DESC, customer_id DESC); + +CREATE MATERIALIZED VIEW IF NOT EXISTS thingsboard.asset_by_tenant_and_search_text AS + SELECT * + from thingsboard.asset + WHERE tenant_id IS NOT NULL AND customer_id IS NOT NULL AND type IS NOT NULL AND search_text IS NOT NULL AND id IS NOT NULL + PRIMARY KEY ( tenant_id, search_text, id, customer_id, type) + WITH CLUSTERING ORDER BY ( search_text ASC, id DESC, customer_id DESC); + +CREATE MATERIALIZED VIEW IF NOT EXISTS thingsboard.asset_by_tenant_by_type_and_search_text AS + SELECT * + from thingsboard.asset + WHERE tenant_id IS NOT NULL AND customer_id IS NOT NULL AND type IS NOT NULL AND search_text IS NOT NULL AND id IS NOT NULL + PRIMARY KEY ( tenant_id, type, search_text, id, customer_id) + WITH CLUSTERING ORDER BY ( type ASC, search_text ASC, id DESC, customer_id DESC); + +CREATE MATERIALIZED VIEW IF NOT EXISTS thingsboard.asset_by_customer_and_search_text AS + SELECT * + from thingsboard.asset + WHERE tenant_id IS NOT NULL AND customer_id IS NOT NULL AND type IS NOT NULL AND search_text IS NOT NULL AND id IS NOT NULL + PRIMARY KEY ( customer_id, tenant_id, search_text, id, type ) + WITH CLUSTERING ORDER BY ( tenant_id DESC, search_text ASC, id DESC ); + +CREATE MATERIALIZED VIEW IF NOT EXISTS thingsboard.asset_by_customer_by_type_and_search_text AS + SELECT * + from thingsboard.asset + WHERE tenant_id IS NOT NULL AND customer_id IS NOT NULL AND type IS NOT NULL AND search_text IS NOT NULL AND id IS NOT NULL + PRIMARY KEY ( customer_id, tenant_id, type, search_text, id ) + WITH CLUSTERING ORDER BY ( tenant_id DESC, type ASC, search_text ASC, id DESC ); + +CREATE MATERIALIZED VIEW IF NOT EXISTS thingsboard.asset_types_by_tenant AS + SELECT * + from thingsboard.asset + WHERE tenant_id IS NOT NULL AND customer_id IS NOT NULL AND type IS NOT NULL AND id IS NOT NULL + PRIMARY KEY ( (type, tenant_id), id, customer_id) + WITH CLUSTERING ORDER BY ( id ASC, customer_id DESC); + +CREATE TABLE IF NOT EXISTS thingsboard.alarm ( + id timeuuid, + tenant_id timeuuid, + type text, + originator_id timeuuid, + originator_type text, + severity text, + status text, + start_ts bigint, + end_ts bigint, + ack_ts bigint, + clear_ts bigint, + details text, + propagate boolean, + PRIMARY KEY ((tenant_id, originator_id, originator_type), type, id) +) WITH CLUSTERING ORDER BY ( type ASC, id DESC); + +CREATE MATERIALIZED VIEW IF NOT EXISTS thingsboard.alarm_by_id AS + SELECT * + from thingsboard.alarm + WHERE tenant_id IS NOT NULL AND originator_id IS NOT NULL AND originator_type IS NOT NULL AND type IS NOT NULL + AND type IS NOT NULL AND id IS NOT NULL + PRIMARY KEY (id, tenant_id, originator_id, originator_type, type) + WITH CLUSTERING ORDER BY ( tenant_id ASC, originator_id ASC, originator_type ASC, type ASC); + +DROP MATERIALIZED VIEW IF EXISTS thingsboard.relation_by_type_and_child_type; +DROP MATERIALIZED VIEW IF EXISTS thingsboard.reverse_relation; + +DROP TABLE IF EXISTS thingsboard.relation; + +CREATE TABLE IF NOT EXISTS thingsboard.relation ( + from_id timeuuid, + from_type text, + to_id timeuuid, + to_type text, + relation_type_group text, + relation_type text, + additional_info text, + PRIMARY KEY ((from_id, from_type), relation_type_group, relation_type, to_id, to_type) +) WITH CLUSTERING ORDER BY ( relation_type_group ASC, relation_type ASC, to_id ASC, to_type ASC); + +CREATE MATERIALIZED VIEW IF NOT EXISTS thingsboard.relation_by_type_and_child_type AS + SELECT * + from thingsboard.relation + WHERE from_id IS NOT NULL AND from_type IS NOT NULL AND relation_type_group IS NOT NULL AND relation_type IS NOT NULL AND to_id IS NOT NULL AND to_type IS NOT NULL + PRIMARY KEY ((from_id, from_type), relation_type_group, relation_type, to_type, to_id) + WITH CLUSTERING ORDER BY ( relation_type_group ASC, relation_type ASC, to_type ASC, to_id DESC); + +CREATE MATERIALIZED VIEW IF NOT EXISTS thingsboard.reverse_relation AS + SELECT * + from thingsboard.relation + WHERE from_id IS NOT NULL AND from_type IS NOT NULL AND relation_type_group IS NOT NULL AND relation_type IS NOT NULL AND to_id IS NOT NULL AND to_type IS NOT NULL + PRIMARY KEY ((to_id, to_type), relation_type_group, relation_type, from_id, from_type) + WITH CLUSTERING ORDER BY ( relation_type_group ASC, relation_type ASC, from_id ASC, from_type ASC); diff --git a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java index d14c3b5fc5..52d59985fa 100644 --- a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java +++ b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java @@ -25,6 +25,7 @@ import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Service; import org.thingsboard.server.service.component.ComponentDiscoveryService; import org.thingsboard.server.service.install.DatabaseSchemaService; +import org.thingsboard.server.service.install.DatabaseUpgradeService; import org.thingsboard.server.service.install.SystemDataLoaderService; import java.nio.file.Files; @@ -35,6 +36,12 @@ import java.nio.file.Paths; @Slf4j public class ThingsboardInstallService { + @Value("${install.upgrade:false}") + private Boolean isUpgrade; + + @Value("${install.upgrade.form_version:1.2.3}") + private String upgradeFromVersion; + @Value("${install.data_dir}") private String dataDir; @@ -44,6 +51,9 @@ public class ThingsboardInstallService { @Autowired private DatabaseSchemaService databaseSchemaService; + @Autowired + private DatabaseUpgradeService databaseUpgradeService; + @Autowired private ComponentDiscoveryService componentDiscoveryService; @@ -55,35 +65,67 @@ public class ThingsboardInstallService { public void performInstall() { try { - log.info("Starting ThingsBoard Installation..."); + if (isUpgrade) { + log.info("Starting ThingsBoard Upgrade from version {} ...", upgradeFromVersion); - if (this.dataDir == null) { - throw new RuntimeException("'install.data_dir' property should specified!"); - } - if (!Files.isDirectory(Paths.get(this.dataDir))) { - throw new RuntimeException("'install.data_dir' property value is not a valid directory!"); + switch (upgradeFromVersion) { + case "1.2.3": + log.info("Upgrading ThingsBoard from version {} to 1.3.0 ...", upgradeFromVersion); + + databaseUpgradeService.upgradeDatabase(upgradeFromVersion); + + log.info("Updating system data..."); + + systemDataLoaderService.deleteSystemWidgetBundle("charts"); + systemDataLoaderService.deleteSystemWidgetBundle("cards"); + systemDataLoaderService.deleteSystemWidgetBundle("maps"); + systemDataLoaderService.deleteSystemWidgetBundle("analogue_gauges"); + systemDataLoaderService.deleteSystemWidgetBundle("digital_gauges"); + systemDataLoaderService.deleteSystemWidgetBundle("gpio_widgets"); + systemDataLoaderService.deleteSystemWidgetBundle("alarm_widgets"); + + systemDataLoaderService.loadSystemWidgets(); + + break; + default: + throw new RuntimeException("Unable to upgrade ThingsBoard, unsupported fromVersion: " + upgradeFromVersion); + + } + log.info("Upgrade finished successfully!"); + + } else { + + log.info("Starting ThingsBoard Installation..."); + + if (this.dataDir == null) { + throw new RuntimeException("'install.data_dir' property should specified!"); + } + if (!Files.isDirectory(Paths.get(this.dataDir))) { + throw new RuntimeException("'install.data_dir' property value is not a valid directory!"); + } + + log.info("Installing DataBase schema..."); + + databaseSchemaService.createDatabaseSchema(); + + log.info("Loading system data..."); + + componentDiscoveryService.discoverComponents(); + + systemDataLoaderService.createSysAdmin(); + systemDataLoaderService.createAdminSettings(); + systemDataLoaderService.loadSystemWidgets(); + systemDataLoaderService.loadSystemPlugins(); + systemDataLoaderService.loadSystemRules(); + + if (loadDemo) { + log.info("Loading demo data..."); + systemDataLoaderService.loadDemoData(); + } + log.info("Installation finished successfully!"); } - log.info("Installing DataBase schema..."); - databaseSchemaService.createDatabaseSchema(); - - log.info("Loading system data..."); - - componentDiscoveryService.discoverComponents(); - - systemDataLoaderService.createSysAdmin(); - systemDataLoaderService.createAdminSettings(); - systemDataLoaderService.loadSystemWidgets(); - systemDataLoaderService.loadSystemPlugins(); - systemDataLoaderService.loadSystemRules(); - - if (loadDemo) { - log.info("Loading demo data..."); - systemDataLoaderService.loadDemoData(); - } - - log.info("Finished!"); } catch (Exception e) { log.error("Unexpected error during ThingsBoard installation!", e); throw new ThingsboardInstallException("Unexpected error during ThingsBoard installation!", e); diff --git a/application/src/main/java/org/thingsboard/server/service/install/CassandraDatabaseSchemaService.java b/application/src/main/java/org/thingsboard/server/service/install/CassandraDatabaseSchemaService.java index f146a2702d..ea18b9280d 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/CassandraDatabaseSchemaService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/CassandraDatabaseSchemaService.java @@ -23,7 +23,7 @@ import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Service; import org.thingsboard.server.dao.cassandra.CassandraInstallCluster; import org.thingsboard.server.dao.util.NoSqlDao; -import org.thingsboard.server.install.cql.CQLStatementsParser; +import org.thingsboard.server.service.install.cql.CQLStatementsParser; import java.nio.file.Path; import java.nio.file.Paths; diff --git a/application/src/main/java/org/thingsboard/server/service/install/CassandraDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/CassandraDatabaseUpgradeService.java new file mode 100644 index 0000000000..4e769f7fb6 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/install/CassandraDatabaseUpgradeService.java @@ -0,0 +1,132 @@ +/** + * Copyright © 2016-2017 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.thingsboard.server.service.install; + +import com.datastax.driver.core.KeyspaceMetadata; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Profile; +import org.springframework.stereotype.Service; +import org.thingsboard.server.dao.cassandra.CassandraCluster; +import org.thingsboard.server.dao.cassandra.CassandraInstallCluster; +import org.thingsboard.server.dao.util.NoSqlDao; +import org.thingsboard.server.service.install.cql.CQLStatementsParser; +import org.thingsboard.server.service.install.cql.CassandraDbHelper; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.List; + +@Service +@NoSqlDao +@Profile("install") +@Slf4j +public class CassandraDatabaseUpgradeService implements DatabaseUpgradeService { + + private static final String SCHEMA_UPDATE_CQL = "schema_update.cql"; + + @Value("${install.data_dir}") + private String dataDir; + + @Autowired + private CassandraCluster cluster; + + @Autowired + private CassandraInstallCluster installCluster; + + @Override + public void upgradeDatabase(String fromVersion) throws Exception { + + switch (fromVersion) { + case "1.2.3": + + log.info("Upgrading Cassandara DataBase from version {} to 1.3.0 ...", fromVersion); + + //Dump devices, assets and relations + + KeyspaceMetadata ks = cluster.getCluster().getMetadata().getKeyspace(cluster.getKeyspaceName()); + + log.info("Dumping devices ..."); + Path devicesDump = CassandraDbHelper.dumpCfIfExists(ks, cluster.getSession(), "device", + new String[]{"id", "tenant_id", "customer_id", "name", "search_text", "additional_info"}, + "tb-devices"); + if (devicesDump != null) { + CassandraDbHelper.appendToEndOfLine(devicesDump, "default"); + } + log.info("Devices dumped."); + + log.info("Dumping assets ..."); + Path assetsDump = CassandraDbHelper.dumpCfIfExists(ks, cluster.getSession(), "asset", + new String[]{"id", "tenant_id", "customer_id", "name", "search_text", "additional_info", "type"}, + "tb-assets"); + log.info("Assets dumped."); + + log.info("Dumping relations ..."); + Path relationsDump = CassandraDbHelper.dumpCfIfExists(ks, cluster.getSession(), "relation", + new String[]{"from_id", "from_type", "to_id", "to_type", "relation_type", "additional_info"}, + "tb-relations"); + if (relationsDump != null) { + CassandraDbHelper.appendToEndOfLine(relationsDump, "COMMON"); + } + log.info("Relations dumped."); + + log.info("Updating schema ..."); + Path schemaUpdateFile = Paths.get(this.dataDir, "upgrade", "1.3.0", SCHEMA_UPDATE_CQL); + loadCql(schemaUpdateFile); + log.info("Schema updated."); + + //Restore devices, assets and relations + + log.info("Restoring devices ..."); + if (devicesDump != null) { + CassandraDbHelper.loadCf(ks, cluster.getSession(), "device", + new String[]{"id", "tenant_id", "customer_id", "name", "search_text", "additional_info", "type"}, devicesDump); + Files.deleteIfExists(devicesDump); + } + log.info("Devices restored."); + + log.info("Restoring assets ..."); + if (assetsDump != null) { + CassandraDbHelper.loadCf(ks, cluster.getSession(), "asset", + new String[]{"id", "tenant_id", "customer_id", "name", "search_text", "additional_info", "type"}, assetsDump); + Files.deleteIfExists(assetsDump); + } + log.info("Assets restored."); + + log.info("Restoring relations ..."); + if (relationsDump != null) { + CassandraDbHelper.loadCf(ks, cluster.getSession(), "relation", + new String[]{"from_id", "from_type", "to_id", "to_type", "relation_type", "additional_info", "relation_type_group"}, relationsDump); + Files.deleteIfExists(relationsDump); + } + log.info("Relations restored."); + + break; + default: + throw new RuntimeException("Unable to upgrade Cassandra database, unsupported fromVersion: " + fromVersion); + } + + } + + private void loadCql(Path cql) throws Exception { + List statements = new CQLStatementsParser(cql).getStatements(); + statements.forEach(statement -> installCluster.getSession().execute(statement)); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/install/DatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/DatabaseUpgradeService.java new file mode 100644 index 0000000000..215e7bbca5 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/install/DatabaseUpgradeService.java @@ -0,0 +1,23 @@ +/** + * Copyright © 2016-2017 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.thingsboard.server.service.install; + +public interface DatabaseUpgradeService { + + void upgradeDatabase(String fromVersion) throws Exception; + +} diff --git a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java index ff55274bf1..1da39ab4f7 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java @@ -41,6 +41,7 @@ import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.dao.device.DeviceCredentialsService; import org.thingsboard.server.dao.device.DeviceService; +import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.plugin.PluginService; import org.thingsboard.server.dao.rule.RuleService; import org.thingsboard.server.dao.settings.AdminSettingsService; @@ -227,6 +228,14 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { loadDashboards(Paths.get(dataDir, JSON_DIR, DEMO_DIR, DASHBOARDS_DIR), demoTenant.getId(), null); } + @Override + public void deleteSystemWidgetBundle(String bundleAlias) throws Exception { + WidgetsBundle widgetsBundle = widgetsBundleService.findWidgetsBundleByTenantIdAndAlias(new TenantId(ModelConstants.NULL_UUID), bundleAlias); + if (widgetsBundle != null) { + widgetsBundleService.deleteWidgetsBundle(widgetsBundle.getId()); + } + } + private User createUser(Authority authority, TenantId tenantId, CustomerId customerId, diff --git a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java new file mode 100644 index 0000000000..fed838814e --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java @@ -0,0 +1,37 @@ +/** + * Copyright © 2016-2017 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.thingsboard.server.service.install; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.context.annotation.Profile; +import org.springframework.stereotype.Service; +import org.thingsboard.server.dao.util.SqlDao; + +@Service +@Profile("install") +@Slf4j +@SqlDao +public class SqlDatabaseUpgradeService implements DatabaseUpgradeService { + + @Override + public void upgradeDatabase(String fromVersion) throws Exception { + switch (fromVersion) { + default: + throw new RuntimeException("Unable to upgrade SQL database, unsupported fromVersion: " + fromVersion); + } + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/install/SystemDataLoaderService.java b/application/src/main/java/org/thingsboard/server/service/install/SystemDataLoaderService.java index 1a07428a12..8ca6f142c4 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/SystemDataLoaderService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/SystemDataLoaderService.java @@ -29,4 +29,6 @@ public interface SystemDataLoaderService { void loadDemoData() throws Exception; + void deleteSystemWidgetBundle(String bundleAlias) throws Exception; + } diff --git a/application/src/main/java/org/thingsboard/server/install/cql/CQLStatementsParser.java b/application/src/main/java/org/thingsboard/server/service/install/cql/CQLStatementsParser.java similarity index 98% rename from application/src/main/java/org/thingsboard/server/install/cql/CQLStatementsParser.java rename to application/src/main/java/org/thingsboard/server/service/install/cql/CQLStatementsParser.java index 4c31d5cca3..6caa76754f 100644 --- a/application/src/main/java/org/thingsboard/server/install/cql/CQLStatementsParser.java +++ b/application/src/main/java/org/thingsboard/server/service/install/cql/CQLStatementsParser.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.thingsboard.server.install.cql; +package org.thingsboard.server.service.install.cql; import lombok.extern.slf4j.Slf4j; diff --git a/application/src/main/java/org/thingsboard/server/service/install/cql/CassandraDbHelper.java b/application/src/main/java/org/thingsboard/server/service/install/cql/CassandraDbHelper.java new file mode 100644 index 0000000000..4a92db212b --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/install/cql/CassandraDbHelper.java @@ -0,0 +1,169 @@ +/** + * Copyright © 2016-2017 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.thingsboard.server.service.install.cql; + +import com.datastax.driver.core.*; +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVParser; +import org.apache.commons.csv.CSVPrinter; +import org.apache.commons.csv.CSVRecord; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.*; + +public class CassandraDbHelper { + + private static final CSVFormat CSV_DUMP_FORMAT = CSVFormat.DEFAULT.withNullString("\\N"); + + public static Path dumpCfIfExists(KeyspaceMetadata ks, Session session, String cfName, + String[] columns, String dumpPrefix) throws Exception { + if (ks.getTable(cfName) != null) { + Path dumpFile = Files.createTempFile(dumpPrefix, null); + Files.deleteIfExists(dumpFile); + try (CSVPrinter csvPrinter = new CSVPrinter(Files.newBufferedWriter(dumpFile), CSV_DUMP_FORMAT)) { + Statement stmt = new SimpleStatement("SELECT * FROM " + cfName); + stmt.setFetchSize(1000); + ResultSet rs = session.execute(stmt); + Iterator iter = rs.iterator(); + while (iter.hasNext()) { + Row row = iter.next(); + if (row != null) { + dumpRow(row, columns, csvPrinter); + } + } + } + return dumpFile; + } else { + return null; + } + } + + public static void appendToEndOfLine(Path targetDumpFile, String toAppend) throws Exception { + Path tmp = Files.createTempFile(null, null); + try (CSVParser csvParser = new CSVParser(Files.newBufferedReader(targetDumpFile), CSV_DUMP_FORMAT)) { + try (CSVPrinter csvPrinter = new CSVPrinter(Files.newBufferedWriter(tmp), CSV_DUMP_FORMAT)) { + csvParser.forEach(record -> { + List newRecord = new ArrayList<>(); + record.forEach(val -> newRecord.add(val)); + newRecord.add(toAppend); + try { + csvPrinter.printRecord(newRecord); + } catch (IOException e) { + throw new RuntimeException("Error appending to EOL", e); + } + }); + } + } + Files.move(tmp, targetDumpFile, StandardCopyOption.REPLACE_EXISTING); + } + + public static void loadCf(KeyspaceMetadata ks, Session session, String cfName, String[] columns, Path sourceFile) throws Exception { + TableMetadata tableMetadata = ks.getTable(cfName); + PreparedStatement prepared = session.prepare(createInsertStatement(cfName, columns)); + try (CSVParser csvParser = new CSVParser(Files.newBufferedReader(sourceFile), CSV_DUMP_FORMAT.withHeader(columns))) { + csvParser.forEach(record -> { + BoundStatement boundStatement = prepared.bind(); + for (String column : columns) { + setColumnValue(tableMetadata, column, record, boundStatement); + } + session.execute(boundStatement); + }); + } + } + + + private static void dumpRow(Row row, String[] columns, CSVPrinter csvPrinter) throws Exception { + List record = new ArrayList<>(); + for (String column : columns) { + record.add(getColumnValue(column, row)); + } + csvPrinter.printRecord(record); + } + + private static String getColumnValue(String column, Row row) { + String str = ""; + int index = row.getColumnDefinitions().getIndexOf(column); + if (index > -1) { + DataType type = row.getColumnDefinitions().getType(index); + try { + if (row.isNull(index)) { + return null; + } else if (type == DataType.cdouble()) { + str = new Double(row.getDouble(index)).toString(); + } else if (type == DataType.cint()) { + str = new Integer(row.getInt(index)).toString(); + } else if (type == DataType.uuid()) { + str = row.getUUID(index).toString(); + } else if (type == DataType.timeuuid()) { + str = row.getUUID(index).toString(); + } else if (type == DataType.cfloat()) { + str = new Float(row.getFloat(index)).toString(); + } else if (type == DataType.timestamp()) { + str = ""+row.getTimestamp(index).getTime(); + } else { + str = row.getString(index); + } + } catch (Exception e) { + str = ""; + } + } + return str; + } + + private static String createInsertStatement(String cfName, String[] columns) { + StringBuilder insertStatementBuilder = new StringBuilder(); + insertStatementBuilder.append("INSERT INTO ").append(cfName).append(" ("); + for (String column : columns) { + insertStatementBuilder.append(column).append(","); + } + insertStatementBuilder.deleteCharAt(insertStatementBuilder.length() - 1); + insertStatementBuilder.append(") VALUES ("); + for (String column : columns) { + insertStatementBuilder.append("?").append(","); + } + insertStatementBuilder.deleteCharAt(insertStatementBuilder.length() - 1); + insertStatementBuilder.append(")"); + return insertStatementBuilder.toString(); + } + + private static void setColumnValue(TableMetadata tableMetadata, String column, + CSVRecord record, BoundStatement boundStatement) { + String value = record.get(column); + DataType type = tableMetadata.getColumn(column).getType(); + if (value == null) { + boundStatement.setToNull(column); + } else if (type == DataType.cdouble()) { + boundStatement.setDouble(column, Double.valueOf(value)); + } else if (type == DataType.cint()) { + boundStatement.setInt(column, Integer.valueOf(value)); + } else if (type == DataType.uuid()) { + boundStatement.setUUID(column, UUID.fromString(value)); + } else if (type == DataType.timeuuid()) { + boundStatement.setUUID(column, UUID.fromString(value)); + } else if (type == DataType.cfloat()) { + boundStatement.setFloat(column, Float.valueOf(value)); + } else if (type == DataType.timestamp()) { + boundStatement.setTimestamp(column, new Date(Long.valueOf(value))); + } else { + boundStatement.setString(column, value); + } + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/cassandra/AbstractCassandraCluster.java b/dao/src/main/java/org/thingsboard/server/dao/cassandra/AbstractCassandraCluster.java index ea4f9d11be..2b9000d7dd 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cassandra/AbstractCassandraCluster.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cassandra/AbstractCassandraCluster.java @@ -127,6 +127,10 @@ public abstract class AbstractCassandraCluster { } } + public String getKeyspaceName() { + return keyspaceName; + } + private boolean isInstall() { return environment.acceptsProfiles("install"); } diff --git a/pom.xml b/pom.xml index 7b2c5f8c34..34c0dac876 100755 --- a/pom.xml +++ b/pom.xml @@ -46,6 +46,8 @@ 18.0 3.4 1.5.0 + 2.5 + 1.4 2.8.8.1 2.2.6 2.11 @@ -564,6 +566,16 @@ commons-validator ${commons-validator.version} + + commons-io + commons-io + ${commons-io.version} + + + org.apache.commons + commons-csv + ${commons-csv.version} + com.fasterxml.jackson.core jackson-databind