dendi-wang/thingsboard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactored upgrade services

Browse Source
This commit is contained in:
YevhenBondarenko 2024-11-28 15:17:10 +01:00
parent 9a90c381d1
commit b61d7b6b17
15 changed files with 82 additions and 368 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

0
application/src/main/data/upgrade/3.8.1/schema_update.sql → application/src/main/data/upgrade/basic/schema_update.sql
View File

26
application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java
View File

@ -24,7 +24,7 @@ import org.springframework.context.annotation.Profile;
import org.springframework.stereotype.Service;
import org.thingsboard.server.service.component.ComponentDiscoveryService;
import org.thingsboard.server.service.install.DatabaseEntitiesUpgradeService;
import org.thingsboard.server.service.install.DatabaseSchemaVersionService;
import org.thingsboard.server.service.install.DatabaseSchemaSettingsService;
import org.thingsboard.server.service.install.EntityDatabaseSchemaService;
import org.thingsboard.server.service.install.InstallScripts;
import org.thingsboard.server.service.install.NoSqlKeyspaceService;
@ -89,7 +89,7 @@ public class ThingsboardInstallService {
private InstallScripts installScripts;
@Autowired
private DatabaseSchemaVersionService databaseSchemaVersionService;
private DatabaseSchemaSettingsService databaseSchemaVersionService;
public void performInstall() {
try {
@ -102,21 +102,15 @@ public class ThingsboardInstallService {
} else if (upgradeFromVersion.equals("3.6.2-images")) {
installScripts.updateImages();
} else {
upgradeFromVersion = databaseSchemaVersionService.validateSchemaSettings(upgradeFromVersion);
cacheCleanupService.clearCache(upgradeFromVersion);
switch (upgradeFromVersion) {
case "3.8.0":
log.info("Upgrading ThingsBoard from version 3.8.0 to 3.8.1 ...");
case "3.8.1":
log.info("Upgrading ThingsBoard from version 3.8.1 to 3.9.0 ...");
databaseEntitiesUpgradeService.upgradeDatabase("3.8.1");
databaseSchemaVersionService.validateSchemaSettings();
//TODO DON'T FORGET to update SUPPORTED_VERSIONS_FROM in DatabaseSchemaVersionService,
// this list should include last version and can include previous versions without upgrade
String fromVersion = databaseSchemaVersionService.getDbSchemaVersion();
String toVersion = databaseSchemaVersionService.getPackageSchemaVersion();
cacheCleanupService.clearCache(fromVersion, toVersion);
databaseEntitiesUpgradeService.upgradeDatabase(fromVersion, toVersion);
// dataUpdateService.updateData(fromVersion, toVersion);
installScripts.updateResourcesUsage();
//TODO DON'T FORGET to update switch statement in the CacheCleanupService if you need to clear the cache (include previous versions without upgrade)
//TODO DON'T FORGET to update SUPPORTED_VERSIONS_FROM, this list should include last version and can include previous versions without upgrade
break;
default:
throw new RuntimeException("Unable to upgrade ThingsBoard, unsupported fromVersion: " + upgradeFromVersion);
}
entityDatabaseSchemaService.createDatabaseSchema(false);
entityDatabaseSchemaService.createOrUpdateViewsAndFunctions();

117
application/src/main/java/org/thingsboard/server/service/install/AbstractSqlTsDatabaseUpgradeService.java
View File

@ -1,117 +0,0 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.install;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.SQLWarning;
import java.sql.Statement;
@Slf4j
public abstract class AbstractSqlTsDatabaseUpgradeService {
protected static final String CALL_REGEX = "call ";
protected static final String DROP_TABLE = "DROP TABLE ";
protected static final String DROP_PROCEDURE_IF_EXISTS = "DROP PROCEDURE IF EXISTS ";
protected static final String TS_KV_SQL = "ts_kv.sql";
protected static final String PATH_TO_USERS_PUBLIC_FOLDER = "C:\\Users\\Public";
protected static final String THINGSBOARD_WINDOWS_UPGRADE_DIR = "THINGSBOARD_WINDOWS_UPGRADE_DIR";
@Value("${spring.datasource.url}")
protected String dbUrl;
@Value("${spring.datasource.username}")
protected String dbUserName;
@Value("${spring.datasource.password}")
protected String dbPassword;
@Autowired
protected InstallScripts installScripts;
protected abstract void loadSql(Connection conn, String fileName, String version);
protected void loadFunctions(Path sqlFile, Connection conn) throws Exception {
String sql = new String(Files.readAllBytes(sqlFile), StandardCharsets.UTF_8);
conn.createStatement().execute(sql); //NOSONAR, ignoring because method used to execute thingsboard database upgrade script
}
protected boolean checkVersion(Connection conn) {
boolean versionValid = false;
try {
Statement statement = conn.createStatement();
ResultSet resultSet = statement.executeQuery("SELECT current_setting('server_version_num')");
resultSet.next();
if(resultSet.getLong(1) > 110000) {
versionValid = true;
}
statement.close();
} catch (Exception e) {
log.info("Failed to check current PostgreSQL version due to: {}", e.getMessage());
}
return versionValid;
}
protected boolean isOldSchema(Connection conn, long fromVersion) {
boolean isOldSchema = true;
try {
Statement statement = conn.createStatement();
statement.execute("CREATE TABLE IF NOT EXISTS tb_schema_settings ( schema_version bigint NOT NULL, CONSTRAINT tb_schema_settings_pkey PRIMARY KEY (schema_version));");
Thread.sleep(1000);
ResultSet resultSet = statement.executeQuery("SELECT schema_version FROM tb_schema_settings;");
if (resultSet.next()) {
isOldSchema = resultSet.getLong(1) <= fromVersion;
} else {
resultSet.close();
statement.execute("INSERT INTO tb_schema_settings (schema_version) VALUES (" + fromVersion + ")");
}
statement.close();
} catch (InterruptedException | SQLException e) {
log.info("Failed to check current PostgreSQL schema due to: {}", e.getMessage());
}
return isOldSchema;
}
protected void executeQuery(Connection conn, String query) {
try {
Statement statement = conn.createStatement();
statement.execute(query); //NOSONAR, ignoring because method used to execute thingsboard database upgrade script
SQLWarning warnings = statement.getWarnings();
if (warnings != null) {
log.info("{}", warnings.getMessage());
SQLWarning nextWarning = warnings.getNextWarning();
while (nextWarning != null) {
log.info("{}", nextWarning.getMessage());
nextWarning = nextWarning.getNextWarning();
}
}
Thread.sleep(2000);
log.info("Successfully executed query: {}", query);
} catch (InterruptedException | SQLException e) {
log.error("Failed to execute query: {} due to: {}", query, e.getMessage());
throw new RuntimeException("Failed to execute query:" + query + " due to: ", e);
}
}
}

37
application/src/main/java/org/thingsboard/server/service/install/CassandraTsDatabaseUpgradeService.java
View File

@ -1,37 +0,0 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.install;
import lombok.extern.slf4j.Slf4j;
import org.springframework.context.annotation.Profile;
import org.springframework.stereotype.Service;
import org.thingsboard.server.dao.util.NoSqlTsDao;
@Service
@NoSqlTsDao
@Profile("install")
@Slf4j
public class CassandraTsDatabaseUpgradeService extends AbstractCassandraDatabaseUpgradeService implements DatabaseTsUpgradeService {
@Override
public void upgradeDatabase(String fromVersion) throws Exception {
switch (fromVersion) {
default:
throw new RuntimeException("Unable to upgrade Cassandra database, unsupported fromVersion: " + fromVersion);
}
}
}

2
application/src/main/java/org/thingsboard/server/service/install/DatabaseEntitiesUpgradeService.java
View File

@ -17,6 +17,6 @@ package org.thingsboard.server.service.install;
public interface DatabaseEntitiesUpgradeService {
void upgradeDatabase(String fromVersion) throws Exception;
void upgradeDatabase(String fromVersion, String toVersion) throws Exception;
}

8
application/src/main/java/org/thingsboard/server/service/install/DatabaseSchemaVersionService.java → application/src/main/java/org/thingsboard/server/service/install/DatabaseSchemaSettingsService.java
View File

@ -15,10 +15,14 @@
*/
package org.thingsboard.server.service.install;
public interface DatabaseSchemaVersionService {
String validateSchemaSettings(String upgradeFromVersion);
public interface DatabaseSchemaSettingsService {
void validateSchemaSettings();
void createSchemaSettings();
void updateSchemaVersion();
String getPackageSchemaVersion();
String getDbSchemaVersion();
}

22
application/src/main/java/org/thingsboard/server/service/install/DatabaseTsUpgradeService.java
View File

@ -1,22 +0,0 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.install;
public interface DatabaseTsUpgradeService {
void upgradeDatabase(String fromVersion) throws Exception;
}

90
application/src/main/java/org/thingsboard/server/service/install/DefaultDatabaseSchemaVersionService.java → application/src/main/java/org/thingsboard/server/service/install/DefaultDatabaseSchemaSettingsService.java
View File

@ -22,7 +22,6 @@ import org.springframework.context.annotation.Profile;
import org.springframework.dao.EmptyResultDataAccessException;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Service;
import org.thingsboard.server.common.data.StringUtils;
import org.thingsboard.server.service.install.update.DefaultDataUpdateService;
import java.util.List;
@ -31,7 +30,7 @@ import java.util.List;
@Profile("install")
@Slf4j
@RequiredArgsConstructor
public class DefaultDatabaseSchemaVersionService implements DatabaseSchemaVersionService {
public class DefaultDatabaseSchemaSettingsService implements DatabaseSchemaSettingsService {
private static final String CURRENT_PRODUCT = "CE";
private static final List<String> SUPPORTED_VERSIONS_FOR_UPGRADE = List.of("3.8.0", "3.8.1");
@ -39,52 +38,35 @@ public class DefaultDatabaseSchemaVersionService implements DatabaseSchemaVersio
private final BuildProperties buildProperties;
private final JdbcTemplate jdbcTemplate;
private String packageSchemaVersion;
private String schemaVersionFromDb;
@Override
public String validateSchemaSettings(String upgradeFromVersion) {
public void validateSchemaSettings() {
//TODO: remove after release (3.9.0)
createProductIfNotExists();
if (StringUtils.isNotEmpty(upgradeFromVersion) && DefaultDataUpdateService.getEnv("SKIP_SCHEMA_VERSION_CHECK", false)) {
String dbSchemaVersion = getDbSchemaVersion();
if (DefaultDataUpdateService.getEnv("SKIP_SCHEMA_VERSION_CHECK", false)) {
log.info("Skipped DB schema version check due to SKIP_SCHEMA_VERSION_CHECK set to 'true'.");
return upgradeFromVersion;
return;
}
if (dbSchemaVersion.equals(getPackageSchemaVersion())) {
onSchemaSettingsError("Upgrade failed: database already upgraded to current version. You can set SKIP_SCHEMA_VERSION_CHECK to 'true' if force re-upgrade needed.");
}
if (!SUPPORTED_VERSIONS_FOR_UPGRADE.contains(dbSchemaVersion)) {
onSchemaSettingsError(String.format("Upgrade failed: database version '%s' is not supported for upgrade. Supported versions are: %s.",
dbSchemaVersion, SUPPORTED_VERSIONS_FOR_UPGRADE
));
}
String product = getProductFromDb();
if (!CURRENT_PRODUCT.equals(product)) {
onSchemaSettingsError(String.format("Upgrade failed: can't upgrade ThingsBoard %s database using ThingsBoard %s.", product, CURRENT_PRODUCT));
}
Long schemaVersionFromDb = getSchemaVersionFromDb();
if (schemaVersionFromDb == null) {
onSchemaSettingsError("Upgrade failed: the database schema version is missing.");
}
long currentSchemaVersion = getCurrentSchemaVersion();
if (currentSchemaVersion == schemaVersionFromDb) {
onSchemaSettingsError("Upgrade failed: database already upgraded to current version. You can set SKIP_SCHEMA_VERSION_CHECK to 'true' if force re-upgrade needed.");
}
long major = schemaVersionFromDb / 1000000;
long minor = (schemaVersionFromDb % 1000000) / 1000;
long patch = schemaVersionFromDb % 1000;
String currentSchemaVersionFromDb = major + "." + minor + "." + patch;
if (!SUPPORTED_VERSIONS_FOR_UPGRADE.contains(currentSchemaVersionFromDb)) {
onSchemaSettingsError(String.format("Upgrade failed: database version '%s' is not supported for upgrade. Supported versions are: %s.",
currentSchemaVersionFromDb, SUPPORTED_VERSIONS_FOR_UPGRADE
));
}
if (StringUtils.isEmpty(upgradeFromVersion)) {
upgradeFromVersion = currentSchemaVersionFromDb;
} else if (!SUPPORTED_VERSIONS_FOR_UPGRADE.contains(upgradeFromVersion)) {
onSchemaSettingsError(String.format("Upgrade failed: 'versionFrom' '%s' is not supported for upgrade. Supported versions are: %s.",
upgradeFromVersion, SUPPORTED_VERSIONS_FOR_UPGRADE));
}
return upgradeFromVersion;
}
@Deprecated(forRemoval = true, since = "3.9.0")
@ -104,13 +86,39 @@ public class DefaultDatabaseSchemaVersionService implements DatabaseSchemaVersio
public void createSchemaSettings() {
Long schemaVersion = getSchemaVersionFromDb();
if (schemaVersion == null) {
jdbcTemplate.execute("INSERT INTO tb_schema_settings (schema_version, product) VALUES (" + getCurrentSchemaVersion() + ", '" + CURRENT_PRODUCT + "')");
jdbcTemplate.execute("INSERT INTO tb_schema_settings (schema_version, product) VALUES (" + getPackageSchemaVersionForDb() + ", '" + CURRENT_PRODUCT + "')");
}
}
@Override
public void updateSchemaVersion() {
jdbcTemplate.execute("UPDATE tb_schema_settings SET schema_version = " + getCurrentSchemaVersion());
jdbcTemplate.execute("UPDATE tb_schema_settings SET schema_version = " + getPackageSchemaVersionForDb());
}
@Override
public String getPackageSchemaVersion() {
if (packageSchemaVersion == null) {
packageSchemaVersion = buildProperties.getVersion().replaceAll("[^\\d.]", "");
}
return packageSchemaVersion;
}
@Override
public String getDbSchemaVersion() {
if (schemaVersionFromDb == null) {
Long version = getSchemaVersionFromDb();
if (version == null) {
onSchemaSettingsError("Upgrade failed: the database schema version is missing.");
}
@SuppressWarnings("DataFlowIssue")
long major = version / 1000000;
long minor = (version % 1000000) / 1000;
long patch = version % 1000;
schemaVersionFromDb = major + "." + minor + "." + patch;
}
return schemaVersionFromDb;
}
private Long getSchemaVersionFromDb() {
@ -121,8 +129,8 @@ public class DefaultDatabaseSchemaVersionService implements DatabaseSchemaVersio
return jdbcTemplate.queryForList("SELECT product FROM tb_schema_settings", String.class).stream().findFirst().orElse(null);
}
private long getCurrentSchemaVersion() {
String[] versionParts = buildProperties.getVersion().replaceAll("[^\\d.]", "").split("\\.");
private long getPackageSchemaVersionForDb() {
String[] versionParts = getPackageSchemaVersion().split("\\.");
long major = Integer.parseInt(versionParts[0]);
long minor = Integer.parseInt(versionParts[1]);

9
application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java
View File

@ -51,11 +51,10 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService
}
@Override
public void upgradeDatabase(String fromVersion) {
switch (fromVersion) {
case "3.8.1" -> loadSql(getSchemaUpdateFile(fromVersion));
default -> throw new RuntimeException("Unsupported fromVersion '" + fromVersion + "'");
}
public void upgradeDatabase(String fromVersion, String toVersion) {
log.info("Updating schema from version {} to {} ...", fromVersion, toVersion);
loadSql(getSchemaUpdateFile("basic"));
log.info("Schema updated.");
}
private Path getSchemaUpdateFile(String version) {

51
application/src/main/java/org/thingsboard/server/service/install/SqlTsDatabaseUpgradeService.java
View File

@ -1,51 +0,0 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.install;
import lombok.extern.slf4j.Slf4j;
import org.springframework.context.annotation.Profile;
import org.springframework.stereotype.Service;
import org.thingsboard.server.dao.util.SqlTsDao;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.sql.Connection;
@Service
@Profile("install")
@Slf4j
@SqlTsDao
public class SqlTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgradeService implements DatabaseTsUpgradeService {
@Override
public void upgradeDatabase(String fromVersion) throws Exception {
switch (fromVersion) {
default:
throw new RuntimeException("Unable to upgrade SQL database, unsupported fromVersion: " + fromVersion);
}
}
@Override
protected void loadSql(Connection conn, String fileName, String version) {
Path schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", version, fileName);
try {
loadFunctions(schemaUpdateFile, conn);
log.info("Functions successfully loaded!");
} catch (Exception e) {
log.info("Failed to load PostgreSQL upgrade functions due to: {}", e.getMessage());
}
}
}

55
application/src/main/java/org/thingsboard/server/service/install/TimescaleTsDatabaseUpgradeService.java
View File

@ -1,55 +0,0 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.install;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Profile;
import org.springframework.stereotype.Service;
import org.thingsboard.server.dao.util.TimescaleDBTsDao;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.sql.Connection;
@Service
@Profile("install")
@Slf4j
@TimescaleDBTsDao
public class TimescaleTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgradeService implements DatabaseTsUpgradeService {
@Autowired
private InstallScripts installScripts;
@Override
public void upgradeDatabase(String fromVersion) throws Exception {
switch (fromVersion) {
default:
throw new RuntimeException("Unable to upgrade SQL database, unsupported fromVersion: " + fromVersion);
}
}
@Override
protected void loadSql(Connection conn, String fileName, String version) {
Path schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", version, fileName);
try {
loadFunctions(schemaUpdateFile, conn);
log.info("Functions successfully loaded!");
} catch (Exception e) {
log.info("Failed to load PostgreSQL upgrade functions due to: {}", e.getMessage());
}
}
}

2
application/src/main/java/org/thingsboard/server/service/install/update/CacheCleanupService.java
View File

@ -17,6 +17,6 @@ package org.thingsboard.server.service.install.update;
public interface CacheCleanupService {
void clearCache(String fromVersion) throws Exception;
void clearCache(String from, String to) throws Exception;
}

2
application/src/main/java/org/thingsboard/server/service/install/update/DataUpdateService.java
View File

@ -17,7 +17,7 @@ package org.thingsboard.server.service.install.update;
public interface DataUpdateService {
void updateData(String fromVersion) throws Exception;
void updateData(String fromVersion, String toVersion) throws Exception;
void upgradeRuleNodes();
}

11
application/src/main/java/org/thingsboard/server/service/install/update/DefaultCacheCleanupService.java
View File

@ -42,16 +42,9 @@ public class DefaultCacheCleanupService implements CacheCleanupService {
* to discover which tables were changed
* */
@Override
public void clearCache(String fromVersion) throws Exception {
switch (fromVersion) {
case "3.8.0":
case "3.8.1":
log.info("Clearing cache to upgrade from version {} to 3.9.0", fromVersion);
public void clearCache(String from, String to) throws Exception {
log.info("Clearing cache to upgrade from version {} to {}", from, to);
clearAllCaches();
break;
default:
//Do nothing, since cache cleanup is optional.
}
}
void clearAllCaches() {

8
application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java
View File

@ -58,11 +58,9 @@ public class DefaultDataUpdateService implements DataUpdateService {
JpaExecutorService jpaExecutorService;
@Override
public void updateData(String fromVersion) throws Exception {
switch (fromVersion) {
default:
throw new RuntimeException("Unable to update data, unsupported fromVersion: " + fromVersion);
}
public void updateData(String fromVersion, String toVersion) throws Exception {
log.info("Updating data from version {} to {} ...", fromVersion, toVersion);
log.info("Data updated.");
}
@Override