diff --git a/application/src/main/data/upgrade/basic/schema_update.sql b/application/src/main/data/upgrade/basic/schema_update.sql index 31833112ad..5d66056701 100644 --- a/application/src/main/data/upgrade/basic/schema_update.sql +++ b/application/src/main/data/upgrade/basic/schema_update.sql @@ -14,202 +14,6 @@ -- limitations under the License. -- -ALTER TABLE user_credentials ADD COLUMN IF NOT EXISTS last_login_ts BIGINT; -UPDATE user_credentials c SET last_login_ts = (SELECT (additional_info::json ->> 'lastLoginTs')::bigint FROM tb_user u WHERE u.id = c.user_id) - WHERE last_login_ts IS NULL; - -ALTER TABLE user_credentials ADD COLUMN IF NOT EXISTS failed_login_attempts INT; -UPDATE user_credentials c SET failed_login_attempts = (SELECT (additional_info::json ->> 'failedLoginAttempts')::int FROM tb_user u WHERE u.id = c.user_id) - WHERE failed_login_attempts IS NULL; - -UPDATE tb_user SET additional_info = (additional_info::jsonb - 'lastLoginTs' - 'failedLoginAttempts' - 'userCredentialsEnabled')::text - WHERE additional_info IS NOT NULL AND additional_info != 'null' AND jsonb_typeof(additional_info::jsonb) = 'object'; - --- UPDATE RULE NODE DEBUG MODE TO DEBUG STRATEGY START - -ALTER TABLE rule_node ADD COLUMN IF NOT EXISTS debug_settings varchar(1024) DEFAULT null; -DO -$$ - BEGIN - IF EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'rule_node' AND column_name = 'debug_mode') - THEN - UPDATE rule_node SET debug_settings = '{"failuresEnabled": true, "allEnabledUntil": ' || cast((extract(epoch from now()) + 900) * 1000 as bigint) || '}' WHERE debug_mode = true; -- 15 minutes according to thingsboard.yml default settings. - ALTER TABLE rule_node DROP COLUMN debug_mode; - END IF; - END -$$; - --- UPDATE RULE NODE DEBUG MODE TO DEBUG STRATEGY END - - --- CREATE MOBILE APP BUNDLES FROM EXISTING APPS - -CREATE TABLE IF NOT EXISTS mobile_app_bundle ( - id uuid NOT NULL CONSTRAINT mobile_app_bundle_pkey PRIMARY KEY, - created_time bigint NOT NULL, - tenant_id uuid, - title varchar(255), - description varchar(1024), - android_app_id uuid UNIQUE, - ios_app_id uuid UNIQUE, - layout_config varchar(16384), - oauth2_enabled boolean, - CONSTRAINT fk_android_app_id FOREIGN KEY (android_app_id) REFERENCES mobile_app(id) ON DELETE SET NULL, - CONSTRAINT fk_ios_app_id FOREIGN KEY (ios_app_id) REFERENCES mobile_app(id) ON DELETE SET NULL -); -CREATE INDEX IF NOT EXISTS mobile_app_bundle_tenant_id ON mobile_app_bundle(tenant_id); - -ALTER TABLE mobile_app ADD COLUMN IF NOT EXISTS platform_type varchar(32), - ADD COLUMN IF NOT EXISTS status varchar(32), - ADD COLUMN IF NOT EXISTS version_info varchar(100000), - ADD COLUMN IF NOT EXISTS store_info varchar(16384), - DROP CONSTRAINT IF EXISTS mobile_app_pkg_name_key, - DROP CONSTRAINT IF EXISTS mobile_app_unq_key; - --- rename mobile_app_oauth2_client to mobile_app_bundle_oauth2_client -DO -$$ - BEGIN - -- in case of running the upgrade script a second time - IF EXISTS(SELECT 1 FROM information_schema.tables WHERE table_name = 'mobile_app_oauth2_client') THEN - ALTER TABLE mobile_app_oauth2_client RENAME TO mobile_app_bundle_oauth2_client; - ALTER TABLE mobile_app_bundle_oauth2_client DROP CONSTRAINT IF EXISTS fk_domain; - ALTER TABLE mobile_app_bundle_oauth2_client RENAME COLUMN mobile_app_id TO mobile_app_bundle_id; - END IF; - END; -$$; - - -CREATE EXTENSION IF NOT EXISTS "uuid-ossp"; - --- duplicate each mobile app and create mobile app bundle for the pair of android and ios app -DO -$$ - DECLARE - generatedBundleId uuid; - iosAppId uuid; - mobileAppRecord RECORD; - BEGIN - -- in case of running the upgrade script a second time - IF EXISTS(SELECT 1 FROM information_schema.columns WHERE table_name = 'mobile_app' and column_name = 'oauth2_enabled') THEN - UPDATE mobile_app SET platform_type = 'ANDROID' WHERE platform_type IS NULL; - UPDATE mobile_app SET status = 'DRAFT' WHERE mobile_app.status IS NULL; - FOR mobileAppRecord IN SELECT * FROM mobile_app - LOOP - -- duplicate app for iOS platform type - iosAppId := uuid_generate_v4(); - INSERT INTO mobile_app(id, created_time, tenant_id, pkg_name, app_secret, platform_type, status) - VALUES (iosAppId, mobileAppRecord.created_time, mobileAppRecord.tenant_id, mobileAppRecord.pkg_name, mobileAppRecord.app_secret, 'IOS', mobileAppRecord.status) - ON CONFLICT DO NOTHING; - -- create bundle for android and iOS app - generatedBundleId := uuid_generate_v4(); - INSERT INTO mobile_app_bundle(id, created_time, tenant_id, title, android_app_id, ios_app_id, oauth2_enabled) - VALUES (generatedBundleId, mobileAppRecord.created_time, mobileAppRecord.tenant_id, - mobileAppRecord.pkg_name || ' (autogenerated)', mobileAppRecord.id, iosAppId, mobileAppRecord.oauth2_enabled) - ON CONFLICT DO NOTHING; - UPDATE mobile_app_bundle_oauth2_client SET mobile_app_bundle_id = generatedBundleId WHERE mobile_app_bundle_id = mobileAppRecord.id; - END LOOP; - END IF; - IF NOT EXISTS(SELECT 1 FROM pg_constraint WHERE conname = 'fk_mobile_app_bundle_oauth2_client_bundle_id') THEN - ALTER TABLE mobile_app_bundle_oauth2_client ADD CONSTRAINT fk_mobile_app_bundle_oauth2_client_bundle_id - FOREIGN KEY (mobile_app_bundle_id) REFERENCES mobile_app_bundle(id) ON DELETE CASCADE; - END IF; - ALTER TABLE mobile_app DROP COLUMN IF EXISTS oauth2_enabled; - IF NOT EXISTS(SELECT 1 FROM pg_constraint WHERE conname = 'mobile_app_pkg_name_platform_unq_key') THEN - ALTER TABLE mobile_app ADD CONSTRAINT mobile_app_pkg_name_platform_unq_key UNIQUE (pkg_name, platform_type); - END IF; - END; -$$; - -ALTER TABLE IF EXISTS mobile_app_settings RENAME TO qr_code_settings; -ALTER TABLE qr_code_settings ADD COLUMN IF NOT EXISTS mobile_app_bundle_id uuid, - ADD COLUMN IF NOT EXISTS android_enabled boolean, - ADD COLUMN IF NOT EXISTS ios_enabled boolean; - --- migrate mobile apps from qr code settings to mobile_app, create mobile app bundle for the pair of apps -DO -$$ - DECLARE - androidPkgName varchar; - iosPkgName varchar; - androidAppId uuid; - iosAppId uuid; - generatedBundleId uuid; - qrCodeRecord RECORD; - BEGIN - -- in case of running the upgrade script a second time - IF EXISTS(SELECT 1 FROM information_schema.columns WHERE table_name = 'qr_code_settings' AND column_name = 'android_config') THEN - FOR qrCodeRecord IN SELECT * FROM qr_code_settings - LOOP - generatedBundleId := NULL; - -- migrate android config - IF (qrCodeRecord.android_config::jsonb ->> 'appPackage' IS NOT NULL) THEN - androidPkgName := qrCodeRecord.android_config::jsonb ->> 'appPackage'; - SELECT id into androidAppId FROM mobile_app WHERE pkg_name = androidPkgName AND platform_type = 'ANDROID'; - IF androidAppId IS NULL THEN - androidAppId := uuid_generate_v4(); - INSERT INTO mobile_app(id, created_time, tenant_id, pkg_name, platform_type, status, store_info) - VALUES (androidAppId, (extract(epoch from now()) * 1000), qrCodeRecord.tenant_id, - androidPkgName, 'ANDROID', 'DRAFT', qrCodeRecord.android_config::jsonb - 'appPackage' - 'enabled'); - generatedBundleId := uuid_generate_v4(); - INSERT INTO mobile_app_bundle(id, created_time, tenant_id, title, android_app_id) - VALUES (generatedBundleId, (extract(epoch from now()) * 1000), qrCodeRecord.tenant_id, androidPkgName || ' (autogenerated)', androidAppId); - UPDATE qr_code_settings SET mobile_app_bundle_id = generatedBundleId; - ELSE - UPDATE mobile_app SET store_info = qrCodeRecord.android_config::jsonb - 'appPackage' - 'enabled' WHERE id = androidAppId; - UPDATE qr_code_settings SET mobile_app_bundle_id = (SELECT id FROM mobile_app_bundle WHERE mobile_app_bundle.android_app_id = androidAppId); - END IF; - END IF; - UPDATE qr_code_settings SET android_enabled = (qrCodeRecord.android_config::jsonb ->> 'enabled')::boolean WHERE id = qrCodeRecord.id; - - -- migrate ios config - IF (qrCodeRecord.ios_config::jsonb ->> 'appId' IS NOT NULL) THEN - iosPkgName := substring(qrCodeRecord.ios_config::jsonb ->> 'appId', strpos(qrCodeRecord.ios_config::jsonb ->> 'appId', '.') + 1); - SELECT id INTO iosAppId FROM mobile_app WHERE pkg_name = iosPkgName AND platform_type = 'IOS'; - IF iosAppId IS NULL THEN - iosAppId := uuid_generate_v4(); - INSERT INTO mobile_app(id, created_time, tenant_id, pkg_name, platform_type, status, store_info) - VALUES (iosAppId, (extract(epoch from now()) * 1000), qrCodeRecord.tenant_id, - iosPkgName, 'IOS', 'DRAFT', qrCodeRecord.ios_config::jsonb - 'enabled'); - IF generatedBundleId IS NULL THEN - generatedBundleId := uuid_generate_v4(); - INSERT INTO mobile_app_bundle(id, created_time, tenant_id, title, ios_app_id) - VALUES (generatedBundleId, (extract(epoch from now()) * 1000), qrCodeRecord.tenant_id, iosPkgName || ' (autogenerated)', iosAppId); - UPDATE qr_code_settings SET mobile_app_bundle_id = generatedBundleId; - ELSE - UPDATE mobile_app_bundle SET ios_app_id = iosAppId WHERE id = generatedBundleId; - END IF; - ELSE - UPDATE qr_code_settings SET mobile_app_bundle_id = (SELECT id FROM mobile_app_bundle WHERE mobile_app_bundle.ios_app_id = iosAppId); - UPDATE mobile_app SET store_info = qrCodeRecord.ios_config::jsonb - 'enabled' WHERE id = iosAppId; - END IF; - END IF; - UPDATE qr_code_settings SET ios_enabled = (qrCodeRecord.ios_config::jsonb -> 'enabled')::boolean WHERE id = qrCodeRecord.id; - END LOOP; - ALTER TABLE qr_code_settings RENAME CONSTRAINT mobile_app_settings_tenant_id_unq_key TO qr_code_settings_tenant_id_unq_key; - ALTER TABLE qr_code_settings RENAME CONSTRAINT mobile_app_settings_pkey TO qr_code_settings_pkey; - END IF; - ALTER TABLE qr_code_settings DROP COLUMN IF EXISTS android_config, DROP COLUMN IF EXISTS ios_config; - END; -$$; - --- update constraint name -DO -$$ - BEGIN - ALTER TABLE domain DROP CONSTRAINT IF EXISTS domain_unq_key; - IF NOT EXISTS(SELECT 1 FROM pg_constraint WHERE conname = 'domain_name_key') THEN - ALTER TABLE domain ADD CONSTRAINT domain_name_key UNIQUE (name); - END IF; - END; -$$; - --- UPDATE RESOURCE JS_MODULE SUB TYPE START - -UPDATE resource SET resource_sub_type = 'EXTENSION' WHERE resource_type = 'JS_MODULE' AND resource_sub_type IS NULL; - --- UPDATE RESOURCE JS_MODULE SUB TYPE END - -- UPDATE SAVE TIME SERIES NODES START DO $$ diff --git a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java index 622c0ab2dc..ea4c059222 100644 --- a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java +++ b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java @@ -35,8 +35,6 @@ import org.thingsboard.server.service.install.migrate.TsLatestMigrateService; import org.thingsboard.server.service.install.update.CacheCleanupService; import org.thingsboard.server.service.install.update.DataUpdateService; -import static org.thingsboard.server.service.install.update.DefaultDataUpdateService.getEnv; - @Service @Profile("install") @Slf4j @@ -99,8 +97,6 @@ public class ThingsboardInstallService { if ("cassandra-latest-to-postgres".equals(upgradeFromVersion)) { log.info("Migrating ThingsBoard latest timeseries data from cassandra to SQL database ..."); latestMigrateService.migrate(); - } else if (upgradeFromVersion.equals("3.9.0-resources")) { - installScripts.updateResourcesUsage(); } else { // TODO DON'T FORGET to update SUPPORTED_VERSIONS_FROM in DefaultDatabaseSchemaSettingsService databaseSchemaVersionService.validateSchemaSettings(); @@ -118,25 +114,16 @@ public class ThingsboardInstallService { entityDatabaseSchemaService.createOrUpdateDeviceInfoView(persistToTelemetry); // Creates missing indexes. entityDatabaseSchemaService.createDatabaseIndexes(); - // Runs upgrade scripts that are not possible in plain SQL. + // TODO: cleanup update code after each release - if (!getEnv("SKIP_RESOURCES_USAGE_MIGRATION", false)) { - installScripts.setUpdateResourcesUsage(true); - } else { - log.info("Skipping resources usage migration. Run the upgrade with fromVersion as '3.9.0-resources' to migrate"); - } - if (installScripts.isUpdateResourcesUsage()) { - installScripts.updateResourcesUsage(); - } + + // Runs upgrade scripts that are not possible in plain SQL. dataUpdateService.updateData(); log.info("Updating system data..."); dataUpdateService.upgradeRuleNodes(); systemDataLoaderService.loadSystemWidgets(); installScripts.loadSystemLwm2mResources(); installScripts.loadSystemImagesAndResources(); - if (installScripts.isUpdateImages()) { - installScripts.updateImages(); - } databaseSchemaVersionService.updateSchemaVersion(); } log.info("Upgrade finished successfully!"); diff --git a/application/src/main/java/org/thingsboard/server/service/install/DefaultDatabaseSchemaSettingsService.java b/application/src/main/java/org/thingsboard/server/service/install/DefaultDatabaseSchemaSettingsService.java index 6fa8210125..3b21685d22 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/DefaultDatabaseSchemaSettingsService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/DefaultDatabaseSchemaSettingsService.java @@ -17,8 +17,6 @@ package org.thingsboard.server.service.install; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Profile; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Service; @@ -34,24 +32,16 @@ public class DefaultDatabaseSchemaSettingsService implements DatabaseSchemaSetti // This list should include all versions which are compatible for the upgrade. // The compatibility cycle usually breaks when we have some scripts written in Java that may not work after new release. - private static final List SUPPORTED_VERSIONS_FOR_UPGRADE = List.of("3.8.0", "3.8.1"); + private static final List SUPPORTED_VERSIONS_FOR_UPGRADE = List.of("3.9.0"); private final ProjectInfo projectInfo; private final JdbcTemplate jdbcTemplate; - @Value("${install.upgrade.from_version:}") - private String upgradeFromVersion; - private String packageSchemaVersion; private String schemaVersionFromDb; @Override public void validateSchemaSettings() { - //TODO: remove after release (3.9.0) - createProductIfNotExists(); - - String dbSchemaVersion = getDbSchemaVersion(); - if (DefaultDataUpdateService.getEnv("SKIP_SCHEMA_VERSION_CHECK", false)) { log.info("Skipped DB schema version check due to SKIP_SCHEMA_VERSION_CHECK set to 'true'."); return; @@ -62,6 +52,7 @@ public class DefaultDatabaseSchemaSettingsService implements DatabaseSchemaSetti onSchemaSettingsError(String.format("Upgrade failed: can't upgrade ThingsBoard %s database using ThingsBoard %s.", product, projectInfo.getProductType())); } + String dbSchemaVersion = getDbSchemaVersion(); if (dbSchemaVersion.equals(getPackageSchemaVersion())) { onSchemaSettingsError("Upgrade failed: database already upgraded to current version. You can set SKIP_SCHEMA_VERSION_CHECK to 'true' if force re-upgrade needed."); } @@ -73,14 +64,6 @@ public class DefaultDatabaseSchemaSettingsService implements DatabaseSchemaSetti } } - @Deprecated(forRemoval = true, since = "3.9.0") - private void createProductIfNotExists() { - boolean isCommunityEdition = jdbcTemplate.queryForList( - "SELECT 1 FROM information_schema.tables WHERE table_name = 'integration'", Integer.class).isEmpty(); - String product = isCommunityEdition ? "CE" : "PE"; - jdbcTemplate.execute("ALTER TABLE tb_schema_settings ADD COLUMN IF NOT EXISTS product varchar(2) DEFAULT '" + product + "'"); - } - @Override public void createSchemaSettings() { Long schemaVersion = getSchemaVersionFromDb(); @@ -105,15 +88,6 @@ public class DefaultDatabaseSchemaSettingsService implements DatabaseSchemaSetti @Override public String getDbSchemaVersion() { if (schemaVersionFromDb == null) { - if (StringUtils.isNotBlank(upgradeFromVersion)) { - /* - * TODO - Remove after the release of 3.9.0: - * This a temporary workaround due to the issue that schema version in the - * tb_schema_settings was set as 3.6.4 during the install of 3.8.1. - * */ - schemaVersionFromDb = upgradeFromVersion; - return schemaVersionFromDb; - } Long version = getSchemaVersionFromDb(); if (version == null) { onSchemaSettingsError("Upgrade failed: the database schema version is missing."); diff --git a/application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java b/application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java index a6ccb85e11..b230be0d57 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java +++ b/application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java @@ -17,8 +17,6 @@ package org.thingsboard.server.service.install; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; -import lombok.Getter; -import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; @@ -120,11 +118,6 @@ public class InstallScripts { @Autowired private ResourcesUpdater resourcesUpdater; - @Getter @Setter - private boolean updateImages = false; - - @Getter @Setter - private boolean updateResourcesUsage = false; @Autowired private ImageService imageService; @@ -395,14 +388,6 @@ public class InstallScripts { } } - public void updateImages() { - resourcesUpdater.updateWidgetsBundlesImages(); - resourcesUpdater.updateWidgetTypesImages(); - resourcesUpdater.updateDashboardsImages(); - resourcesUpdater.updateDeviceProfilesImages(); - resourcesUpdater.updateAssetProfilesImages(); - } - public void loadSystemImagesAndResources() { log.info("Loading system images and resources..."); Stream dashboardsFiles = Stream.concat(listDir(Paths.get(getDataDir(), JSON_DIR, DEMO_DIR, DASHBOARDS_DIR)), @@ -512,11 +497,6 @@ public class InstallScripts { } } - public void updateResourcesUsage() { - resourcesUpdater.updateDashboardsResources(); - resourcesUpdater.updateWidgetsResources(); - } - private void loadSystemResources(Path dir, ResourceType resourceType, ResourceSubType resourceSubType) { listDir(dir).forEach(resourceFile -> { String resourceKey = resourceFile.getFileName().toString(); diff --git a/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java b/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java index e8622fae37..cea938bce9 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java @@ -34,7 +34,6 @@ import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.dao.sql.JpaExecutorService; import org.thingsboard.server.service.component.ComponentDiscoveryService; import org.thingsboard.server.service.component.RuleNodeClassInfo; -import org.thingsboard.server.service.install.InstallScripts; import org.thingsboard.server.utils.TbNodeUpgradeUtils; import java.util.ArrayList; @@ -58,9 +57,6 @@ public class DefaultDataUpdateService implements DataUpdateService { @Autowired JpaExecutorService jpaExecutorService; - @Autowired - private InstallScripts installScripts; - @Override public void updateData() throws Exception { log.info("Updating data ...");