dendi-wang/thingsboard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

psql & timescale upgrade (#2748)

Browse Source 
* update psql & timescale update

* fix windows upgrade
This commit is contained in:
ShvaykaD 2020-05-11 11:19:01 +03:00 committed by GitHub
parent bedc624e09
commit 8d0d23e620
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 161 additions and 114 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

80
application/src/main/data/upgrade/2.4.3/schema_update_psql_ts.sql
View File

@ -164,25 +164,24 @@ BEGIN
END; END;
$$; $$;
-- call insert_into_ts_kv(); CREATE OR REPLACE FUNCTION to_uuid(IN entity_id varchar, OUT uuid_id uuid) AS
$$
BEGIN
uuid_id := substring(entity_id, 8, 8) || '-' || substring(entity_id, 4, 4) || '-1' || substring(entity_id, 1, 3) ||
'-' || substring(entity_id, 16, 4) || '-' || substring(entity_id, 20, 12);
END;
$$ LANGUAGE plpgsql;
CREATE OR REPLACE PROCEDURE insert_into_ts_kv() LANGUAGE plpgsql AS $$ CREATE OR REPLACE PROCEDURE insert_into_ts_kv(IN path_to_file varchar) LANGUAGE plpgsql AS $$
DECLARE BEGIN
insert_size CONSTANT integer := 10000; EXECUTE format ('COPY (SELECT to_uuid(entity_id) AS entity_id,
insert_counter integer DEFAULT 0;
insert_record RECORD;
insert_cursor CURSOR FOR SELECT CONCAT(entity_id_uuid_first_part, '-', entity_id_uuid_second_part, '-1', entity_id_uuid_third_part, '-', entity_id_uuid_fourth_part, '-', entity_id_uuid_fifth_part)::uuid AS entity_id,
ts_kv_records.key AS key, ts_kv_records.key AS key,
ts_kv_records.ts AS ts, ts_kv_records.ts AS ts,
ts_kv_records.bool_v AS bool_v, ts_kv_records.bool_v AS bool_v,
ts_kv_records.str_v AS str_v, ts_kv_records.str_v AS str_v,
ts_kv_records.long_v AS long_v, ts_kv_records.long_v AS long_v,
ts_kv_records.dbl_v AS dbl_v ts_kv_records.dbl_v AS dbl_v
FROM (SELECT SUBSTRING(entity_id, 8, 8) AS entity_id_uuid_first_part, FROM (SELECT entity_id AS entity_id,
SUBSTRING(entity_id, 4, 4) AS entity_id_uuid_second_part,
SUBSTRING(entity_id, 1, 3) AS entity_id_uuid_third_part,
SUBSTRING(entity_id, 16, 4) AS entity_id_uuid_fourth_part,
SUBSTRING(entity_id, 20) AS entity_id_uuid_fifth_part,
key_id AS key, key_id AS key,
ts, ts,
bool_v, bool_v,
@ -190,46 +189,23 @@ DECLARE
long_v, long_v,
dbl_v dbl_v
FROM ts_kv_old FROM ts_kv_old
INNER JOIN ts_kv_dictionary ON (ts_kv_old.key = ts_kv_dictionary.key)) AS ts_kv_records; INNER JOIN ts_kv_dictionary ON (ts_kv_old.key = ts_kv_dictionary.key)) AS ts_kv_records) TO %L;', path_to_file);
BEGIN EXECUTE format ('COPY ts_kv FROM %L', path_to_file);
OPEN insert_cursor; END
LOOP
insert_counter := insert_counter + 1;
FETCH insert_cursor INTO insert_record;
IF NOT FOUND THEN
RAISE NOTICE '% records have been inserted into the partitioned ts_kv!',insert_counter - 1;
EXIT;
END IF;
INSERT INTO ts_kv(entity_id, key, ts, bool_v, str_v, long_v, dbl_v)
VALUES (insert_record.entity_id, insert_record.key, insert_record.ts, insert_record.bool_v, insert_record.str_v,
insert_record.long_v, insert_record.dbl_v);
IF MOD(insert_counter, insert_size) = 0 THEN
RAISE NOTICE '% records have been inserted into the partitioned ts_kv!',insert_counter;
END IF;
END LOOP;
CLOSE insert_cursor;
END;
$$; $$;
-- call insert_into_ts_kv_latest(); -- call insert_into_ts_kv_latest();
CREATE OR REPLACE PROCEDURE insert_into_ts_kv_latest() LANGUAGE plpgsql AS $$ CREATE OR REPLACE PROCEDURE insert_into_ts_kv_latest(IN path_to_file varchar) LANGUAGE plpgsql AS $$
DECLARE BEGIN
insert_size CONSTANT integer := 10000; EXECUTE format ('COPY (SELECT to_uuid(entity_id) AS entity_id,
insert_counter integer DEFAULT 0;
insert_record RECORD;
insert_cursor CURSOR FOR SELECT CONCAT(entity_id_uuid_first_part, '-', entity_id_uuid_second_part, '-1', entity_id_uuid_third_part, '-', entity_id_uuid_fourth_part, '-', entity_id_uuid_fifth_part)::uuid AS entity_id,
ts_kv_latest_records.key AS key, ts_kv_latest_records.key AS key,
ts_kv_latest_records.ts AS ts, ts_kv_latest_records.ts AS ts,
ts_kv_latest_records.bool_v AS bool_v, ts_kv_latest_records.bool_v AS bool_v,
ts_kv_latest_records.str_v AS str_v, ts_kv_latest_records.str_v AS str_v,
ts_kv_latest_records.long_v AS long_v, ts_kv_latest_records.long_v AS long_v,
ts_kv_latest_records.dbl_v AS dbl_v ts_kv_latest_records.dbl_v AS dbl_v
FROM (SELECT SUBSTRING(entity_id, 8, 8) AS entity_id_uuid_first_part, FROM (SELECT entity_id AS entity_id,
SUBSTRING(entity_id, 4, 4) AS entity_id_uuid_second_part,
SUBSTRING(entity_id, 1, 3) AS entity_id_uuid_third_part,
SUBSTRING(entity_id, 16, 4) AS entity_id_uuid_fourth_part,
SUBSTRING(entity_id, 20) AS entity_id_uuid_fifth_part,
key_id AS key, key_id AS key,
ts, ts,
bool_v, bool_v,
@ -237,24 +213,8 @@ DECLARE
long_v, long_v,
dbl_v dbl_v
FROM ts_kv_latest_old FROM ts_kv_latest_old
INNER JOIN ts_kv_dictionary ON (ts_kv_latest_old.key = ts_kv_dictionary.key)) AS ts_kv_latest_records; INNER JOIN ts_kv_dictionary ON (ts_kv_latest_old.key = ts_kv_dictionary.key)) AS ts_kv_latest_records) TO %L;', path_to_file);
BEGIN EXECUTE format ('COPY ts_kv_latest FROM %L', path_to_file);
OPEN insert_cursor;
LOOP
insert_counter := insert_counter + 1;
FETCH insert_cursor INTO insert_record;
IF NOT FOUND THEN
RAISE NOTICE '% records have been inserted into the ts_kv_latest!',insert_counter - 1;
EXIT;
END IF;
INSERT INTO ts_kv_latest(entity_id, key, ts, bool_v, str_v, long_v, dbl_v)
VALUES (insert_record.entity_id, insert_record.key, insert_record.ts, insert_record.bool_v, insert_record.str_v,
insert_record.long_v, insert_record.dbl_v);
IF MOD(insert_counter, insert_size) = 0 THEN
RAISE NOTICE '% records have been inserted into the ts_kv_latest!',insert_counter;
END IF;
END LOOP;
CLOSE insert_cursor;
END; END;
$$; $$;

69
application/src/main/data/upgrade/2.4.3/schema_update_timescale_ts.sql
View File

@ -96,51 +96,36 @@ BEGIN
END; END;
$$; $$;
CREATE OR REPLACE FUNCTION to_uuid(IN entity_id varchar, OUT uuid_id uuid) AS
$$
BEGIN
uuid_id := substring(entity_id, 8, 8) || '-' || substring(entity_id, 4, 4) || '-1' || substring(entity_id, 1, 3) ||
'-' || substring(entity_id, 16, 4) || '-' || substring(entity_id, 20, 12);
END;
$$ LANGUAGE plpgsql;
-- call insert_into_ts_kv(); -- call insert_into_ts_kv();
CREATE OR REPLACE PROCEDURE insert_into_ts_kv() LANGUAGE plpgsql AS $$ CREATE OR REPLACE PROCEDURE insert_into_ts_kv(IN path_to_file varchar) LANGUAGE plpgsql AS $$
DECLARE
insert_size CONSTANT integer := 10000;
insert_counter integer DEFAULT 0;
insert_record RECORD;
insert_cursor CURSOR FOR SELECT CONCAT(entity_id_uuid_first_part, '-', entity_id_uuid_second_part, '-1', entity_id_uuid_third_part, '-', entity_id_uuid_fourth_part, '-', entity_id_uuid_fifth_part)::uuid AS entity_id,
new_ts_kv_records.key AS key,
new_ts_kv_records.ts AS ts,
new_ts_kv_records.bool_v AS bool_v,
new_ts_kv_records.str_v AS str_v,
new_ts_kv_records.long_v AS long_v,
new_ts_kv_records.dbl_v AS dbl_v
FROM (SELECT SUBSTRING(entity_id, 8, 8) AS entity_id_uuid_first_part,
SUBSTRING(entity_id, 4, 4) AS entity_id_uuid_second_part,
SUBSTRING(entity_id, 1, 3) AS entity_id_uuid_third_part,
SUBSTRING(entity_id, 16, 4) AS entity_id_uuid_fourth_part,
SUBSTRING(entity_id, 20) AS entity_id_uuid_fifth_part,
key_id AS key,
ts,
bool_v,
str_v,
long_v,
dbl_v
FROM tenant_ts_kv_old
INNER JOIN ts_kv_dictionary ON (tenant_ts_kv_old.key = ts_kv_dictionary.key)) AS new_ts_kv_records;
BEGIN BEGIN
OPEN insert_cursor;
LOOP EXECUTE format ('COPY (SELECT to_uuid(entity_id) AS entity_id,
insert_counter := insert_counter + 1; new_ts_kv_records.key AS key,
FETCH insert_cursor INTO insert_record; new_ts_kv_records.ts AS ts,
IF NOT FOUND THEN new_ts_kv_records.bool_v AS bool_v,
RAISE NOTICE '% records have been inserted into the new ts_kv table!',insert_counter - 1; new_ts_kv_records.str_v AS str_v,
EXIT; new_ts_kv_records.long_v AS long_v,
END IF; new_ts_kv_records.dbl_v AS dbl_v
INSERT INTO ts_kv(entity_id, key, ts, bool_v, str_v, long_v, dbl_v) FROM (SELECT entity_id AS entity_id,
VALUES (insert_record.entity_id, insert_record.key, insert_record.ts, insert_record.bool_v, insert_record.str_v, key_id AS key,
insert_record.long_v, insert_record.dbl_v); ts,
IF MOD(insert_counter, insert_size) = 0 THEN bool_v,
RAISE NOTICE '% records have been inserted into the new ts_kv table!',insert_counter; str_v,
END IF; long_v,
END LOOP; dbl_v
CLOSE insert_cursor; FROM tenant_ts_kv_old
INNER JOIN ts_kv_dictionary ON (tenant_ts_kv_old.key = ts_kv_dictionary.key)) AS new_ts_kv_records) TO %L;', path_to_file);
EXECUTE format ('COPY ts_kv FROM %L', path_to_file);
END; END;
$$; $$;

3
application/src/main/java/org/thingsboard/server/service/install/AbstractSqlTsDatabaseUpgradeService.java
View File

@ -34,6 +34,9 @@ public abstract class AbstractSqlTsDatabaseUpgradeService {
protected static final String CALL_REGEX = "call "; protected static final String CALL_REGEX = "call ";
protected static final String DROP_TABLE = "DROP TABLE "; protected static final String DROP_TABLE = "DROP TABLE ";
protected static final String DROP_PROCEDURE_IF_EXISTS = "DROP PROCEDURE IF EXISTS "; protected static final String DROP_PROCEDURE_IF_EXISTS = "DROP PROCEDURE IF EXISTS ";
protected static final String TS_KV_SQL = "ts_kv.sql";
protected static final String PATH_TO_USERS_PUBLIC_FOLDER = "C:\\Users\\Public";
protected static final String THINGSBOARD_WINDOWS_UPGRADE_DIR = "THINGSBOARD_WINDOWS_UPGRADE_DIR";
@Value("${spring.datasource.url}") @Value("${spring.datasource.url}")
protected String dbUrl; protected String dbUrl;

67
application/src/main/java/org/thingsboard/server/service/install/PsqlTsDatabaseUpgradeService.java
View File

@ -16,12 +16,17 @@
package org.thingsboard.server.service.install; package org.thingsboard.server.service.install;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.SystemUtils;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Profile; import org.springframework.context.annotation.Profile;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.thingsboard.server.dao.util.PsqlDao; import org.thingsboard.server.dao.util.PsqlDao;
import org.thingsboard.server.dao.util.SqlTsDao; import org.thingsboard.server.dao.util.SqlTsDao;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
import java.nio.file.Paths; import java.nio.file.Paths;
import java.sql.Connection; import java.sql.Connection;
@ -37,6 +42,7 @@ public class PsqlTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgradeSe
@Value("${sql.postgres.ts_key_value_partitioning:MONTHS}") @Value("${sql.postgres.ts_key_value_partitioning:MONTHS}")
private String partitionType; private String partitionType;
private static final String TS_KV_LATEST_SQL = "ts_kv_latest.sql";
private static final String LOAD_FUNCTIONS_SQL = "schema_update_psql_ts.sql"; private static final String LOAD_FUNCTIONS_SQL = "schema_update_psql_ts.sql";
private static final String LOAD_TTL_FUNCTIONS_SQL = "schema_update_ttl.sql"; private static final String LOAD_TTL_FUNCTIONS_SQL = "schema_update_ttl.sql";
private static final String LOAD_DROP_PARTITIONS_FUNCTIONS_SQL = "schema_update_psql_drop_partitions.sql"; private static final String LOAD_DROP_PARTITIONS_FUNCTIONS_SQL = "schema_update_psql_drop_partitions.sql";
@ -49,15 +55,13 @@ public class PsqlTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgradeSe
private static final String CREATE_PARTITIONS = "create_partitions(IN partition_type varchar)"; private static final String CREATE_PARTITIONS = "create_partitions(IN partition_type varchar)";
private static final String CREATE_TS_KV_DICTIONARY_TABLE = "create_ts_kv_dictionary_table()"; private static final String CREATE_TS_KV_DICTIONARY_TABLE = "create_ts_kv_dictionary_table()";
private static final String INSERT_INTO_DICTIONARY = "insert_into_dictionary()"; private static final String INSERT_INTO_DICTIONARY = "insert_into_dictionary()";
private static final String INSERT_INTO_TS_KV = "insert_into_ts_kv()"; private static final String INSERT_INTO_TS_KV = "insert_into_ts_kv(IN path_to_file varchar)";
private static final String INSERT_INTO_TS_KV_LATEST = "insert_into_ts_kv_latest()"; private static final String INSERT_INTO_TS_KV_LATEST = "insert_into_ts_kv_latest(IN path_to_file varchar)";
private static final String CALL_CREATE_PARTITION_TS_KV_TABLE = CALL_REGEX + CREATE_PARTITION_TS_KV_TABLE; private static final String CALL_CREATE_PARTITION_TS_KV_TABLE = CALL_REGEX + CREATE_PARTITION_TS_KV_TABLE;
private static final String CALL_CREATE_NEW_TS_KV_LATEST_TABLE = CALL_REGEX + CREATE_NEW_TS_KV_LATEST_TABLE; private static final String CALL_CREATE_NEW_TS_KV_LATEST_TABLE = CALL_REGEX + CREATE_NEW_TS_KV_LATEST_TABLE;
private static final String CALL_CREATE_TS_KV_DICTIONARY_TABLE = CALL_REGEX + CREATE_TS_KV_DICTIONARY_TABLE; private static final String CALL_CREATE_TS_KV_DICTIONARY_TABLE = CALL_REGEX + CREATE_TS_KV_DICTIONARY_TABLE;
private static final String CALL_INSERT_INTO_DICTIONARY = CALL_REGEX + INSERT_INTO_DICTIONARY; private static final String CALL_INSERT_INTO_DICTIONARY = CALL_REGEX + INSERT_INTO_DICTIONARY;
private static final String CALL_INSERT_INTO_TS_KV = CALL_REGEX + INSERT_INTO_TS_KV;
private static final String CALL_INSERT_INTO_TS_KV_LATEST = CALL_REGEX + INSERT_INTO_TS_KV_LATEST;
private static final String DROP_TABLE_TS_KV_OLD = DROP_TABLE + TS_KV_OLD; private static final String DROP_TABLE_TS_KV_OLD = DROP_TABLE + TS_KV_OLD;
private static final String DROP_TABLE_TS_KV_LATEST_OLD = DROP_TABLE + TS_KV_LATEST_OLD; private static final String DROP_TABLE_TS_KV_LATEST_OLD = DROP_TABLE + TS_KV_LATEST_OLD;
@ -94,9 +98,58 @@ public class PsqlTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgradeSe
} }
executeQuery(conn, CALL_CREATE_TS_KV_DICTIONARY_TABLE); executeQuery(conn, CALL_CREATE_TS_KV_DICTIONARY_TABLE);
executeQuery(conn, CALL_INSERT_INTO_DICTIONARY); executeQuery(conn, CALL_INSERT_INTO_DICTIONARY);
executeQuery(conn, CALL_INSERT_INTO_TS_KV);
executeQuery(conn, CALL_CREATE_NEW_TS_KV_LATEST_TABLE); Path pathToTempTsKvFile;
executeQuery(conn, CALL_INSERT_INTO_TS_KV_LATEST); Path pathToTempTsKvLatestFile;
if (SystemUtils.IS_OS_WINDOWS) {
log.info("Lookup for environment variable: " + THINGSBOARD_WINDOWS_UPGRADE_DIR + " ...");
Path pathToDir;
String thingsboardWindowsUpgradeDir = System.getenv("THINGSBOARD_WINDOWS_UPGRADE_DIR");
if (StringUtils.isNotEmpty(thingsboardWindowsUpgradeDir)) {
log.info("Environment variable: " + THINGSBOARD_WINDOWS_UPGRADE_DIR + " found!");
pathToDir = Paths.get(thingsboardWindowsUpgradeDir);
} else {
log.info("Failed to lookup environment variable: " + THINGSBOARD_WINDOWS_UPGRADE_DIR);
pathToDir = Paths.get(PATH_TO_USERS_PUBLIC_FOLDER);
}
log.info("Directory: " + pathToDir + " will be used for creation temporary upgrade files!");
try {
Path tsKvFile = Files.createTempFile(pathToDir, "ts_kv", ".sql");
Path tsKvLatestFile = Files.createTempFile(pathToDir, "ts_kv_latest", ".sql");
pathToTempTsKvFile = tsKvFile.toAbsolutePath();
pathToTempTsKvLatestFile = tsKvLatestFile.toAbsolutePath();
executeQuery(conn, "call insert_into_ts_kv('" + pathToTempTsKvFile + "')");
executeQuery(conn, CALL_CREATE_NEW_TS_KV_LATEST_TABLE);
executeQuery(conn, "call insert_into_ts_kv_latest('" + pathToTempTsKvLatestFile + "');");
} catch (IOException | SecurityException e) {
throw new RuntimeException("Failed to create time-series upgrade files due to: " + e);
}
} else {
Path tempDirPath = Files.createTempDirectory("ts_kv");
File tempDirAsFile = tempDirPath.toFile();
boolean writable = tempDirAsFile.setWritable(true, false);
boolean readable = tempDirAsFile.setReadable(true, false);
boolean executable = tempDirAsFile.setExecutable(true, false);
if (writable && readable && executable) {
pathToTempTsKvFile = tempDirPath.resolve(TS_KV_SQL).toAbsolutePath();
pathToTempTsKvLatestFile = tempDirPath.resolve(TS_KV_LATEST_SQL).toAbsolutePath();
executeQuery(conn, "call insert_into_ts_kv('" + pathToTempTsKvFile + "')");
executeQuery(conn, CALL_CREATE_NEW_TS_KV_LATEST_TABLE);
executeQuery(conn, "call insert_into_ts_kv_latest('" + pathToTempTsKvLatestFile + "');");
} else {
throw new RuntimeException("Failed to grant write permissions for the: " + tempDirPath + "folder!");
}
}
if (pathToTempTsKvFile.toFile().exists() && pathToTempTsKvLatestFile.toFile().exists()) {
boolean deleteTsKvFile = pathToTempTsKvFile.toFile().delete();
if (deleteTsKvFile) {
log.info("Successfully deleted the temp file for ts_kv table upgrade!");
}
boolean deleteTsKvLatestFile = pathToTempTsKvLatestFile.toFile().delete();
if (deleteTsKvLatestFile) {
log.info("Successfully deleted the temp file for ts_kv_latest table upgrade!");
}
}
executeQuery(conn, DROP_TABLE_TS_KV_OLD); executeQuery(conn, DROP_TABLE_TS_KV_OLD);
executeQuery(conn, DROP_TABLE_TS_KV_LATEST_OLD); executeQuery(conn, DROP_TABLE_TS_KV_LATEST_OLD);

56
application/src/main/java/org/thingsboard/server/service/install/TimescaleTsDatabaseUpgradeService.java
View File

@ -16,6 +16,8 @@
package org.thingsboard.server.service.install; package org.thingsboard.server.service.install;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.SystemUtils;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Profile; import org.springframework.context.annotation.Profile;
@ -23,6 +25,9 @@ import org.springframework.stereotype.Service;
import org.thingsboard.server.dao.util.PsqlDao; import org.thingsboard.server.dao.util.PsqlDao;
import org.thingsboard.server.dao.util.TimescaleDBTsDao; import org.thingsboard.server.dao.util.TimescaleDBTsDao;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
import java.nio.file.Paths; import java.nio.file.Paths;
import java.sql.Connection; import java.sql.Connection;
@ -47,14 +52,13 @@ public class TimescaleTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgr
private static final String CREATE_NEW_TS_KV_TABLE = "create_new_ts_kv_table()"; private static final String CREATE_NEW_TS_KV_TABLE = "create_new_ts_kv_table()";
private static final String CREATE_TS_KV_DICTIONARY_TABLE = "create_ts_kv_dictionary_table()"; private static final String CREATE_TS_KV_DICTIONARY_TABLE = "create_ts_kv_dictionary_table()";
private static final String INSERT_INTO_DICTIONARY = "insert_into_dictionary()"; private static final String INSERT_INTO_DICTIONARY = "insert_into_dictionary()";
private static final String INSERT_INTO_TS_KV = "insert_into_ts_kv()"; private static final String INSERT_INTO_TS_KV = "insert_into_ts_kv(IN path_to_file varchar)";
private static final String INSERT_INTO_TS_KV_LATEST = "insert_into_ts_kv_latest()"; private static final String INSERT_INTO_TS_KV_LATEST = "insert_into_ts_kv_latest()";
private static final String CALL_CREATE_TS_KV_LATEST_TABLE = CALL_REGEX + CREATE_TS_KV_LATEST_TABLE; private static final String CALL_CREATE_TS_KV_LATEST_TABLE = CALL_REGEX + CREATE_TS_KV_LATEST_TABLE;
private static final String CALL_CREATE_NEW_TENANT_TS_KV_TABLE = CALL_REGEX + CREATE_NEW_TS_KV_TABLE; private static final String CALL_CREATE_NEW_TENANT_TS_KV_TABLE = CALL_REGEX + CREATE_NEW_TS_KV_TABLE;
private static final String CALL_CREATE_TS_KV_DICTIONARY_TABLE = CALL_REGEX + CREATE_TS_KV_DICTIONARY_TABLE; private static final String CALL_CREATE_TS_KV_DICTIONARY_TABLE = CALL_REGEX + CREATE_TS_KV_DICTIONARY_TABLE;
private static final String CALL_INSERT_INTO_DICTIONARY = CALL_REGEX + INSERT_INTO_DICTIONARY; private static final String CALL_INSERT_INTO_DICTIONARY = CALL_REGEX + INSERT_INTO_DICTIONARY;
private static final String CALL_INSERT_INTO_TS_KV = CALL_REGEX + INSERT_INTO_TS_KV;
private static final String CALL_INSERT_INTO_TS_KV_LATEST = CALL_REGEX + INSERT_INTO_TS_KV_LATEST; private static final String CALL_INSERT_INTO_TS_KV_LATEST = CALL_REGEX + INSERT_INTO_TS_KV_LATEST;
private static final String DROP_OLD_TENANT_TS_KV_TABLE = DROP_TABLE + TENANT_TS_KV_OLD_TABLE; private static final String DROP_OLD_TENANT_TS_KV_TABLE = DROP_TABLE + TENANT_TS_KV_OLD_TABLE;
@ -63,7 +67,7 @@ public class TimescaleTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgr
private static final String DROP_PROCEDURE_CREATE_TENANT_TS_KV_TABLE_COPY = DROP_PROCEDURE_IF_EXISTS + CREATE_NEW_TS_KV_TABLE; private static final String DROP_PROCEDURE_CREATE_TENANT_TS_KV_TABLE_COPY = DROP_PROCEDURE_IF_EXISTS + CREATE_NEW_TS_KV_TABLE;
private static final String DROP_PROCEDURE_CREATE_TS_KV_DICTIONARY_TABLE = DROP_PROCEDURE_IF_EXISTS + CREATE_TS_KV_DICTIONARY_TABLE; private static final String DROP_PROCEDURE_CREATE_TS_KV_DICTIONARY_TABLE = DROP_PROCEDURE_IF_EXISTS + CREATE_TS_KV_DICTIONARY_TABLE;
private static final String DROP_PROCEDURE_INSERT_INTO_DICTIONARY = DROP_PROCEDURE_IF_EXISTS + INSERT_INTO_DICTIONARY; private static final String DROP_PROCEDURE_INSERT_INTO_DICTIONARY = DROP_PROCEDURE_IF_EXISTS + INSERT_INTO_DICTIONARY;
private static final String DROP_PROCEDURE_INSERT_INTO_TENANT_TS_KV = DROP_PROCEDURE_IF_EXISTS + INSERT_INTO_TS_KV; private static final String DROP_PROCEDURE_INSERT_INTO_TS_KV = DROP_PROCEDURE_IF_EXISTS + INSERT_INTO_TS_KV;
private static final String DROP_PROCEDURE_INSERT_INTO_TS_KV_LATEST = DROP_PROCEDURE_IF_EXISTS + INSERT_INTO_TS_KV_LATEST; private static final String DROP_PROCEDURE_INSERT_INTO_TS_KV_LATEST = DROP_PROCEDURE_IF_EXISTS + INSERT_INTO_TS_KV_LATEST;
@Autowired @Autowired
@ -91,7 +95,49 @@ public class TimescaleTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgr
executeQuery(conn, CALL_CREATE_TS_KV_DICTIONARY_TABLE); executeQuery(conn, CALL_CREATE_TS_KV_DICTIONARY_TABLE);
executeQuery(conn, CALL_INSERT_INTO_DICTIONARY); executeQuery(conn, CALL_INSERT_INTO_DICTIONARY);
executeQuery(conn, CALL_INSERT_INTO_TS_KV);
Path pathToTempTsKvFile;
if (SystemUtils.IS_OS_WINDOWS) {
Path pathToDir;
log.info("Lookup for environment variable: " + THINGSBOARD_WINDOWS_UPGRADE_DIR + " ...");
String thingsboardWindowsUpgradeDir = System.getenv(THINGSBOARD_WINDOWS_UPGRADE_DIR);
if (StringUtils.isNotEmpty(thingsboardWindowsUpgradeDir)) {
log.info("Environment variable: " + THINGSBOARD_WINDOWS_UPGRADE_DIR + " found!");
pathToDir = Paths.get(thingsboardWindowsUpgradeDir);
} else {
log.info("Failed to lookup environment variable: " + THINGSBOARD_WINDOWS_UPGRADE_DIR);
pathToDir = Paths.get(PATH_TO_USERS_PUBLIC_FOLDER);
}
log.info("Directory: " + pathToDir + " will be used for creation temporary upgrade file!");
try {
Path tsKvFile = Files.createTempFile(pathToDir, "ts_kv", ".sql");
pathToTempTsKvFile = tsKvFile.toAbsolutePath();
executeQuery(conn, "call insert_into_ts_kv('" + pathToTempTsKvFile + "')");
pathToTempTsKvFile.toFile().deleteOnExit();
} catch (IOException | SecurityException e) {
throw new RuntimeException("Failed to create time-series upgrade files due to: " + e);
}
} else {
Path tempDirPath = Files.createTempDirectory("ts_kv");
File tempDirAsFile = tempDirPath.toFile();
boolean writable = tempDirAsFile.setWritable(true, false);
boolean readable = tempDirAsFile.setReadable(true, false);
boolean executable = tempDirAsFile.setExecutable(true, false);
if (writable && readable && executable) {
pathToTempTsKvFile = tempDirPath.resolve(TS_KV_SQL).toAbsolutePath();
executeQuery(conn, "call insert_into_ts_kv('" + pathToTempTsKvFile + "')");
} else {
throw new RuntimeException("Failed to grant write permissions for the: " + tempDirPath + "folder!");
}
}
if (pathToTempTsKvFile.toFile().exists()) {
boolean deleteTsKvFile = pathToTempTsKvFile.toFile().delete();
if (deleteTsKvFile) {
log.info("Successfully deleted the temp file for ts_kv table upgrade!");
}
}
executeQuery(conn, CALL_INSERT_INTO_TS_KV_LATEST); executeQuery(conn, CALL_INSERT_INTO_TS_KV_LATEST);
executeQuery(conn, DROP_OLD_TENANT_TS_KV_TABLE); executeQuery(conn, DROP_OLD_TENANT_TS_KV_TABLE);
@ -100,7 +146,7 @@ public class TimescaleTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgr
executeQuery(conn, DROP_PROCEDURE_CREATE_TENANT_TS_KV_TABLE_COPY); executeQuery(conn, DROP_PROCEDURE_CREATE_TENANT_TS_KV_TABLE_COPY);
executeQuery(conn, DROP_PROCEDURE_CREATE_TS_KV_DICTIONARY_TABLE); executeQuery(conn, DROP_PROCEDURE_CREATE_TS_KV_DICTIONARY_TABLE);
executeQuery(conn, DROP_PROCEDURE_INSERT_INTO_DICTIONARY); executeQuery(conn, DROP_PROCEDURE_INSERT_INTO_DICTIONARY);
executeQuery(conn, DROP_PROCEDURE_INSERT_INTO_TENANT_TS_KV); executeQuery(conn, DROP_PROCEDURE_INSERT_INTO_TS_KV);
executeQuery(conn, DROP_PROCEDURE_INSERT_INTO_TS_KV_LATEST); executeQuery(conn, DROP_PROCEDURE_INSERT_INTO_TS_KV_LATEST);
executeQuery(conn, "ALTER TABLE ts_kv ADD COLUMN IF NOT EXISTS json_v json;"); executeQuery(conn, "ALTER TABLE ts_kv ADD COLUMN IF NOT EXISTS json_v json;");