diff --git a/application/src/main/data/upgrade/3.4.1/schema_update.sql b/application/src/main/data/upgrade/3.4.1/schema_update.sql index 1b75a1c1e1..79cde8167f 100644 --- a/application/src/main/data/upgrade/3.4.1/schema_update.sql +++ b/application/src/main/data/upgrade/3.4.1/schema_update.sql @@ -14,7 +14,27 @@ -- limitations under the License. -- -CREATE TABLE IF NOT EXISTS tmp_audit_log ( +DO +$$ + DECLARE table_partition RECORD; + BEGIN + -- in case of running the upgrade script a second time: + IF NOT (SELECT exists(SELECT FROM pg_tables WHERE tablename = 'old_audit_log')) THEN + ALTER TABLE audit_log RENAME TO old_audit_log; + ALTER INDEX IF EXISTS idx_audit_log_tenant_id_and_created_time RENAME TO idx_old_audit_log_tenant_id_and_created_time; + + FOR table_partition IN SELECT tablename AS name, split_part(tablename, '_', 3) AS partition_ts + FROM pg_tables WHERE tablename LIKE 'audit_log_%' + LOOP + EXECUTE format('ALTER TABLE %s RENAME TO old_audit_log_%s', table_partition.name, table_partition.partition_ts); + END LOOP; + ELSE + RAISE NOTICE 'Table old_audit_log already exists, leaving as is'; + END IF; + END; +$$; + +CREATE TABLE IF NOT EXISTS audit_log ( id uuid NOT NULL, created_time bigint NOT NULL, tenant_id uuid, @@ -29,21 +49,7 @@ CREATE TABLE IF NOT EXISTS tmp_audit_log ( action_status varchar(255), action_failure_details varchar(1000000) ) PARTITION BY RANGE (created_time); -CREATE INDEX IF NOT EXISTS idx_tmp_audit_log_tenant_id_and_created_time ON tmp_audit_log(tenant_id, created_time DESC); - -CREATE OR REPLACE PROCEDURE rename_old_audit_logs_partitions() - LANGUAGE plpgsql AS -$$ -DECLARE - table_partition RECORD; -BEGIN - FOR table_partition IN SELECT tablename AS name, split_part(tablename, '_', 3) AS partition_ts - FROM pg_tables WHERE tablename LIKE 'audit_log_%' - LOOP - EXECUTE format('ALTER TABLE %s RENAME TO old_audit_log_%s', table_partition.name, table_partition.partition_ts); - END LOOP; -END; -$$; +CREATE INDEX IF NOT EXISTS idx_audit_log_tenant_id_and_created_time ON audit_log(tenant_id, created_time DESC); CREATE OR REPLACE PROCEDURE migrate_audit_logs(IN start_time_ms BIGINT, IN end_time_ms BIGINT, IN partition_size_ms BIGINT) LANGUAGE plpgsql AS @@ -52,17 +58,17 @@ DECLARE p RECORD; partition_end_ts BIGINT; BEGIN - FOR p IN SELECT DISTINCT (created_time - created_time % partition_size_ms) AS partition_ts FROM audit_log + FOR p IN SELECT DISTINCT (created_time - created_time % partition_size_ms) AS partition_ts FROM old_audit_log WHERE created_time >= start_time_ms AND created_time < end_time_ms LOOP partition_end_ts = p.partition_ts + partition_size_ms; RAISE NOTICE '[audit_log] Partition to create : [%-%]', p.partition_ts, partition_end_ts; - EXECUTE format('CREATE TABLE IF NOT EXISTS audit_log_%s PARTITION OF tmp_audit_log ' || + EXECUTE format('CREATE TABLE IF NOT EXISTS audit_log_%s PARTITION OF audit_log ' || 'FOR VALUES FROM ( %s ) TO ( %s )', p.partition_ts, p.partition_ts, partition_end_ts); END LOOP; - INSERT INTO tmp_audit_log - SELECT * FROM audit_log + INSERT INTO audit_log + SELECT * FROM old_audit_log WHERE created_time >= start_time_ms AND created_time < end_time_ms; END; $$; diff --git a/application/src/main/resources/logback.xml b/application/src/main/resources/logback.xml index 0bf081b35a..941bd7d278 100644 --- a/application/src/main/resources/logback.xml +++ b/application/src/main/resources/logback.xml @@ -28,7 +28,6 @@ - diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/audit/JpaAuditLogDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/audit/JpaAuditLogDao.java index a3ee6ea843..365dd82686 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/audit/JpaAuditLogDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/audit/JpaAuditLogDao.java @@ -154,7 +154,6 @@ public class JpaAuditLogDao extends JpaAbstractDao imp throw new RuntimeException(error); } - jdbcTemplate.execute("CALL rename_old_audit_logs_partitions()"); while (startTime < currentTime) { var endTime = startTime + partitionStepInMs; log.info("Migrating audit logs for time period: {} - {}", startTime, endTime); @@ -163,9 +162,7 @@ public class JpaAuditLogDao extends JpaAbstractDao imp } log.info("Audit logs migration finished"); - jdbcTemplate.execute("DROP TABLE IF EXISTS audit_log"); - jdbcTemplate.execute("ALTER TABLE tmp_audit_log RENAME TO audit_log"); - jdbcTemplate.execute("ALTER INDEX IF EXISTS idx_tmp_audit_log_tenant_id_and_created_time RENAME TO idx_audit_log_tenant_id_and_created_time"); + jdbcTemplate.execute("DROP TABLE IF EXISTS old_audit_log"); } private void callMigrationFunction(long startTime, long endTime, long partitionSizeInMs) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/sql/SqlPartitioningRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/sql/SqlPartitioningRepository.java index 94097c590a..8083f17a5a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/sql/SqlPartitioningRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/sql/SqlPartitioningRepository.java @@ -17,7 +17,6 @@ package org.thingsboard.server.dao.sqlts.insert.sql; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.exception.ExceptionUtils; -import org.hibernate.exception.ConstraintViolationException; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.dao.DataAccessException; import org.springframework.jdbc.core.JdbcTemplate; @@ -68,17 +67,16 @@ public class SqlPartitioningRepository { save(partition); log.trace("Adding partition to map: {}", partition); partitions.put(partition.getStart(), partition); - } catch (Exception ex) { // fixme: check - log.trace("Error occurred during partition save:", ex); - // todo: if partitions накладаються, потестити (ConstraintViolationException) - // todo: SKIP_MIGRATION тільки для того щоб не переносити данні, таблицю треба створити partitioned. - // fixme: update script -// if (ExceptionUtils.indexOfThrowable(ex, ConstraintViolationException.class) >= 0) { -// log.warn("Saving partition [{}] rejected. Data will be saved to the DEFAULT partition.", partition.getPartitionDate()); -// partitions.put(partition.getStart(), partition); -// } else { -// throw ex; -// } + } catch (RuntimeException e) { + log.trace("Error occurred during partition save:", e); + String msg = ExceptionUtils.getRootCauseMessage(e); + if (msg.contains("would overlap partition")) { + log.warn("Couldn't save {} partition for {}, data will be saved to the default partition. SQL error: {}", + partition.getPartitionDate(), table, msg); + partitions.put(partition.getStart(), partition); + } else { + throw e; + } } finally { partitionCreationLock.unlock(); }