diff --git a/application/src/main/data/upgrade/2.4.3/schema_update_ttl.sql b/application/src/main/data/upgrade/2.4.3/schema_update_ttl.sql index ff1fb5129b..bd40434ffe 100644 --- a/application/src/main/data/upgrade/2.4.3/schema_update_ttl.sql +++ b/application/src/main/data/upgrade/2.4.3/schema_update_ttl.sql @@ -123,3 +123,18 @@ BEGIN END LOOP; END $$; + +CREATE OR REPLACE PROCEDURE cleanup_events_by_ttl(IN system_ttl bigint, INOUT deleted bigint) + LANGUAGE plpgsql AS +$$ + DECLARE + ttl_ts bigint; +BEGIN + IF system_ttl > 0 THEN + ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - system_ttl::bigint * 1000)::bigint; + EXECUTE format( + 'WITH deleted AS (DELETE FROM event WHERE ts < %L::bigint RETURNING *) SELECT count(*) FROM deleted', ttl_ts) into deleted; + END IF; +END +$$; + diff --git a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java index ef03e3ec43..d7454db0b6 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java @@ -221,6 +221,11 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService } } } + try { + long ts = System.currentTimeMillis(); + conn.createStatement().execute("ALTER TABLE event ADD COLUMN ts bigint DEFAULT " + ts + ";"); //tiNOSONAR, ignoring because method used to execute thingsboard database upgrade script + } catch (Exception e) { + } log.info("Schema updated."); } break; diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/events/EventsCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/events/EventsCleanUpService.java index bb7d112bb1..e10ea30893 100644 --- a/application/src/main/java/org/thingsboard/server/service/ttl/events/EventsCleanUpService.java +++ b/application/src/main/java/org/thingsboard/server/service/ttl/events/EventsCleanUpService.java @@ -50,8 +50,7 @@ public class EventsCleanUpService extends AbstractCleanUpService { @Override protected void doCleanUp(Connection connection) { - log.info("ttl: [{}]", ttl); - log.info("ttlTaskExecutionEnabled: [{}]", ttlTaskExecutionEnabled); - // TODO: 4/15/20 Do a clean up. + long totalEventsRemoved = executeQuery(connection, "call cleanup_events_by_ttl(" + ttl + ", 0);"); + log.info("Total events removed by TTL: [{}]", totalEventsRemoved); } } \ No newline at end of file diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java index 96ce14c459..4a2f8aef23 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java @@ -32,6 +32,9 @@ public class ModelConstants { public static final String NULL_UUID_STR = UUIDConverter.fromTimeUUID(NULL_UUID); public static final TenantId SYSTEM_TENANT = new TenantId(ModelConstants.NULL_UUID); + // this is the difference between midnight October 15, 1582 UTC and midnight January 1, 1970 UTC as 100 nanosecond units + public static final long EPOCH_DIFF = 122192928000000000L; + /** * Generic constants. */ diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/EventEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/EventEntity.java index d775ce771c..c0ad279d5b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/EventEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/EventEntity.java @@ -37,6 +37,9 @@ import javax.persistence.EnumType; import javax.persistence.Enumerated; import javax.persistence.Table; +import java.util.UUID; + +import static org.thingsboard.server.dao.model.ModelConstants.EPOCH_DIFF; import static org.thingsboard.server.dao.model.ModelConstants.EVENT_BODY_PROPERTY; import static org.thingsboard.server.dao.model.ModelConstants.EVENT_COLUMN_FAMILY_NAME; import static org.thingsboard.server.dao.model.ModelConstants.EVENT_ENTITY_ID_PROPERTY; @@ -44,6 +47,7 @@ import static org.thingsboard.server.dao.model.ModelConstants.EVENT_ENTITY_TYPE_ import static org.thingsboard.server.dao.model.ModelConstants.EVENT_TENANT_ID_PROPERTY; import static org.thingsboard.server.dao.model.ModelConstants.EVENT_TYPE_PROPERTY; import static org.thingsboard.server.dao.model.ModelConstants.EVENT_UID_PROPERTY; +import static org.thingsboard.server.dao.model.ModelConstants.TS_COLUMN; @Data @EqualsAndHashCode(callSuper = true) @@ -73,9 +77,13 @@ public class EventEntity extends BaseSqlEntity implements BaseEntity implements BaseEntity 0 THEN + ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - system_ttl::bigint * 1000)::bigint; + EXECUTE format( + 'WITH deleted AS (DELETE FROM event WHERE ts < %L::bigint RETURNING *) SELECT count(*) FROM deleted', ttl_ts) into deleted; + END IF; +END +$$;