dendi-wang/thingsboard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

added ttl for events

Browse Source
This commit is contained in:
Dmytro Shvaika 2020-04-22 13:03:53 +03:00
parent 3a3cee375b
commit dc3d1dd794
6 changed files with 52 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
application/src/main/data/upgrade/2.4.3/schema_update_ttl.sql
View File

@ -123,3 +123,18 @@ BEGIN
END LOOP;
END
$$;
CREATE OR REPLACE PROCEDURE cleanup_events_by_ttl(IN system_ttl bigint, INOUT deleted bigint)
LANGUAGE plpgsql AS
$$
DECLARE
ttl_ts bigint;
BEGIN
IF system_ttl > 0 THEN
ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - system_ttl::bigint * 1000)::bigint;
EXECUTE format(
'WITH deleted AS (DELETE FROM event WHERE ts < %L::bigint RETURNING *) SELECT count(*) FROM deleted', ttl_ts) into deleted;
END IF;
END
$$;

5
application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java
View File

@ -221,6 +221,11 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService
}
}
}
try {
long ts = System.currentTimeMillis();
conn.createStatement().execute("ALTER TABLE event ADD COLUMN ts bigint DEFAULT " + ts + ";"); //tiNOSONAR, ignoring because method used to execute thingsboard database upgrade script
} catch (Exception e) {
}
log.info("Schema updated.");
}
break;

5
application/src/main/java/org/thingsboard/server/service/ttl/events/EventsCleanUpService.java
View File

@ -50,8 +50,7 @@ public class EventsCleanUpService extends AbstractCleanUpService {
@Override
protected void doCleanUp(Connection connection) {
log.info("ttl: [{}]", ttl);
log.info("ttlTaskExecutionEnabled: [{}]", ttlTaskExecutionEnabled);
// TODO: 4/15/20 Do a clean up.
long totalEventsRemoved = executeQuery(connection, "call cleanup_events_by_ttl(" + ttl + ", 0);");
log.info("Total events removed by TTL: [{}]", totalEventsRemoved);
}
}

3
dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java
View File

@ -32,6 +32,9 @@ public class ModelConstants {
public static final String NULL_UUID_STR = UUIDConverter.fromTimeUUID(NULL_UUID);
public static final TenantId SYSTEM_TENANT = new TenantId(ModelConstants.NULL_UUID);
// this is the difference between midnight October 15, 1582 UTC and midnight January 1, 1970 UTC as 100 nanosecond units
public static final long EPOCH_DIFF = 122192928000000000L;
/**
* Generic constants.
*/

12
dao/src/main/java/org/thingsboard/server/dao/model/sql/EventEntity.java
View File

@ -37,6 +37,9 @@ import javax.persistence.EnumType;
import javax.persistence.Enumerated;
import javax.persistence.Table;
import java.util.UUID;
import static org.thingsboard.server.dao.model.ModelConstants.EPOCH_DIFF;
import static org.thingsboard.server.dao.model.ModelConstants.EVENT_BODY_PROPERTY;
import static org.thingsboard.server.dao.model.ModelConstants.EVENT_COLUMN_FAMILY_NAME;
import static org.thingsboard.server.dao.model.ModelConstants.EVENT_ENTITY_ID_PROPERTY;
@ -44,6 +47,7 @@ import static org.thingsboard.server.dao.model.ModelConstants.EVENT_ENTITY_TYPE_
import static org.thingsboard.server.dao.model.ModelConstants.EVENT_TENANT_ID_PROPERTY;
import static org.thingsboard.server.dao.model.ModelConstants.EVENT_TYPE_PROPERTY;
import static org.thingsboard.server.dao.model.ModelConstants.EVENT_UID_PROPERTY;
import static org.thingsboard.server.dao.model.ModelConstants.TS_COLUMN;
@Data
@EqualsAndHashCode(callSuper = true)
@ -73,9 +77,13 @@ public class EventEntity extends BaseSqlEntity<Event> implements BaseEntity<Eve
@Column(name = EVENT_BODY_PROPERTY)
private JsonNode body;
@Column(name = TS_COLUMN)
private long ts;
public EventEntity(Event event) {
if (event.getId() != null) {
this.setId(event.getId().getId());
this.ts = getTs(event.getId().getId());
}
if (event.getTenantId() != null) {
this.tenantId = toString(event.getTenantId().getId());
@ -101,4 +109,8 @@ public class EventEntity extends BaseSqlEntity<Event> implements BaseEntity<Eve
event.setUid(eventUid);
return event;
}
private long getTs(UUID uuid) {
return (uuid.timestamp() - EPOCH_DIFF) / 1000;
}
}

15
dao/src/main/resources/sql/schema-entities.sql
View File

@ -144,6 +144,7 @@ CREATE TABLE IF NOT EXISTS event (
event_type varchar(255),
event_uid varchar(255),
tenant_id varchar(31),
ts bigint,
CONSTRAINT event_unq_key UNIQUE (tenant_id, entity_type, entity_id, event_type, event_uid)
);
@ -249,3 +250,17 @@ CREATE TABLE IF NOT EXISTS entity_view (
search_text varchar(255),
additional_info varchar
);
CREATE OR REPLACE PROCEDURE cleanup_events_by_ttl(IN system_ttl bigint, INOUT deleted bigint)
LANGUAGE plpgsql AS
$$
DECLARE
ttl_ts bigint;
BEGIN
IF system_ttl > 0 THEN
ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - system_ttl::bigint * 1000)::bigint;
EXECUTE format(
'WITH deleted AS (DELETE FROM event WHERE ts < %L::bigint RETURNING *) SELECT count(*) FROM deleted', ttl_ts) into deleted;
END IF;
END
$$;