Merged with main branch and fixed tests
This commit is contained in:
		
						commit
						02f24ca672
					
				@ -23,6 +23,7 @@ import org.springframework.beans.factory.annotation.Value;
 | 
			
		||||
import org.springframework.stereotype.Service;
 | 
			
		||||
import org.thingsboard.server.common.data.id.TenantId;
 | 
			
		||||
import org.thingsboard.server.common.msg.TbMsg;
 | 
			
		||||
import org.thingsboard.server.common.transport.quota.tenant.TenantQuotaService;
 | 
			
		||||
import org.thingsboard.server.dao.queue.MsgQueue;
 | 
			
		||||
 | 
			
		||||
import javax.annotation.PostConstruct;
 | 
			
		||||
@ -48,6 +49,9 @@ public class DefaultMsgQueueService implements MsgQueueService {
 | 
			
		||||
    @Autowired
 | 
			
		||||
    private MsgQueue msgQueue;
 | 
			
		||||
 | 
			
		||||
    @Autowired
 | 
			
		||||
    private TenantQuotaService quotaService;
 | 
			
		||||
 | 
			
		||||
    private ScheduledExecutorService cleanupExecutor;
 | 
			
		||||
 | 
			
		||||
    private Map<TenantId, AtomicLong> pendingCountPerTenant = new ConcurrentHashMap<>();
 | 
			
		||||
@ -70,6 +74,11 @@ public class DefaultMsgQueueService implements MsgQueueService {
 | 
			
		||||
 | 
			
		||||
    @Override
 | 
			
		||||
    public ListenableFuture<Void> put(TenantId tenantId, TbMsg msg, UUID nodeId, long clusterPartition) {
 | 
			
		||||
        if(quotaService.isQuotaExceeded(tenantId.getId().toString())) {
 | 
			
		||||
            log.warn("Tenant TbMsg Quota exceeded for [{}:{}] . Reject", tenantId.getId());
 | 
			
		||||
            return Futures.immediateFailedFuture(new RuntimeException("Tenant TbMsg Quota exceeded"));
 | 
			
		||||
        }
 | 
			
		||||
 | 
			
		||||
        AtomicLong pendingMsgCount = pendingCountPerTenant.computeIfAbsent(tenantId, key -> new AtomicLong());
 | 
			
		||||
        if (pendingMsgCount.incrementAndGet() < queueMaxSize) {
 | 
			
		||||
            return msgQueue.put(tenantId, msg, nodeId, clusterPartition);
 | 
			
		||||
 | 
			
		||||
@ -131,9 +131,28 @@ quota:
 | 
			
		||||
    whitelist: "${QUOTA_HOST_WHITELIST:localhost,127.0.0.1}"
 | 
			
		||||
    # Array of blacklist hosts
 | 
			
		||||
    blacklist: "${QUOTA_HOST_BLACKLIST:}"
 | 
			
		||||
  log:
 | 
			
		||||
    topSize: 10
 | 
			
		||||
    intervalMin: 2
 | 
			
		||||
    log:
 | 
			
		||||
      topSize: 10
 | 
			
		||||
      intervalMin: 2
 | 
			
		||||
  rule:
 | 
			
		||||
    tenant:
 | 
			
		||||
      # Max allowed number of API requests in interval for single tenant
 | 
			
		||||
      limit: "${QUOTA_TENANT_LIMIT:100000}"
 | 
			
		||||
      # Interval duration
 | 
			
		||||
      intervalMs: "${QUOTA_TENANT_INTERVAL_MS:60000}"
 | 
			
		||||
      # Maximum silence duration for tenant after which Tenant removed from QuotaService. Must be bigger than intervalMs
 | 
			
		||||
      ttlMs: "${QUOTA_TENANT_TTL_MS:60000}"
 | 
			
		||||
      # Interval for scheduled task that cleans expired records. TTL is used for expiring
 | 
			
		||||
      cleanPeriodMs: "${QUOTA_TENANT_CLEAN_PERIOD_MS:300000}"
 | 
			
		||||
      # Enable Host API Limits
 | 
			
		||||
      enabled: "${QUOTA_TENANT_ENABLED:false}"
 | 
			
		||||
      # Array of whitelist tenants
 | 
			
		||||
      whitelist: "${QUOTA_TENANT_WHITELIST:}"
 | 
			
		||||
      # Array of blacklist tenants
 | 
			
		||||
      blacklist: "${QUOTA_HOST_BLACKLIST:}"
 | 
			
		||||
      log:
 | 
			
		||||
        topSize: 10
 | 
			
		||||
        intervalMin: 2
 | 
			
		||||
 | 
			
		||||
database:
 | 
			
		||||
  type: "${DATABASE_TYPE:sql}" # cassandra OR sql
 | 
			
		||||
 | 
			
		||||
@ -15,33 +15,24 @@
 | 
			
		||||
 */
 | 
			
		||||
package org.thingsboard.server.common.transport.quota;
 | 
			
		||||
 | 
			
		||||
import lombok.extern.slf4j.Slf4j;
 | 
			
		||||
import org.springframework.beans.factory.annotation.Value;
 | 
			
		||||
import org.springframework.stereotype.Service;
 | 
			
		||||
import org.thingsboard.server.common.transport.quota.inmemory.HostRequestIntervalRegistry;
 | 
			
		||||
import org.thingsboard.server.common.transport.quota.inmemory.IntervalRegistryCleaner;
 | 
			
		||||
import org.thingsboard.server.common.transport.quota.inmemory.IntervalRegistryLogger;
 | 
			
		||||
import org.thingsboard.server.common.transport.quota.inmemory.KeyBasedIntervalRegistry;
 | 
			
		||||
 | 
			
		||||
import javax.annotation.PostConstruct;
 | 
			
		||||
import javax.annotation.PreDestroy;
 | 
			
		||||
 | 
			
		||||
/**
 | 
			
		||||
 * @author Vitaliy Paromskiy
 | 
			
		||||
 * @version 1.0
 | 
			
		||||
 */
 | 
			
		||||
@Service
 | 
			
		||||
@Slf4j
 | 
			
		||||
public class HostRequestsQuotaService implements QuotaService {
 | 
			
		||||
public class AbstractQuotaService implements QuotaService {
 | 
			
		||||
 | 
			
		||||
    private final HostRequestIntervalRegistry requestRegistry;
 | 
			
		||||
    private final HostRequestLimitPolicy requestsPolicy;
 | 
			
		||||
    private final KeyBasedIntervalRegistry requestRegistry;
 | 
			
		||||
    private final RequestLimitPolicy requestsPolicy;
 | 
			
		||||
    private final IntervalRegistryCleaner registryCleaner;
 | 
			
		||||
    private final IntervalRegistryLogger registryLogger;
 | 
			
		||||
    private final boolean enabled;
 | 
			
		||||
 | 
			
		||||
    public HostRequestsQuotaService(HostRequestIntervalRegistry requestRegistry, HostRequestLimitPolicy requestsPolicy,
 | 
			
		||||
    public AbstractQuotaService(KeyBasedIntervalRegistry requestRegistry, RequestLimitPolicy requestsPolicy,
 | 
			
		||||
                                    IntervalRegistryCleaner registryCleaner, IntervalRegistryLogger registryLogger,
 | 
			
		||||
                                    @Value("${quota.host.enabled}") boolean enabled) {
 | 
			
		||||
                                    boolean enabled) {
 | 
			
		||||
        this.requestRegistry = requestRegistry;
 | 
			
		||||
        this.requestsPolicy = requestsPolicy;
 | 
			
		||||
        this.registryCleaner = registryCleaner;
 | 
			
		||||
@ -0,0 +1,30 @@
 | 
			
		||||
/**
 | 
			
		||||
 * Copyright © 2016-2018 The Thingsboard Authors
 | 
			
		||||
 *
 | 
			
		||||
 * Licensed under the Apache License, Version 2.0 (the "License");
 | 
			
		||||
 * you may not use this file except in compliance with the License.
 | 
			
		||||
 * You may obtain a copy of the License at
 | 
			
		||||
 *
 | 
			
		||||
 *     http://www.apache.org/licenses/LICENSE-2.0
 | 
			
		||||
 *
 | 
			
		||||
 * Unless required by applicable law or agreed to in writing, software
 | 
			
		||||
 * distributed under the License is distributed on an "AS IS" BASIS,
 | 
			
		||||
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 | 
			
		||||
 * See the License for the specific language governing permissions and
 | 
			
		||||
 * limitations under the License.
 | 
			
		||||
 */
 | 
			
		||||
package org.thingsboard.server.common.transport.quota;
 | 
			
		||||
 | 
			
		||||
 | 
			
		||||
public abstract class RequestLimitPolicy {
 | 
			
		||||
 | 
			
		||||
    private final long limit;
 | 
			
		||||
 | 
			
		||||
    public RequestLimitPolicy(long limit) {
 | 
			
		||||
        this.limit = limit;
 | 
			
		||||
    }
 | 
			
		||||
 | 
			
		||||
    public boolean isValid(long currentValue) {
 | 
			
		||||
        return currentValue <= limit;
 | 
			
		||||
    }
 | 
			
		||||
}
 | 
			
		||||
@ -0,0 +1,29 @@
 | 
			
		||||
/**
 | 
			
		||||
 * Copyright © 2016-2018 The Thingsboard Authors
 | 
			
		||||
 *
 | 
			
		||||
 * Licensed under the Apache License, Version 2.0 (the "License");
 | 
			
		||||
 * you may not use this file except in compliance with the License.
 | 
			
		||||
 * You may obtain a copy of the License at
 | 
			
		||||
 *
 | 
			
		||||
 *     http://www.apache.org/licenses/LICENSE-2.0
 | 
			
		||||
 *
 | 
			
		||||
 * Unless required by applicable law or agreed to in writing, software
 | 
			
		||||
 * distributed under the License is distributed on an "AS IS" BASIS,
 | 
			
		||||
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 | 
			
		||||
 * See the License for the specific language governing permissions and
 | 
			
		||||
 * limitations under the License.
 | 
			
		||||
 */
 | 
			
		||||
package org.thingsboard.server.common.transport.quota.host;
 | 
			
		||||
 | 
			
		||||
import org.springframework.beans.factory.annotation.Value;
 | 
			
		||||
import org.springframework.stereotype.Component;
 | 
			
		||||
import org.thingsboard.server.common.transport.quota.inmemory.IntervalRegistryCleaner;
 | 
			
		||||
 | 
			
		||||
@Component
 | 
			
		||||
public class HostIntervalRegistryCleaner extends IntervalRegistryCleaner {
 | 
			
		||||
 | 
			
		||||
    public HostIntervalRegistryCleaner(HostRequestIntervalRegistry intervalRegistry,
 | 
			
		||||
                                       @Value("${quota.host.cleanPeriodMs}") long cleanPeriodMs) {
 | 
			
		||||
        super(intervalRegistry, cleanPeriodMs);
 | 
			
		||||
    }
 | 
			
		||||
}
 | 
			
		||||
@ -0,0 +1,52 @@
 | 
			
		||||
/**
 | 
			
		||||
 * Copyright © 2016-2018 The Thingsboard Authors
 | 
			
		||||
 *
 | 
			
		||||
 * Licensed under the Apache License, Version 2.0 (the "License");
 | 
			
		||||
 * you may not use this file except in compliance with the License.
 | 
			
		||||
 * You may obtain a copy of the License at
 | 
			
		||||
 *
 | 
			
		||||
 *     http://www.apache.org/licenses/LICENSE-2.0
 | 
			
		||||
 *
 | 
			
		||||
 * Unless required by applicable law or agreed to in writing, software
 | 
			
		||||
 * distributed under the License is distributed on an "AS IS" BASIS,
 | 
			
		||||
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 | 
			
		||||
 * See the License for the specific language governing permissions and
 | 
			
		||||
 * limitations under the License.
 | 
			
		||||
 */
 | 
			
		||||
package org.thingsboard.server.common.transport.quota.host;
 | 
			
		||||
 | 
			
		||||
import lombok.extern.slf4j.Slf4j;
 | 
			
		||||
import org.springframework.beans.factory.annotation.Value;
 | 
			
		||||
import org.springframework.stereotype.Component;
 | 
			
		||||
import org.thingsboard.server.common.transport.quota.inmemory.IntervalRegistryLogger;
 | 
			
		||||
 | 
			
		||||
import java.util.Map;
 | 
			
		||||
import java.util.concurrent.TimeUnit;
 | 
			
		||||
 | 
			
		||||
@Component
 | 
			
		||||
@Slf4j
 | 
			
		||||
public class HostIntervalRegistryLogger extends IntervalRegistryLogger {
 | 
			
		||||
 | 
			
		||||
    private final long logIntervalMin;
 | 
			
		||||
 | 
			
		||||
    public HostIntervalRegistryLogger(@Value("${quota.host.log.topSize}") int topSize,
 | 
			
		||||
                                      @Value("${quota.host.log.intervalMin}") long logIntervalMin,
 | 
			
		||||
                                      HostRequestIntervalRegistry intervalRegistry) {
 | 
			
		||||
        super(topSize, logIntervalMin, intervalRegistry);
 | 
			
		||||
        this.logIntervalMin = logIntervalMin;
 | 
			
		||||
    }
 | 
			
		||||
 | 
			
		||||
    protected void log(Map<String, Long> top, int uniqHosts, long requestsCount) {
 | 
			
		||||
        long rps = requestsCount / TimeUnit.MINUTES.toSeconds(logIntervalMin);
 | 
			
		||||
        StringBuilder builder = new StringBuilder("Quota Statistic : ");
 | 
			
		||||
        builder.append("uniqHosts : ").append(uniqHosts).append("; ");
 | 
			
		||||
        builder.append("requestsCount : ").append(requestsCount).append("; ");
 | 
			
		||||
        builder.append("RPS : ").append(rps).append(" ");
 | 
			
		||||
        builder.append("top -> ");
 | 
			
		||||
        for (Map.Entry<String, Long> host : top.entrySet()) {
 | 
			
		||||
            builder.append(host.getKey()).append(" : ").append(host.getValue()).append("; ");
 | 
			
		||||
        }
 | 
			
		||||
 | 
			
		||||
        log.info(builder.toString());
 | 
			
		||||
    }
 | 
			
		||||
}
 | 
			
		||||
@ -0,0 +1,37 @@
 | 
			
		||||
/**
 | 
			
		||||
 * Copyright © 2016-2018 The Thingsboard Authors
 | 
			
		||||
 *
 | 
			
		||||
 * Licensed under the Apache License, Version 2.0 (the "License");
 | 
			
		||||
 * you may not use this file except in compliance with the License.
 | 
			
		||||
 * You may obtain a copy of the License at
 | 
			
		||||
 *
 | 
			
		||||
 *     http://www.apache.org/licenses/LICENSE-2.0
 | 
			
		||||
 *
 | 
			
		||||
 * Unless required by applicable law or agreed to in writing, software
 | 
			
		||||
 * distributed under the License is distributed on an "AS IS" BASIS,
 | 
			
		||||
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 | 
			
		||||
 * See the License for the specific language governing permissions and
 | 
			
		||||
 * limitations under the License.
 | 
			
		||||
 */
 | 
			
		||||
package org.thingsboard.server.common.transport.quota.host;
 | 
			
		||||
 | 
			
		||||
import lombok.extern.slf4j.Slf4j;
 | 
			
		||||
import org.springframework.beans.factory.annotation.Value;
 | 
			
		||||
import org.springframework.stereotype.Component;
 | 
			
		||||
import org.thingsboard.server.common.transport.quota.inmemory.KeyBasedIntervalRegistry;
 | 
			
		||||
 | 
			
		||||
/**
 | 
			
		||||
 * @author Vitaliy Paromskiy
 | 
			
		||||
 * @version 1.0
 | 
			
		||||
 */
 | 
			
		||||
@Component
 | 
			
		||||
@Slf4j
 | 
			
		||||
public class HostRequestIntervalRegistry extends KeyBasedIntervalRegistry {
 | 
			
		||||
 | 
			
		||||
    public HostRequestIntervalRegistry(@Value("${quota.host.intervalMs}") long intervalDurationMs,
 | 
			
		||||
                                       @Value("${quota.host.ttlMs}") long ttlMs,
 | 
			
		||||
                                       @Value("${quota.host.whitelist}") String whiteList,
 | 
			
		||||
                                       @Value("${quota.host.blacklist}") String blackList) {
 | 
			
		||||
        super(intervalDurationMs, ttlMs, whiteList, blackList, "host");
 | 
			
		||||
    }
 | 
			
		||||
}
 | 
			
		||||
@ -13,26 +13,21 @@
 | 
			
		||||
 * See the License for the specific language governing permissions and
 | 
			
		||||
 * limitations under the License.
 | 
			
		||||
 */
 | 
			
		||||
package org.thingsboard.server.common.transport.quota;
 | 
			
		||||
package org.thingsboard.server.common.transport.quota.host;
 | 
			
		||||
 | 
			
		||||
import org.springframework.beans.factory.annotation.Value;
 | 
			
		||||
import org.springframework.stereotype.Component;
 | 
			
		||||
import org.thingsboard.server.common.transport.quota.RequestLimitPolicy;
 | 
			
		||||
 | 
			
		||||
/**
 | 
			
		||||
 * @author Vitaliy Paromskiy
 | 
			
		||||
 * @version 1.0
 | 
			
		||||
 */
 | 
			
		||||
@Component
 | 
			
		||||
public class HostRequestLimitPolicy {
 | 
			
		||||
 | 
			
		||||
    private final long limit;
 | 
			
		||||
public class HostRequestLimitPolicy extends RequestLimitPolicy {
 | 
			
		||||
 | 
			
		||||
    public HostRequestLimitPolicy(@Value("${quota.host.limit}") long limit) {
 | 
			
		||||
        this.limit = limit;
 | 
			
		||||
    }
 | 
			
		||||
 | 
			
		||||
    public boolean isValid(long currentValue) {
 | 
			
		||||
        return currentValue <= limit;
 | 
			
		||||
        super(limit);
 | 
			
		||||
    }
 | 
			
		||||
 | 
			
		||||
}
 | 
			
		||||
@ -0,0 +1,37 @@
 | 
			
		||||
/**
 | 
			
		||||
 * Copyright © 2016-2018 The Thingsboard Authors
 | 
			
		||||
 *
 | 
			
		||||
 * Licensed under the Apache License, Version 2.0 (the "License");
 | 
			
		||||
 * you may not use this file except in compliance with the License.
 | 
			
		||||
 * You may obtain a copy of the License at
 | 
			
		||||
 *
 | 
			
		||||
 *     http://www.apache.org/licenses/LICENSE-2.0
 | 
			
		||||
 *
 | 
			
		||||
 * Unless required by applicable law or agreed to in writing, software
 | 
			
		||||
 * distributed under the License is distributed on an "AS IS" BASIS,
 | 
			
		||||
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 | 
			
		||||
 * See the License for the specific language governing permissions and
 | 
			
		||||
 * limitations under the License.
 | 
			
		||||
 */
 | 
			
		||||
package org.thingsboard.server.common.transport.quota.host;
 | 
			
		||||
 | 
			
		||||
import lombok.extern.slf4j.Slf4j;
 | 
			
		||||
import org.springframework.beans.factory.annotation.Value;
 | 
			
		||||
import org.springframework.stereotype.Service;
 | 
			
		||||
import org.thingsboard.server.common.transport.quota.AbstractQuotaService;
 | 
			
		||||
 | 
			
		||||
/**
 | 
			
		||||
 * @author Vitaliy Paromskiy
 | 
			
		||||
 * @version 1.0
 | 
			
		||||
 */
 | 
			
		||||
@Service
 | 
			
		||||
@Slf4j
 | 
			
		||||
public class HostRequestsQuotaService extends AbstractQuotaService {
 | 
			
		||||
 | 
			
		||||
    public HostRequestsQuotaService(HostRequestIntervalRegistry requestRegistry, HostRequestLimitPolicy requestsPolicy,
 | 
			
		||||
                                    HostIntervalRegistryCleaner registryCleaner, HostIntervalRegistryLogger registryLogger,
 | 
			
		||||
                                    @Value("${quota.host.enabled}") boolean enabled) {
 | 
			
		||||
        super(requestRegistry, requestsPolicy, registryCleaner, registryLogger, enabled);
 | 
			
		||||
    }
 | 
			
		||||
 | 
			
		||||
}
 | 
			
		||||
@ -16,10 +16,7 @@
 | 
			
		||||
package org.thingsboard.server.common.transport.quota.inmemory;
 | 
			
		||||
 | 
			
		||||
import lombok.extern.slf4j.Slf4j;
 | 
			
		||||
import org.springframework.beans.factory.annotation.Value;
 | 
			
		||||
import org.springframework.stereotype.Component;
 | 
			
		||||
 | 
			
		||||
import javax.annotation.PreDestroy;
 | 
			
		||||
import java.util.concurrent.Executors;
 | 
			
		||||
import java.util.concurrent.ScheduledExecutorService;
 | 
			
		||||
import java.util.concurrent.TimeUnit;
 | 
			
		||||
@ -28,15 +25,14 @@ import java.util.concurrent.TimeUnit;
 | 
			
		||||
 * @author Vitaliy Paromskiy
 | 
			
		||||
 * @version 1.0
 | 
			
		||||
 */
 | 
			
		||||
@Component
 | 
			
		||||
@Slf4j
 | 
			
		||||
public class IntervalRegistryCleaner {
 | 
			
		||||
public abstract class IntervalRegistryCleaner {
 | 
			
		||||
 | 
			
		||||
    private final HostRequestIntervalRegistry intervalRegistry;
 | 
			
		||||
    private final KeyBasedIntervalRegistry intervalRegistry;
 | 
			
		||||
    private final long cleanPeriodMs;
 | 
			
		||||
    private ScheduledExecutorService executor;
 | 
			
		||||
 | 
			
		||||
    public IntervalRegistryCleaner(HostRequestIntervalRegistry intervalRegistry, @Value("${quota.host.cleanPeriodMs}") long cleanPeriodMs) {
 | 
			
		||||
    public IntervalRegistryCleaner(KeyBasedIntervalRegistry intervalRegistry, long cleanPeriodMs) {
 | 
			
		||||
        this.intervalRegistry = intervalRegistry;
 | 
			
		||||
        this.cleanPeriodMs = cleanPeriodMs;
 | 
			
		||||
    }
 | 
			
		||||
 | 
			
		||||
@ -17,8 +17,6 @@ package org.thingsboard.server.common.transport.quota.inmemory;
 | 
			
		||||
 | 
			
		||||
import com.google.common.collect.MinMaxPriorityQueue;
 | 
			
		||||
import lombok.extern.slf4j.Slf4j;
 | 
			
		||||
import org.springframework.beans.factory.annotation.Value;
 | 
			
		||||
import org.springframework.stereotype.Component;
 | 
			
		||||
 | 
			
		||||
import java.util.Comparator;
 | 
			
		||||
import java.util.Map;
 | 
			
		||||
@ -32,17 +30,15 @@ import java.util.stream.Collectors;
 | 
			
		||||
 * @author Vitaliy Paromskiy
 | 
			
		||||
 * @version 1.0
 | 
			
		||||
 */
 | 
			
		||||
@Component
 | 
			
		||||
@Slf4j
 | 
			
		||||
public class IntervalRegistryLogger {
 | 
			
		||||
public abstract class IntervalRegistryLogger {
 | 
			
		||||
 | 
			
		||||
    private final int topSize;
 | 
			
		||||
    private final HostRequestIntervalRegistry intervalRegistry;
 | 
			
		||||
    private final KeyBasedIntervalRegistry intervalRegistry;
 | 
			
		||||
    private final long logIntervalMin;
 | 
			
		||||
    private ScheduledExecutorService executor;
 | 
			
		||||
 | 
			
		||||
    public IntervalRegistryLogger(@Value("${quota.log.topSize}") int topSize, @Value("${quota.log.intervalMin}") long logIntervalMin,
 | 
			
		||||
                                  HostRequestIntervalRegistry intervalRegistry) {
 | 
			
		||||
    public IntervalRegistryLogger(int topSize, long logIntervalMin, KeyBasedIntervalRegistry intervalRegistry) {
 | 
			
		||||
        this.topSize = topSize;
 | 
			
		||||
        this.logIntervalMin = logIntervalMin;
 | 
			
		||||
        this.intervalRegistry = intervalRegistry;
 | 
			
		||||
@ -79,17 +75,5 @@ public class IntervalRegistryLogger {
 | 
			
		||||
        return topQueue.stream().collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
 | 
			
		||||
    }
 | 
			
		||||
 | 
			
		||||
    private void log(Map<String, Long> top, int uniqHosts, long requestsCount) {
 | 
			
		||||
        long rps = requestsCount / TimeUnit.MINUTES.toSeconds(logIntervalMin);
 | 
			
		||||
        StringBuilder builder = new StringBuilder("Quota Statistic : ");
 | 
			
		||||
        builder.append("uniqHosts : ").append(uniqHosts).append("; ");
 | 
			
		||||
        builder.append("requestsCount : ").append(requestsCount).append("; ");
 | 
			
		||||
        builder.append("RPS : ").append(rps).append(" ");
 | 
			
		||||
        builder.append("top -> ");
 | 
			
		||||
        for (Map.Entry<String, Long> host : top.entrySet()) {
 | 
			
		||||
            builder.append(host.getKey()).append(" : ").append(host.getValue()).append("; ");
 | 
			
		||||
        }
 | 
			
		||||
 | 
			
		||||
        log.info(builder.toString());
 | 
			
		||||
    }
 | 
			
		||||
    protected abstract void log(Map<String, Long> top, int uniqHosts, long requestsCount);
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
@ -18,22 +18,14 @@ package org.thingsboard.server.common.transport.quota.inmemory;
 | 
			
		||||
import com.google.common.collect.Sets;
 | 
			
		||||
import lombok.extern.slf4j.Slf4j;
 | 
			
		||||
import org.apache.commons.lang3.StringUtils;
 | 
			
		||||
import org.springframework.beans.factory.annotation.Value;
 | 
			
		||||
import org.springframework.stereotype.Component;
 | 
			
		||||
 | 
			
		||||
import javax.annotation.PostConstruct;
 | 
			
		||||
import java.util.Map;
 | 
			
		||||
import java.util.Set;
 | 
			
		||||
import java.util.concurrent.ConcurrentHashMap;
 | 
			
		||||
import java.util.stream.Collectors;
 | 
			
		||||
 | 
			
		||||
/**
 | 
			
		||||
 * @author Vitaliy Paromskiy
 | 
			
		||||
 * @version 1.0
 | 
			
		||||
 */
 | 
			
		||||
@Component
 | 
			
		||||
@Slf4j
 | 
			
		||||
public class HostRequestIntervalRegistry {
 | 
			
		||||
public abstract class KeyBasedIntervalRegistry {
 | 
			
		||||
 | 
			
		||||
    private final Map<String, IntervalCount> hostCounts = new ConcurrentHashMap<>();
 | 
			
		||||
    private final long intervalDurationMs;
 | 
			
		||||
@ -41,23 +33,20 @@ public class HostRequestIntervalRegistry {
 | 
			
		||||
    private final Set<String> whiteList;
 | 
			
		||||
    private final Set<String> blackList;
 | 
			
		||||
 | 
			
		||||
    public HostRequestIntervalRegistry(@Value("${quota.host.intervalMs}") long intervalDurationMs,
 | 
			
		||||
                                       @Value("${quota.host.ttlMs}") long ttlMs,
 | 
			
		||||
                                       @Value("${quota.host.whitelist}") String whiteList,
 | 
			
		||||
                                       @Value("${quota.host.blacklist}") String blackList) {
 | 
			
		||||
    public KeyBasedIntervalRegistry(long intervalDurationMs, long ttlMs, String whiteList, String blackList, String name) {
 | 
			
		||||
        this.intervalDurationMs = intervalDurationMs;
 | 
			
		||||
        this.ttlMs = ttlMs;
 | 
			
		||||
        this.whiteList = Sets.newHashSet(StringUtils.split(whiteList, ','));
 | 
			
		||||
        this.blackList = Sets.newHashSet(StringUtils.split(blackList, ','));
 | 
			
		||||
        validate(name);
 | 
			
		||||
    }
 | 
			
		||||
 | 
			
		||||
    @PostConstruct
 | 
			
		||||
    public void init() {
 | 
			
		||||
    private void validate(String name) {
 | 
			
		||||
        if (ttlMs < intervalDurationMs) {
 | 
			
		||||
            log.warn("TTL for IntervalRegistry [{}] smaller than interval duration [{}]", ttlMs, intervalDurationMs);
 | 
			
		||||
            log.warn("TTL for {} IntervalRegistry [{}] smaller than interval duration [{}]", name, ttlMs, intervalDurationMs);
 | 
			
		||||
        }
 | 
			
		||||
        log.info("Start Host Quota Service with whitelist {}", whiteList);
 | 
			
		||||
        log.info("Start Host Quota Service with blacklist {}", blackList);
 | 
			
		||||
        log.info("Start {} KeyBasedIntervalRegistry with whitelist {}", name, whiteList);
 | 
			
		||||
        log.info("Start {} KeyBasedIntervalRegistry with blacklist {}", name, blackList);
 | 
			
		||||
    }
 | 
			
		||||
 | 
			
		||||
    public long tick(String clientHostId) {
 | 
			
		||||
@ -0,0 +1,29 @@
 | 
			
		||||
/**
 | 
			
		||||
 * Copyright © 2016-2018 The Thingsboard Authors
 | 
			
		||||
 *
 | 
			
		||||
 * Licensed under the Apache License, Version 2.0 (the "License");
 | 
			
		||||
 * you may not use this file except in compliance with the License.
 | 
			
		||||
 * You may obtain a copy of the License at
 | 
			
		||||
 *
 | 
			
		||||
 *     http://www.apache.org/licenses/LICENSE-2.0
 | 
			
		||||
 *
 | 
			
		||||
 * Unless required by applicable law or agreed to in writing, software
 | 
			
		||||
 * distributed under the License is distributed on an "AS IS" BASIS,
 | 
			
		||||
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 | 
			
		||||
 * See the License for the specific language governing permissions and
 | 
			
		||||
 * limitations under the License.
 | 
			
		||||
 */
 | 
			
		||||
package org.thingsboard.server.common.transport.quota.tenant;
 | 
			
		||||
 | 
			
		||||
import org.springframework.beans.factory.annotation.Value;
 | 
			
		||||
import org.springframework.stereotype.Component;
 | 
			
		||||
import org.thingsboard.server.common.transport.quota.inmemory.IntervalRegistryCleaner;
 | 
			
		||||
 | 
			
		||||
@Component
 | 
			
		||||
public class TenantIntervalRegistryCleaner extends IntervalRegistryCleaner {
 | 
			
		||||
 | 
			
		||||
    public TenantIntervalRegistryCleaner(TenantMsgsIntervalRegistry intervalRegistry,
 | 
			
		||||
                                         @Value("${quota.rule.tenant.cleanPeriodMs}") long cleanPeriodMs) {
 | 
			
		||||
        super(intervalRegistry, cleanPeriodMs);
 | 
			
		||||
    }
 | 
			
		||||
}
 | 
			
		||||
@ -0,0 +1,52 @@
 | 
			
		||||
/**
 | 
			
		||||
 * Copyright © 2016-2018 The Thingsboard Authors
 | 
			
		||||
 *
 | 
			
		||||
 * Licensed under the Apache License, Version 2.0 (the "License");
 | 
			
		||||
 * you may not use this file except in compliance with the License.
 | 
			
		||||
 * You may obtain a copy of the License at
 | 
			
		||||
 *
 | 
			
		||||
 *     http://www.apache.org/licenses/LICENSE-2.0
 | 
			
		||||
 *
 | 
			
		||||
 * Unless required by applicable law or agreed to in writing, software
 | 
			
		||||
 * distributed under the License is distributed on an "AS IS" BASIS,
 | 
			
		||||
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 | 
			
		||||
 * See the License for the specific language governing permissions and
 | 
			
		||||
 * limitations under the License.
 | 
			
		||||
 */
 | 
			
		||||
package org.thingsboard.server.common.transport.quota.tenant;
 | 
			
		||||
 | 
			
		||||
import lombok.extern.slf4j.Slf4j;
 | 
			
		||||
import org.springframework.beans.factory.annotation.Value;
 | 
			
		||||
import org.springframework.stereotype.Component;
 | 
			
		||||
import org.thingsboard.server.common.transport.quota.inmemory.IntervalRegistryLogger;
 | 
			
		||||
 | 
			
		||||
import java.util.Map;
 | 
			
		||||
import java.util.concurrent.TimeUnit;
 | 
			
		||||
 | 
			
		||||
@Slf4j
 | 
			
		||||
@Component
 | 
			
		||||
public class TenantIntervalRegistryLogger extends IntervalRegistryLogger {
 | 
			
		||||
 | 
			
		||||
    private final long logIntervalMin;
 | 
			
		||||
 | 
			
		||||
    public TenantIntervalRegistryLogger(@Value("${quota.rule.tenant.log.topSize}") int topSize,
 | 
			
		||||
                                        @Value("${quota.rule.tenant.log.intervalMin}") long logIntervalMin,
 | 
			
		||||
                                        TenantMsgsIntervalRegistry intervalRegistry) {
 | 
			
		||||
        super(topSize, logIntervalMin, intervalRegistry);
 | 
			
		||||
        this.logIntervalMin = logIntervalMin;
 | 
			
		||||
    }
 | 
			
		||||
 | 
			
		||||
    protected void log(Map<String, Long> top, int uniqHosts, long requestsCount) {
 | 
			
		||||
        long rps = requestsCount / TimeUnit.MINUTES.toSeconds(logIntervalMin);
 | 
			
		||||
        StringBuilder builder = new StringBuilder("Tenant Quota Statistic : ");
 | 
			
		||||
        builder.append("uniqTenants : ").append(uniqHosts).append("; ");
 | 
			
		||||
        builder.append("requestsCount : ").append(requestsCount).append("; ");
 | 
			
		||||
        builder.append("RPS : ").append(rps).append(" ");
 | 
			
		||||
        builder.append("top -> ");
 | 
			
		||||
        for (Map.Entry<String, Long> host : top.entrySet()) {
 | 
			
		||||
            builder.append(host.getKey()).append(" : ").append(host.getValue()).append("; ");
 | 
			
		||||
        }
 | 
			
		||||
 | 
			
		||||
        log.info(builder.toString());
 | 
			
		||||
    }
 | 
			
		||||
}
 | 
			
		||||
@ -0,0 +1,31 @@
 | 
			
		||||
/**
 | 
			
		||||
 * Copyright © 2016-2018 The Thingsboard Authors
 | 
			
		||||
 *
 | 
			
		||||
 * Licensed under the Apache License, Version 2.0 (the "License");
 | 
			
		||||
 * you may not use this file except in compliance with the License.
 | 
			
		||||
 * You may obtain a copy of the License at
 | 
			
		||||
 *
 | 
			
		||||
 *     http://www.apache.org/licenses/LICENSE-2.0
 | 
			
		||||
 *
 | 
			
		||||
 * Unless required by applicable law or agreed to in writing, software
 | 
			
		||||
 * distributed under the License is distributed on an "AS IS" BASIS,
 | 
			
		||||
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 | 
			
		||||
 * See the License for the specific language governing permissions and
 | 
			
		||||
 * limitations under the License.
 | 
			
		||||
 */
 | 
			
		||||
package org.thingsboard.server.common.transport.quota.tenant;
 | 
			
		||||
 | 
			
		||||
import org.springframework.beans.factory.annotation.Value;
 | 
			
		||||
import org.springframework.stereotype.Component;
 | 
			
		||||
import org.thingsboard.server.common.transport.quota.inmemory.KeyBasedIntervalRegistry;
 | 
			
		||||
 | 
			
		||||
@Component
 | 
			
		||||
public class TenantMsgsIntervalRegistry extends KeyBasedIntervalRegistry {
 | 
			
		||||
 | 
			
		||||
    public TenantMsgsIntervalRegistry(@Value("${quota.rule.tenant.intervalMs}") long intervalDurationMs,
 | 
			
		||||
                                      @Value("${quota.rule.tenant.ttlMs}") long ttlMs,
 | 
			
		||||
                                      @Value("${quota.rule.tenant.whitelist}") String whiteList,
 | 
			
		||||
                                      @Value("${quota.rule.tenant.blacklist}") String blackList) {
 | 
			
		||||
        super(intervalDurationMs, ttlMs, whiteList, blackList, "Rule Tenant");
 | 
			
		||||
    }
 | 
			
		||||
}
 | 
			
		||||
@ -0,0 +1,30 @@
 | 
			
		||||
/**
 | 
			
		||||
 * Copyright © 2016-2018 The Thingsboard Authors
 | 
			
		||||
 *
 | 
			
		||||
 * Licensed under the Apache License, Version 2.0 (the "License");
 | 
			
		||||
 * you may not use this file except in compliance with the License.
 | 
			
		||||
 * You may obtain a copy of the License at
 | 
			
		||||
 *
 | 
			
		||||
 *     http://www.apache.org/licenses/LICENSE-2.0
 | 
			
		||||
 *
 | 
			
		||||
 * Unless required by applicable law or agreed to in writing, software
 | 
			
		||||
 * distributed under the License is distributed on an "AS IS" BASIS,
 | 
			
		||||
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 | 
			
		||||
 * See the License for the specific language governing permissions and
 | 
			
		||||
 * limitations under the License.
 | 
			
		||||
 */
 | 
			
		||||
package org.thingsboard.server.common.transport.quota.tenant;
 | 
			
		||||
 | 
			
		||||
import org.springframework.beans.factory.annotation.Value;
 | 
			
		||||
import org.springframework.stereotype.Component;
 | 
			
		||||
import org.thingsboard.server.common.transport.quota.AbstractQuotaService;
 | 
			
		||||
 | 
			
		||||
@Component
 | 
			
		||||
public class TenantQuotaService extends AbstractQuotaService {
 | 
			
		||||
 | 
			
		||||
    public TenantQuotaService(TenantMsgsIntervalRegistry requestRegistry, TenantRequestLimitPolicy requestsPolicy,
 | 
			
		||||
                              TenantIntervalRegistryCleaner registryCleaner, TenantIntervalRegistryLogger registryLogger,
 | 
			
		||||
                              @Value("${quota.rule.tenant.enabled}") boolean enabled) {
 | 
			
		||||
        super(requestRegistry, requestsPolicy, registryCleaner, registryLogger, enabled);
 | 
			
		||||
    }
 | 
			
		||||
}
 | 
			
		||||
@ -0,0 +1,28 @@
 | 
			
		||||
/**
 | 
			
		||||
 * Copyright © 2016-2018 The Thingsboard Authors
 | 
			
		||||
 *
 | 
			
		||||
 * Licensed under the Apache License, Version 2.0 (the "License");
 | 
			
		||||
 * you may not use this file except in compliance with the License.
 | 
			
		||||
 * You may obtain a copy of the License at
 | 
			
		||||
 *
 | 
			
		||||
 *     http://www.apache.org/licenses/LICENSE-2.0
 | 
			
		||||
 *
 | 
			
		||||
 * Unless required by applicable law or agreed to in writing, software
 | 
			
		||||
 * distributed under the License is distributed on an "AS IS" BASIS,
 | 
			
		||||
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 | 
			
		||||
 * See the License for the specific language governing permissions and
 | 
			
		||||
 * limitations under the License.
 | 
			
		||||
 */
 | 
			
		||||
package org.thingsboard.server.common.transport.quota.tenant;
 | 
			
		||||
 | 
			
		||||
import org.springframework.beans.factory.annotation.Value;
 | 
			
		||||
import org.springframework.stereotype.Component;
 | 
			
		||||
import org.thingsboard.server.common.transport.quota.RequestLimitPolicy;
 | 
			
		||||
 | 
			
		||||
@Component
 | 
			
		||||
public class TenantRequestLimitPolicy extends RequestLimitPolicy {
 | 
			
		||||
 | 
			
		||||
    public TenantRequestLimitPolicy(@Value("${quota.rule.tenant.limit}") long limit) {
 | 
			
		||||
        super(limit);
 | 
			
		||||
    }
 | 
			
		||||
}
 | 
			
		||||
@ -16,6 +16,7 @@
 | 
			
		||||
package org.thingsboard.server.common.transport.quota;
 | 
			
		||||
 | 
			
		||||
import org.junit.Test;
 | 
			
		||||
import org.thingsboard.server.common.transport.quota.host.HostRequestLimitPolicy;
 | 
			
		||||
 | 
			
		||||
import static org.junit.Assert.assertFalse;
 | 
			
		||||
import static org.junit.Assert.assertTrue;
 | 
			
		||||
 | 
			
		||||
@ -17,9 +17,7 @@ package org.thingsboard.server.common.transport.quota;
 | 
			
		||||
 | 
			
		||||
import org.junit.Before;
 | 
			
		||||
import org.junit.Test;
 | 
			
		||||
import org.thingsboard.server.common.transport.quota.inmemory.HostRequestIntervalRegistry;
 | 
			
		||||
import org.thingsboard.server.common.transport.quota.inmemory.IntervalRegistryCleaner;
 | 
			
		||||
import org.thingsboard.server.common.transport.quota.inmemory.IntervalRegistryLogger;
 | 
			
		||||
import org.thingsboard.server.common.transport.quota.host.*;
 | 
			
		||||
 | 
			
		||||
import static org.junit.Assert.assertFalse;
 | 
			
		||||
import static org.junit.Assert.assertTrue;
 | 
			
		||||
@ -35,8 +33,8 @@ public class HostRequestsQuotaServiceTest {
 | 
			
		||||
 | 
			
		||||
    private HostRequestIntervalRegistry requestRegistry = mock(HostRequestIntervalRegistry.class);
 | 
			
		||||
    private HostRequestLimitPolicy requestsPolicy = mock(HostRequestLimitPolicy.class);
 | 
			
		||||
    private IntervalRegistryCleaner registryCleaner = mock(IntervalRegistryCleaner.class);
 | 
			
		||||
    private IntervalRegistryLogger registryLogger = mock(IntervalRegistryLogger.class);
 | 
			
		||||
    private HostIntervalRegistryCleaner registryCleaner = mock(HostIntervalRegistryCleaner.class);
 | 
			
		||||
    private HostIntervalRegistryLogger registryLogger = mock(HostIntervalRegistryLogger.class);
 | 
			
		||||
 | 
			
		||||
    @Before
 | 
			
		||||
    public void init() {
 | 
			
		||||
 | 
			
		||||
@ -15,11 +15,9 @@
 | 
			
		||||
 */
 | 
			
		||||
package org.thingsboard.server.common.transport.quota.inmemory;
 | 
			
		||||
 | 
			
		||||
import com.google.common.collect.Sets;
 | 
			
		||||
import org.junit.Before;
 | 
			
		||||
import org.junit.Test;
 | 
			
		||||
 | 
			
		||||
import java.util.Collections;
 | 
			
		||||
import org.thingsboard.server.common.transport.quota.host.HostRequestIntervalRegistry;
 | 
			
		||||
 | 
			
		||||
import static org.junit.Assert.assertEquals;
 | 
			
		||||
 | 
			
		||||
 | 
			
		||||
@ -18,6 +18,8 @@ package org.thingsboard.server.common.transport.quota.inmemory;
 | 
			
		||||
import com.google.common.collect.ImmutableMap;
 | 
			
		||||
import org.junit.Before;
 | 
			
		||||
import org.junit.Test;
 | 
			
		||||
import org.thingsboard.server.common.transport.quota.host.HostIntervalRegistryLogger;
 | 
			
		||||
import org.thingsboard.server.common.transport.quota.host.HostRequestIntervalRegistry;
 | 
			
		||||
 | 
			
		||||
import java.util.Collections;
 | 
			
		||||
import java.util.Map;
 | 
			
		||||
@ -37,7 +39,7 @@ public class IntervalRegistryLoggerTest {
 | 
			
		||||
 | 
			
		||||
    @Before
 | 
			
		||||
    public void init() {
 | 
			
		||||
        logger = new IntervalRegistryLogger(3, 10, requestRegistry);
 | 
			
		||||
        logger = new HostIntervalRegistryLogger(3, 10, requestRegistry);
 | 
			
		||||
    }
 | 
			
		||||
 | 
			
		||||
    @Test
 | 
			
		||||
 | 
			
		||||
@ -13,7 +13,7 @@
 | 
			
		||||
 * See the License for the specific language governing permissions and
 | 
			
		||||
 * limitations under the License.
 | 
			
		||||
 */
 | 
			
		||||
package org.thingsboard.server.dao.queue.db.nosql;
 | 
			
		||||
package org.thingsboard.server.dao.queue.db;
 | 
			
		||||
 | 
			
		||||
import lombok.Data;
 | 
			
		||||
import lombok.EqualsAndHashCode;
 | 
			
		||||
@ -13,10 +13,11 @@
 | 
			
		||||
 * See the License for the specific language governing permissions and
 | 
			
		||||
 * limitations under the License.
 | 
			
		||||
 */
 | 
			
		||||
package org.thingsboard.server.dao.queue.db.nosql;
 | 
			
		||||
package org.thingsboard.server.dao.queue.db;
 | 
			
		||||
 | 
			
		||||
import org.springframework.stereotype.Component;
 | 
			
		||||
import org.thingsboard.server.common.msg.TbMsg;
 | 
			
		||||
import org.thingsboard.server.dao.queue.db.MsgAck;
 | 
			
		||||
 | 
			
		||||
import java.util.Collection;
 | 
			
		||||
import java.util.List;
 | 
			
		||||
@ -26,6 +26,8 @@ import org.springframework.stereotype.Component;
 | 
			
		||||
import org.thingsboard.server.common.data.id.TenantId;
 | 
			
		||||
import org.thingsboard.server.common.msg.TbMsg;
 | 
			
		||||
import org.thingsboard.server.dao.queue.MsgQueue;
 | 
			
		||||
import org.thingsboard.server.dao.queue.db.MsgAck;
 | 
			
		||||
import org.thingsboard.server.dao.queue.db.UnprocessedMsgFilter;
 | 
			
		||||
import org.thingsboard.server.dao.queue.db.repository.AckRepository;
 | 
			
		||||
import org.thingsboard.server.dao.queue.db.repository.MsgRepository;
 | 
			
		||||
import org.thingsboard.server.dao.util.NoSqlDao;
 | 
			
		||||
 | 
			
		||||
@ -26,7 +26,7 @@ import com.google.common.util.concurrent.ListenableFuture;
 | 
			
		||||
import org.springframework.beans.factory.annotation.Value;
 | 
			
		||||
import org.springframework.stereotype.Component;
 | 
			
		||||
import org.thingsboard.server.dao.nosql.CassandraAbstractDao;
 | 
			
		||||
import org.thingsboard.server.dao.queue.db.nosql.MsgAck;
 | 
			
		||||
import org.thingsboard.server.dao.queue.db.MsgAck;
 | 
			
		||||
import org.thingsboard.server.dao.queue.db.repository.AckRepository;
 | 
			
		||||
import org.thingsboard.server.dao.util.NoSqlDao;
 | 
			
		||||
 | 
			
		||||
 | 
			
		||||
@ -16,7 +16,7 @@
 | 
			
		||||
package org.thingsboard.server.dao.queue.db.repository;
 | 
			
		||||
 | 
			
		||||
import com.google.common.util.concurrent.ListenableFuture;
 | 
			
		||||
import org.thingsboard.server.dao.queue.db.nosql.MsgAck;
 | 
			
		||||
import org.thingsboard.server.dao.queue.db.MsgAck;
 | 
			
		||||
 | 
			
		||||
import java.util.List;
 | 
			
		||||
import java.util.UUID;
 | 
			
		||||
 | 
			
		||||
@ -16,6 +16,7 @@
 | 
			
		||||
package org.thingsboard.server.dao.relation;
 | 
			
		||||
 | 
			
		||||
import com.google.common.base.Function;
 | 
			
		||||
import com.google.common.util.concurrent.AsyncFunction;
 | 
			
		||||
import com.google.common.util.concurrent.Futures;
 | 
			
		||||
import com.google.common.util.concurrent.ListenableFuture;
 | 
			
		||||
import lombok.extern.slf4j.Slf4j;
 | 
			
		||||
@ -176,97 +177,65 @@ public class BaseRelationService implements RelationService {
 | 
			
		||||
    }
 | 
			
		||||
 | 
			
		||||
    @Override
 | 
			
		||||
    public boolean deleteEntityRelations(EntityId entity) {
 | 
			
		||||
        Cache cache = cacheManager.getCache(RELATIONS_CACHE);
 | 
			
		||||
        log.trace("Executing deleteEntityRelations [{}]", entity);
 | 
			
		||||
        validate(entity);
 | 
			
		||||
        List<ListenableFuture<List<EntityRelation>>> inboundRelationsList = new ArrayList<>();
 | 
			
		||||
        for (RelationTypeGroup typeGroup : RelationTypeGroup.values()) {
 | 
			
		||||
            inboundRelationsList.add(relationDao.findAllByTo(entity, typeGroup));
 | 
			
		||||
        }
 | 
			
		||||
        ListenableFuture<List<List<EntityRelation>>> inboundRelations = Futures.allAsList(inboundRelationsList);
 | 
			
		||||
        ListenableFuture<List<Boolean>> inboundDeletions = Futures.transform(inboundRelations, relations ->
 | 
			
		||||
                getBooleans(relations, cache, true));
 | 
			
		||||
 | 
			
		||||
        ListenableFuture<Boolean> inboundFuture = Futures.transform(inboundDeletions, getListToBooleanFunction());
 | 
			
		||||
        boolean inboundDeleteResult = false;
 | 
			
		||||
    public void deleteEntityRelations(EntityId entityId) {
 | 
			
		||||
        try {
 | 
			
		||||
            inboundDeleteResult = inboundFuture.get();
 | 
			
		||||
            deleteEntityRelationsAsync(entityId).get();
 | 
			
		||||
        } catch (InterruptedException | ExecutionException e) {
 | 
			
		||||
            log.error("Error deleting entity inbound relations", e);
 | 
			
		||||
            throw new RuntimeException(e);
 | 
			
		||||
        }
 | 
			
		||||
 | 
			
		||||
        List<ListenableFuture<List<EntityRelation>>> outboundRelationsList = new ArrayList<>();
 | 
			
		||||
        for (RelationTypeGroup typeGroup : RelationTypeGroup.values()) {
 | 
			
		||||
            outboundRelationsList.add(relationDao.findAllByFrom(entity, typeGroup));
 | 
			
		||||
        }
 | 
			
		||||
        ListenableFuture<List<List<EntityRelation>>> outboundRelations = Futures.allAsList(outboundRelationsList);
 | 
			
		||||
        Futures.transform(outboundRelations, relations -> getBooleans(relations, cache, false));
 | 
			
		||||
 | 
			
		||||
        boolean outboundDeleteResult = relationDao.deleteOutboundRelations(entity);
 | 
			
		||||
        return inboundDeleteResult && outboundDeleteResult;
 | 
			
		||||
    }
 | 
			
		||||
 | 
			
		||||
    private List<Boolean> getBooleans(List<List<EntityRelation>> relations, Cache cache, boolean isRemove) {
 | 
			
		||||
        List<Boolean> results = new ArrayList<>();
 | 
			
		||||
        for (List<EntityRelation> relationList : relations) {
 | 
			
		||||
            relationList.forEach(relation -> checkFromDeleteSync(cache, results, relation, isRemove));
 | 
			
		||||
        }
 | 
			
		||||
        return results;
 | 
			
		||||
    }
 | 
			
		||||
 | 
			
		||||
    private void checkFromDeleteSync(Cache cache, List<Boolean> results, EntityRelation relation, boolean isRemove) {
 | 
			
		||||
        if (isRemove) {
 | 
			
		||||
            results.add(relationDao.deleteRelation(relation));
 | 
			
		||||
        }
 | 
			
		||||
        cacheEviction(relation, cache);
 | 
			
		||||
    }
 | 
			
		||||
 | 
			
		||||
    @Override
 | 
			
		||||
    public ListenableFuture<Boolean> deleteEntityRelationsAsync(EntityId entity) {
 | 
			
		||||
    public ListenableFuture<Void> deleteEntityRelationsAsync(EntityId entityId) {
 | 
			
		||||
        Cache cache = cacheManager.getCache(RELATIONS_CACHE);
 | 
			
		||||
        log.trace("Executing deleteEntityRelationsAsync [{}]", entity);
 | 
			
		||||
        validate(entity);
 | 
			
		||||
        log.trace("Executing deleteEntityRelationsAsync [{}]", entityId);
 | 
			
		||||
        validate(entityId);
 | 
			
		||||
        List<ListenableFuture<List<EntityRelation>>> inboundRelationsList = new ArrayList<>();
 | 
			
		||||
        for (RelationTypeGroup typeGroup : RelationTypeGroup.values()) {
 | 
			
		||||
            inboundRelationsList.add(relationDao.findAllByTo(entity, typeGroup));
 | 
			
		||||
            inboundRelationsList.add(relationDao.findAllByTo(entityId, typeGroup));
 | 
			
		||||
        }
 | 
			
		||||
        ListenableFuture<List<List<EntityRelation>>> inboundRelations = Futures.allAsList(inboundRelationsList);
 | 
			
		||||
        ListenableFuture<List<Boolean>> inboundDeletions = Futures.transformAsync(inboundRelations,
 | 
			
		||||
                relations -> {
 | 
			
		||||
                    List<ListenableFuture<Boolean>> results = getListenableFutures(relations, cache, true);
 | 
			
		||||
                    return Futures.allAsList(results);
 | 
			
		||||
                });
 | 
			
		||||
 | 
			
		||||
        ListenableFuture<Boolean> inboundFuture = Futures.transform(inboundDeletions, getListToBooleanFunction());
 | 
			
		||||
        ListenableFuture<List<List<EntityRelation>>> inboundRelations = Futures.allAsList(inboundRelationsList);
 | 
			
		||||
 | 
			
		||||
        List<ListenableFuture<List<EntityRelation>>> outboundRelationsList = new ArrayList<>();
 | 
			
		||||
        for (RelationTypeGroup typeGroup : RelationTypeGroup.values()) {
 | 
			
		||||
            outboundRelationsList.add(relationDao.findAllByFrom(entity, typeGroup));
 | 
			
		||||
            outboundRelationsList.add(relationDao.findAllByFrom(entityId, typeGroup));
 | 
			
		||||
        }
 | 
			
		||||
        ListenableFuture<List<List<EntityRelation>>> outboundRelations = Futures.allAsList(outboundRelationsList);
 | 
			
		||||
        Futures.transformAsync(outboundRelations, relations -> {
 | 
			
		||||
            List<ListenableFuture<Boolean>> results = getListenableFutures(relations, cache, false);
 | 
			
		||||
            return Futures.allAsList(results);
 | 
			
		||||
        });
 | 
			
		||||
 | 
			
		||||
        ListenableFuture<Boolean> outboundFuture = relationDao.deleteOutboundRelationsAsync(entity);
 | 
			
		||||
        return Futures.transform(Futures.allAsList(Arrays.asList(inboundFuture, outboundFuture)), getListToBooleanFunction());
 | 
			
		||||
        ListenableFuture<List<List<EntityRelation>>> outboundRelations = Futures.allAsList(outboundRelationsList);
 | 
			
		||||
 | 
			
		||||
        ListenableFuture<List<Boolean>> inboundDeletions = Futures.transformAsync(inboundRelations,
 | 
			
		||||
                relations -> {
 | 
			
		||||
                    List<ListenableFuture<Boolean>> results = deleteRelationGroupsAsync(relations, cache, true);
 | 
			
		||||
                    return Futures.allAsList(results);
 | 
			
		||||
                });
 | 
			
		||||
 | 
			
		||||
        ListenableFuture<List<Boolean>> outboundDeletions = Futures.transformAsync(outboundRelations,
 | 
			
		||||
                relations -> {
 | 
			
		||||
                    List<ListenableFuture<Boolean>> results = deleteRelationGroupsAsync(relations, cache, false);
 | 
			
		||||
                    return Futures.allAsList(results);
 | 
			
		||||
                });
 | 
			
		||||
 | 
			
		||||
        ListenableFuture<List<List<Boolean>>> deletionsFuture = Futures.allAsList(inboundDeletions, outboundDeletions);
 | 
			
		||||
 | 
			
		||||
        return Futures.transform(Futures.transformAsync(deletionsFuture, (deletions) -> relationDao.deleteOutboundRelationsAsync(entityId)), result -> null);
 | 
			
		||||
    }
 | 
			
		||||
 | 
			
		||||
    private List<ListenableFuture<Boolean>> getListenableFutures(List<List<EntityRelation>> relations, Cache cache, boolean isRemove) {
 | 
			
		||||
    private List<ListenableFuture<Boolean>> deleteRelationGroupsAsync(List<List<EntityRelation>> relations, Cache cache, boolean deleteFromDb) {
 | 
			
		||||
        List<ListenableFuture<Boolean>> results = new ArrayList<>();
 | 
			
		||||
        for (List<EntityRelation> relationList : relations) {
 | 
			
		||||
            relationList.forEach(relation -> checkFromDeleteAsync(cache, results, relation, isRemove));
 | 
			
		||||
            relationList.forEach(relation -> results.add(deleteAsync(cache, relation, deleteFromDb)));
 | 
			
		||||
        }
 | 
			
		||||
        return results;
 | 
			
		||||
    }
 | 
			
		||||
 | 
			
		||||
    private void checkFromDeleteAsync(Cache cache, List<ListenableFuture<Boolean>> results, EntityRelation relation, boolean isRemove) {
 | 
			
		||||
        if (isRemove) {
 | 
			
		||||
            results.add(relationDao.deleteRelationAsync(relation));
 | 
			
		||||
        }
 | 
			
		||||
    private ListenableFuture<Boolean> deleteAsync(Cache cache, EntityRelation relation, boolean deleteFromDb) {
 | 
			
		||||
        cacheEviction(relation, cache);
 | 
			
		||||
        if (deleteFromDb) {
 | 
			
		||||
            return relationDao.deleteRelationAsync(relation);
 | 
			
		||||
        } else {
 | 
			
		||||
            return Futures.immediateFuture(false);
 | 
			
		||||
        }
 | 
			
		||||
    }
 | 
			
		||||
 | 
			
		||||
    private void cacheEviction(EntityRelation relation, Cache cache) {
 | 
			
		||||
 | 
			
		||||
@ -23,6 +23,7 @@ import org.thingsboard.server.common.data.relation.EntityRelationsQuery;
 | 
			
		||||
import org.thingsboard.server.common.data.relation.RelationTypeGroup;
 | 
			
		||||
 | 
			
		||||
import java.util.List;
 | 
			
		||||
import java.util.concurrent.ExecutionException;
 | 
			
		||||
 | 
			
		||||
/**
 | 
			
		||||
 * Created by ashvayka on 27.04.17.
 | 
			
		||||
@ -47,9 +48,9 @@ public interface RelationService {
 | 
			
		||||
 | 
			
		||||
    ListenableFuture<Boolean> deleteRelationAsync(EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup);
 | 
			
		||||
 | 
			
		||||
    boolean deleteEntityRelations(EntityId entity);
 | 
			
		||||
    void deleteEntityRelations(EntityId entity);
 | 
			
		||||
 | 
			
		||||
    ListenableFuture<Boolean> deleteEntityRelationsAsync(EntityId entity);
 | 
			
		||||
    ListenableFuture<Void> deleteEntityRelationsAsync(EntityId entity);
 | 
			
		||||
 | 
			
		||||
    List<EntityRelation> findByFrom(EntityId from, RelationTypeGroup typeGroup);
 | 
			
		||||
 | 
			
		||||
 | 
			
		||||
@ -127,39 +127,35 @@ public class JpaRelationDao extends JpaAbstractDaoListeningExecutorService imple
 | 
			
		||||
    @Override
 | 
			
		||||
    public boolean deleteRelation(EntityRelation relation) {
 | 
			
		||||
        RelationCompositeKey key = new RelationCompositeKey(relation);
 | 
			
		||||
        boolean relationExistsBeforeDelete = relationRepository.exists(key);
 | 
			
		||||
        relationRepository.delete(key);
 | 
			
		||||
        return relationExistsBeforeDelete;
 | 
			
		||||
        return deleteRelationIfExists(key);
 | 
			
		||||
    }
 | 
			
		||||
 | 
			
		||||
    @Override
 | 
			
		||||
    public ListenableFuture<Boolean> deleteRelationAsync(EntityRelation relation) {
 | 
			
		||||
        RelationCompositeKey key = new RelationCompositeKey(relation);
 | 
			
		||||
        return service.submit(
 | 
			
		||||
                () -> {
 | 
			
		||||
                    boolean relationExistsBeforeDelete = relationRepository.exists(key);
 | 
			
		||||
                    relationRepository.delete(key);
 | 
			
		||||
                    return relationExistsBeforeDelete;
 | 
			
		||||
                });
 | 
			
		||||
                () -> deleteRelationIfExists(key));
 | 
			
		||||
    }
 | 
			
		||||
 | 
			
		||||
    @Override
 | 
			
		||||
    public boolean deleteRelation(EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup) {
 | 
			
		||||
        RelationCompositeKey key = getRelationCompositeKey(from, to, relationType, typeGroup);
 | 
			
		||||
        boolean relationExistsBeforeDelete = relationRepository.exists(key);
 | 
			
		||||
        relationRepository.delete(key);
 | 
			
		||||
        return relationExistsBeforeDelete;
 | 
			
		||||
        return deleteRelationIfExists(key);
 | 
			
		||||
    }
 | 
			
		||||
 | 
			
		||||
    @Override
 | 
			
		||||
    public ListenableFuture<Boolean> deleteRelationAsync(EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup) {
 | 
			
		||||
        RelationCompositeKey key = getRelationCompositeKey(from, to, relationType, typeGroup);
 | 
			
		||||
        return service.submit(
 | 
			
		||||
                () -> {
 | 
			
		||||
                    boolean relationExistsBeforeDelete = relationRepository.exists(key);
 | 
			
		||||
                    relationRepository.delete(key);
 | 
			
		||||
                    return relationExistsBeforeDelete;
 | 
			
		||||
                });
 | 
			
		||||
                () -> deleteRelationIfExists(key));
 | 
			
		||||
    }
 | 
			
		||||
 | 
			
		||||
    private boolean deleteRelationIfExists(RelationCompositeKey key) {
 | 
			
		||||
        boolean relationExistsBeforeDelete = relationRepository.exists(key);
 | 
			
		||||
        if (relationExistsBeforeDelete) {
 | 
			
		||||
            relationRepository.delete(key);
 | 
			
		||||
        }
 | 
			
		||||
        return relationExistsBeforeDelete;
 | 
			
		||||
    }
 | 
			
		||||
 | 
			
		||||
    @Override
 | 
			
		||||
@ -167,7 +163,9 @@ public class JpaRelationDao extends JpaAbstractDaoListeningExecutorService imple
 | 
			
		||||
        boolean relationExistsBeforeDelete = relationRepository
 | 
			
		||||
                .findAllByFromIdAndFromType(UUIDConverter.fromTimeUUID(entity.getId()), entity.getEntityType().name())
 | 
			
		||||
                .size() > 0;
 | 
			
		||||
        relationRepository.deleteByFromIdAndFromType(UUIDConverter.fromTimeUUID(entity.getId()), entity.getEntityType().name());
 | 
			
		||||
        if (relationExistsBeforeDelete) {
 | 
			
		||||
            relationRepository.deleteByFromIdAndFromType(UUIDConverter.fromTimeUUID(entity.getId()), entity.getEntityType().name());
 | 
			
		||||
        }
 | 
			
		||||
        return relationExistsBeforeDelete;
 | 
			
		||||
    }
 | 
			
		||||
 | 
			
		||||
@ -178,7 +176,9 @@ public class JpaRelationDao extends JpaAbstractDaoListeningExecutorService imple
 | 
			
		||||
                    boolean relationExistsBeforeDelete = relationRepository
 | 
			
		||||
                            .findAllByFromIdAndFromType(UUIDConverter.fromTimeUUID(entity.getId()), entity.getEntityType().name())
 | 
			
		||||
                            .size() > 0;
 | 
			
		||||
                    relationRepository.deleteByFromIdAndFromType(UUIDConverter.fromTimeUUID(entity.getId()), entity.getEntityType().name());
 | 
			
		||||
                    if (relationExistsBeforeDelete) {
 | 
			
		||||
                        relationRepository.deleteByFromIdAndFromType(UUIDConverter.fromTimeUUID(entity.getId()), entity.getEntityType().name());
 | 
			
		||||
                    }
 | 
			
		||||
                    return relationExistsBeforeDelete;
 | 
			
		||||
                });
 | 
			
		||||
    }
 | 
			
		||||
 | 
			
		||||
@ -18,6 +18,8 @@ package org.thingsboard.server.dao.queue.db.nosql;
 | 
			
		||||
import com.google.common.collect.Lists;
 | 
			
		||||
import org.junit.Test;
 | 
			
		||||
import org.thingsboard.server.common.msg.TbMsg;
 | 
			
		||||
import org.thingsboard.server.dao.queue.db.MsgAck;
 | 
			
		||||
import org.thingsboard.server.dao.queue.db.UnprocessedMsgFilter;
 | 
			
		||||
 | 
			
		||||
import java.util.Collection;
 | 
			
		||||
import java.util.List;
 | 
			
		||||
 | 
			
		||||
@ -21,9 +21,9 @@ import com.google.common.util.concurrent.ListenableFuture;
 | 
			
		||||
import org.junit.Test;
 | 
			
		||||
import org.springframework.beans.factory.annotation.Autowired;
 | 
			
		||||
import org.springframework.test.util.ReflectionTestUtils;
 | 
			
		||||
import org.thingsboard.server.dao.queue.db.nosql.MsgAck;
 | 
			
		||||
import org.thingsboard.server.dao.service.AbstractServiceTest;
 | 
			
		||||
import org.thingsboard.server.dao.service.DaoNoSqlTest;
 | 
			
		||||
import org.thingsboard.server.dao.queue.db.MsgAck;
 | 
			
		||||
 | 
			
		||||
import java.util.List;
 | 
			
		||||
import java.util.UUID;
 | 
			
		||||
 | 
			
		||||
@ -96,7 +96,7 @@ public abstract class BaseRelationServiceTest extends AbstractServiceTest {
 | 
			
		||||
        saveRelation(relationA);
 | 
			
		||||
        saveRelation(relationB);
 | 
			
		||||
 | 
			
		||||
        Assert.assertTrue(relationService.deleteEntityRelationsAsync(childId).get());
 | 
			
		||||
        Assert.assertNull(relationService.deleteEntityRelationsAsync(childId).get());
 | 
			
		||||
 | 
			
		||||
        Assert.assertFalse(relationService.checkRelation(parentId, childId, EntityRelation.CONTAINS_TYPE, RelationTypeGroup.COMMON).get());
 | 
			
		||||
 | 
			
		||||
 | 
			
		||||
@ -30,4 +30,3 @@ redis.connection.db=0
 | 
			
		||||
redis.connection.password=
 | 
			
		||||
 | 
			
		||||
rule.queue.type=memory
 | 
			
		||||
rule.queue.max_size=10000
 | 
			
		||||
@ -27,6 +27,7 @@ import org.springframework.stereotype.Service;
 | 
			
		||||
import org.thingsboard.server.common.transport.SessionMsgProcessor;
 | 
			
		||||
import org.thingsboard.server.common.transport.auth.DeviceAuthService;
 | 
			
		||||
import org.thingsboard.server.common.transport.quota.QuotaService;
 | 
			
		||||
import org.thingsboard.server.common.transport.quota.host.HostRequestsQuotaService;
 | 
			
		||||
import org.thingsboard.server.transport.coap.adaptors.CoapTransportAdaptor;
 | 
			
		||||
 | 
			
		||||
import javax.annotation.PostConstruct;
 | 
			
		||||
@ -55,7 +56,7 @@ public class CoapTransportService {
 | 
			
		||||
    private DeviceAuthService authService;
 | 
			
		||||
 | 
			
		||||
    @Autowired(required = false)
 | 
			
		||||
    private QuotaService quotaService;
 | 
			
		||||
    private HostRequestsQuotaService quotaService;
 | 
			
		||||
 | 
			
		||||
 | 
			
		||||
    @Value("${coap.bind_address}")
 | 
			
		||||
 | 
			
		||||
@ -50,7 +50,7 @@ import org.thingsboard.server.common.msg.session.*;
 | 
			
		||||
import org.thingsboard.server.common.transport.SessionMsgProcessor;
 | 
			
		||||
import org.thingsboard.server.common.transport.auth.DeviceAuthResult;
 | 
			
		||||
import org.thingsboard.server.common.transport.auth.DeviceAuthService;
 | 
			
		||||
import org.thingsboard.server.common.transport.quota.QuotaService;
 | 
			
		||||
import org.thingsboard.server.common.transport.quota.host.HostRequestsQuotaService;
 | 
			
		||||
 | 
			
		||||
import java.util.ArrayList;
 | 
			
		||||
import java.util.List;
 | 
			
		||||
@ -134,8 +134,8 @@ public class CoapServerTest {
 | 
			
		||||
        }
 | 
			
		||||
 | 
			
		||||
        @Bean
 | 
			
		||||
        public static QuotaService quotaService() {
 | 
			
		||||
            return key -> false;
 | 
			
		||||
        public static HostRequestsQuotaService quotaService() {
 | 
			
		||||
            return new HostRequestsQuotaService(null, null, null, null, false);
 | 
			
		||||
        }
 | 
			
		||||
    }
 | 
			
		||||
 | 
			
		||||
 | 
			
		||||
@ -36,6 +36,7 @@ import org.thingsboard.server.common.transport.SessionMsgProcessor;
 | 
			
		||||
import org.thingsboard.server.common.transport.adaptor.JsonConverter;
 | 
			
		||||
import org.thingsboard.server.common.transport.auth.DeviceAuthService;
 | 
			
		||||
import org.thingsboard.server.common.transport.quota.QuotaService;
 | 
			
		||||
import org.thingsboard.server.common.transport.quota.host.HostRequestsQuotaService;
 | 
			
		||||
import org.thingsboard.server.transport.http.session.HttpSessionCtx;
 | 
			
		||||
 | 
			
		||||
import javax.servlet.http.HttpServletRequest;
 | 
			
		||||
@ -61,7 +62,7 @@ public class DeviceApiController {
 | 
			
		||||
    private DeviceAuthService authService;
 | 
			
		||||
 | 
			
		||||
    @Autowired(required = false)
 | 
			
		||||
    private QuotaService quotaService;
 | 
			
		||||
    private HostRequestsQuotaService quotaService;
 | 
			
		||||
 | 
			
		||||
    @RequestMapping(value = "/{deviceToken}/attributes", method = RequestMethod.GET, produces = "application/json")
 | 
			
		||||
    public DeferredResult<ResponseEntity> getDeviceAttributes(@PathVariable("deviceToken") String deviceToken,
 | 
			
		||||
 | 
			
		||||
@ -29,7 +29,7 @@ import org.springframework.context.ApplicationContext;
 | 
			
		||||
import org.springframework.stereotype.Service;
 | 
			
		||||
import org.thingsboard.server.common.transport.SessionMsgProcessor;
 | 
			
		||||
import org.thingsboard.server.common.transport.auth.DeviceAuthService;
 | 
			
		||||
import org.thingsboard.server.common.transport.quota.QuotaService;
 | 
			
		||||
import org.thingsboard.server.common.transport.quota.host.HostRequestsQuotaService;
 | 
			
		||||
import org.thingsboard.server.dao.device.DeviceService;
 | 
			
		||||
import org.thingsboard.server.dao.relation.RelationService;
 | 
			
		||||
import org.thingsboard.server.transport.mqtt.adaptors.MqttTransportAdaptor;
 | 
			
		||||
@ -67,7 +67,7 @@ public class MqttTransportService {
 | 
			
		||||
    private MqttSslHandlerProvider sslHandlerProvider;
 | 
			
		||||
 | 
			
		||||
    @Autowired(required = false)
 | 
			
		||||
    private QuotaService quotaService;
 | 
			
		||||
    private HostRequestsQuotaService quotaService;
 | 
			
		||||
 | 
			
		||||
    @Value("${mqtt.bind_address}")
 | 
			
		||||
    private String host;
 | 
			
		||||
 | 
			
		||||
		Loading…
	
	
			
			x
			
			
		
	
		Reference in New Issue
	
	Block a user