From 7c2283e3cd727425c9d9ed3eea1c47b7e99f56fa Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Fri, 10 Jun 2022 15:10:36 +0300 Subject: [PATCH 1/3] Add indexes for external_id --- .../src/main/data/upgrade/3.3.4/schema_update.sql | 8 ++++++++ dao/src/main/resources/sql/schema-entities-idx.sql | 14 ++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/application/src/main/data/upgrade/3.3.4/schema_update.sql b/application/src/main/data/upgrade/3.3.4/schema_update.sql index babb630e9e..a5184c27e6 100644 --- a/application/src/main/data/upgrade/3.3.4/schema_update.sql +++ b/application/src/main/data/upgrade/3.3.4/schema_update.sql @@ -29,6 +29,14 @@ ALTER TABLE customer ALTER TABLE widgets_bundle ADD COLUMN IF NOT EXISTS external_id UUID; +CREATE INDEX IF NOT EXISTS idx_device_external_id ON device(tenant_id, external_id); +CREATE INDEX IF NOT EXISTS idx_device_profile_external_id ON device_profile(tenant_id, external_id); +CREATE INDEX IF NOT EXISTS idx_asset_external_id ON asset(tenant_id, external_id); +CREATE INDEX IF NOT EXISTS idx_rule_chain_external_id ON rule_chain(tenant_id, external_id); +CREATE INDEX IF NOT EXISTS idx_dashboard_external_id ON dashboard(tenant_id, external_id); +CREATE INDEX IF NOT EXISTS idx_customer_external_id ON customer(tenant_id, external_id); +CREATE INDEX IF NOT EXISTS idx_widgets_bundle_external_id ON widgets_bundle(tenant_id, external_id); + ALTER TABLE admin_settings ADD COLUMN IF NOT EXISTS tenant_id uuid NOT NULL DEFAULT '13814000-1dd2-11b2-8080-808080808080'; diff --git a/dao/src/main/resources/sql/schema-entities-idx.sql b/dao/src/main/resources/sql/schema-entities-idx.sql index b74d935c12..b47462f94b 100644 --- a/dao/src/main/resources/sql/schema-entities-idx.sql +++ b/dao/src/main/resources/sql/schema-entities-idx.sql @@ -51,3 +51,17 @@ CREATE INDEX IF NOT EXISTS idx_attribute_kv_by_key_and_last_update_ts ON attribu CREATE INDEX IF NOT EXISTS idx_audit_log_tenant_id_and_created_time ON audit_log(tenant_id, created_time); CREATE INDEX IF NOT EXISTS idx_rpc_tenant_id_device_id ON rpc(tenant_id, device_id); + +CREATE INDEX IF NOT EXISTS idx_device_external_id ON device(tenant_id, external_id); + +CREATE INDEX IF NOT EXISTS idx_device_profile_external_id ON device_profile(tenant_id, external_id); + +CREATE INDEX IF NOT EXISTS idx_asset_external_id ON asset(tenant_id, external_id); + +CREATE INDEX IF NOT EXISTS idx_rule_chain_external_id ON rule_chain(tenant_id, external_id); + +CREATE INDEX IF NOT EXISTS idx_dashboard_external_id ON dashboard(tenant_id, external_id); + +CREATE INDEX IF NOT EXISTS idx_customer_external_id ON customer(tenant_id, external_id); + +CREATE INDEX IF NOT EXISTS idx_widgets_bundle_external_id ON widgets_bundle(tenant_id, external_id); From cc52452d75b47fa3b8f315955cf1e250ae9ba1cf Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Fri, 10 Jun 2022 16:17:37 +0300 Subject: [PATCH 2/3] Remove admin settings on tenant deletion --- .../sync/vc/TbAbstractVersionControlSettingsService.java | 2 +- .../server/dao/settings/AdminSettingsService.java | 4 +++- .../thingsboard/server/dao/settings/AdminSettingsDao.java | 2 ++ .../server/dao/settings/AdminSettingsServiceImpl.java | 8 +++++++- .../server/dao/sql/settings/AdminSettingsRepository.java | 2 ++ .../server/dao/sql/settings/JpaAdminSettingsDao.java | 6 ++++++ .../thingsboard/server/dao/tenant/TenantServiceImpl.java | 5 +++++ 7 files changed, 26 insertions(+), 3 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/TbAbstractVersionControlSettingsService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/TbAbstractVersionControlSettingsService.java index c9f5f5ee4a..3a674eece0 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/TbAbstractVersionControlSettingsService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/TbAbstractVersionControlSettingsService.java @@ -72,7 +72,7 @@ public abstract class TbAbstractVersionControlSettingsService { boolean removeByTenantIdAndKey(UUID tenantId, String key); + void removeByTenantId(UUID tenantId); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsServiceImpl.java index 22a279639a..7d6f931ef1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsServiceImpl.java @@ -71,9 +71,15 @@ public class AdminSettingsServiceImpl implements AdminSettingsService { } @Override - public boolean deleteAdminSettings(TenantId tenantId, String key) { + public boolean deleteAdminSettingsByTenantIdAndKey(TenantId tenantId, String key) { log.trace("Executing deleteAdminSettings, tenantId [{}], key [{}]", tenantId, key); Validator.validateString(key, "Incorrect key " + key); return adminSettingsDao.removeByTenantIdAndKey(tenantId.getId(), key); } + + @Override + public void deleteAdminSettingsByTenantId(TenantId tenantId) { + adminSettingsDao.removeByTenantId(tenantId.getId()); + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/settings/AdminSettingsRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/settings/AdminSettingsRepository.java index 0e68327fdf..e607517f52 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/settings/AdminSettingsRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/settings/AdminSettingsRepository.java @@ -29,6 +29,8 @@ public interface AdminSettingsRepository extends JpaRepository Date: Fri, 10 Jun 2022 16:48:12 +0300 Subject: [PATCH 3/3] VC request timing out --- .../DefaultGitVersionControlQueueService.java | 34 ++++++++++++++++--- .../sync/vc/data/PendingGitRequest.java | 4 +++ .../src/main/resources/thingsboard.yml | 1 + 3 files changed, 35 insertions(+), 4 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java index 759bc83acf..f5585fac39 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java @@ -20,6 +20,7 @@ import com.google.common.util.concurrent.SettableFuture; import com.google.protobuf.ByteString; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; @@ -34,9 +35,9 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.sync.ie.EntityExportData; -import org.thingsboard.server.common.data.sync.vc.RepositorySettings; import org.thingsboard.server.common.data.sync.vc.EntityVersion; import org.thingsboard.server.common.data.sync.vc.EntityVersionsDiff; +import org.thingsboard.server.common.data.sync.vc.RepositorySettings; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; @@ -53,6 +54,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.VersionControlRespon import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.TbQueueMsgMetadata; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.scheduler.SchedulerComponent; import org.thingsboard.server.queue.util.DataDecodingEncodingService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.sync.vc.data.ClearRepositoryGitRequest; @@ -67,7 +69,13 @@ import org.thingsboard.server.service.sync.vc.data.PendingGitRequest; import org.thingsboard.server.service.sync.vc.data.VersionsDiffGitRequest; import org.thingsboard.server.service.sync.vc.data.VoidGitRequest; -import java.util.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.function.Consumer; import java.util.function.Function; import java.util.stream.Collectors; @@ -81,16 +89,22 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu private final TbClusterService clusterService; private final DataDecodingEncodingService encodingService; private final DefaultEntitiesVersionControlService entitiesVersionControlService; + private final SchedulerComponent scheduler; private final Map> pendingRequestMap = new HashMap<>(); + @Value("${queue.vc.request-timeout:60000}") + private int requestTimeout; + public DefaultGitVersionControlQueueService(TbServiceInfoProvider serviceInfoProvider, TbClusterService clusterService, DataDecodingEncodingService encodingService, - @Lazy DefaultEntitiesVersionControlService entitiesVersionControlService) { + @Lazy DefaultEntitiesVersionControlService entitiesVersionControlService, + SchedulerComponent scheduler) { this.serviceInfoProvider = serviceInfoProvider; this.clusterService = clusterService; this.encodingService = encodingService; this.entitiesVersionControlService = entitiesVersionControlService; + this.scheduler = scheduler; } @Override @@ -275,6 +289,9 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu var requestBody = enrichFunction.apply(newRequestProto(request, settings)); log.trace("[{}][{}] PUSHING request: {}", request.getTenantId(), request.getRequestId(), requestBody); clusterService.pushMsgToVersionControl(request.getTenantId(), requestBody, callback); + request.setTimeoutTask(scheduler.schedule(() -> { + processTimeout(request.getRequestId()); + }, requestTimeout, TimeUnit.MILLISECONDS)); } else { throw new RuntimeException("Future is already done!"); } @@ -338,12 +355,13 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu @Override public void processResponse(VersionControlResponseMsg vcResponseMsg) { UUID requestId = new UUID(vcResponseMsg.getRequestIdMSB(), vcResponseMsg.getRequestIdLSB()); - PendingGitRequest request = pendingRequestMap.get(requestId); + PendingGitRequest request = pendingRequestMap.remove(requestId); if (request == null) { log.debug("[{}] received stale response: {}", requestId, vcResponseMsg); return; } else { log.debug("[{}] processing response: {}", requestId, vcResponseMsg); + request.getTimeoutTask().cancel(true); } var future = request.getFuture(); if (!StringUtils.isEmpty(vcResponseMsg.getError())) { @@ -399,6 +417,14 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu } } + private void processTimeout(UUID requestId) { + PendingGitRequest pendingRequest = pendingRequestMap.remove(requestId); + if (pendingRequest != null) { + log.debug("[{}] request timed out ({} ms}", requestId, requestTimeout); + pendingRequest.getFuture().setException(new TimeoutException("Request timed out")); + } + } + private PageData toPageData(TransportProtos.ListVersionsResponseMsg listVersionsResponse) { var listVersions = listVersionsResponse.getVersionsList().stream().map(this::getEntityVersion).collect(Collectors.toList()); return new PageData<>(listVersions, listVersionsResponse.getTotalPages(), listVersionsResponse.getTotalElements(), listVersionsResponse.getHasNext()); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/PendingGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/PendingGitRequest.java index fbec600ed0..b34806a6ca 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/PendingGitRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/PendingGitRequest.java @@ -17,9 +17,11 @@ package org.thingsboard.server.service.sync.vc.data; import com.google.common.util.concurrent.SettableFuture; import lombok.Getter; +import lombok.Setter; import org.thingsboard.server.common.data.id.TenantId; import java.util.UUID; +import java.util.concurrent.ScheduledFuture; @Getter public class PendingGitRequest { @@ -28,6 +30,8 @@ public class PendingGitRequest { private final UUID requestId; private final TenantId tenantId; private final SettableFuture future; + @Setter + private ScheduledFuture timeoutTask; public PendingGitRequest(TenantId tenantId) { this.createdTime = System.currentTimeMillis(); diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index ac6fe34905..94f5aa3dc1 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1039,6 +1039,7 @@ queue: partitions: "${TB_QUEUE_VC_PARTITIONS:10}" poll-interval: "${TB_QUEUE_VC_INTERVAL_MS:25}" pack-processing-timeout: "${TB_QUEUE_VC_PACK_PROCESSING_TIMEOUT_MS:60000}" + request-timeout: "${TB_QUEUE_VC_REQUEST_TIMEOUT:60000}" js: # JS Eval request topic request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js_eval.requests}"