dendi-wang/thingsboard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #11949 from thingsboard/fix/gateways-dashboard

Browse Source 
Don't create gateways dashboard for new tenants; process images for system dashboard resource
This commit is contained in:
Viacheslav Klimov 2024-10-29 14:24:56 +02:00 committed by GitHub
commit 8f5305b6f2
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 53 additions and 7073 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7023
application/src/main/data/json/tenant/dashboards/gateways.json
View File

File diff suppressed because it is too large Load Diff

7
application/src/main/java/org/thingsboard/server/service/entitiy/dashboard/DashboardSyncService.java
View File

@ -18,6 +18,7 @@ package org.thingsboard.server.service.entitiy.dashboard;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.thingsboard.server.common.data.ResourceType; import org.thingsboard.server.common.data.ResourceType;
import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantId;
@ -39,6 +40,7 @@ import java.util.stream.Stream;
@TbCoreComponent @TbCoreComponent
@RequiredArgsConstructor @RequiredArgsConstructor
@Slf4j @Slf4j
@ConditionalOnProperty(value = "transport.gateway.dashboard.sync.enabled", havingValue = "true")
public class DashboardSyncService { public class DashboardSyncService {
private final GitSyncService gitSyncService; private final GitSyncService gitSyncService;
@ -46,8 +48,6 @@ public class DashboardSyncService {
private final WidgetsBundleService widgetsBundleService; private final WidgetsBundleService widgetsBundleService;
private final PartitionService partitionService; private final PartitionService partitionService;
@Value("${transport.gateway.dashboard.sync.enabled:true}")
private boolean enabled;
@Value("${transport.gateway.dashboard.sync.repository_url:}") @Value("${transport.gateway.dashboard.sync.repository_url:}")
private String repoUrl; private String repoUrl;
@Value("${transport.gateway.dashboard.sync.branch:main}") @Value("${transport.gateway.dashboard.sync.branch:main}")
@ -60,9 +60,6 @@ public class DashboardSyncService {
@AfterStartUp(order = AfterStartUp.REGULAR_SERVICE) @AfterStartUp(order = AfterStartUp.REGULAR_SERVICE)
public void init() throws Exception { public void init() throws Exception {
if (!enabled) {
return;
}
gitSyncService.registerSync(REPO_KEY, repoUrl, branch, TimeUnit.HOURS.toMillis(fetchFrequencyHours), this::update); gitSyncService.registerSync(REPO_KEY, repoUrl, branch, TimeUnit.HOURS.toMillis(fetchFrequencyHours), this::update);
} }

54
application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java
View File

@ -19,7 +19,6 @@ import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ObjectNode; import com.fasterxml.jackson.databind.node.ObjectNode;
import lombok.Getter; import lombok.Getter;
import lombok.Setter; import lombok.Setter;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
@ -54,8 +53,8 @@ import org.thingsboard.server.service.install.update.ImagesUpdater;
import java.io.IOException; import java.io.IOException;
import java.io.UncheckedIOException; import java.io.UncheckedIOException;
import java.nio.file.DirectoryStream;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.NoSuchFileException;
import java.nio.file.Path; import java.nio.file.Path;
import java.nio.file.Paths; import java.nio.file.Paths;
import java.util.ArrayList; import java.util.ArrayList;
@ -170,7 +169,6 @@ public class InstallScripts {
loadRuleChainsFromPath(tenantId, edgeChainsDir); loadRuleChainsFromPath(tenantId, edgeChainsDir);
} }
@SneakyThrows
private void loadRuleChainsFromPath(TenantId tenantId, Path ruleChainsPath) { private void loadRuleChainsFromPath(TenantId tenantId, Path ruleChainsPath) {
findRuleChainsFromPath(ruleChainsPath).forEach(path -> { findRuleChainsFromPath(ruleChainsPath).forEach(path -> {
try { try {
@ -182,12 +180,10 @@ public class InstallScripts {
}); });
} }
List<Path> findRuleChainsFromPath(Path ruleChainsPath) throws IOException { List<Path> findRuleChainsFromPath(Path ruleChainsPath) {
List<Path> paths = new ArrayList<>(); try (Stream<Path> files = listDir(ruleChainsPath).filter(path -> path.toString().endsWith(InstallScripts.JSON_EXT))) {
try (DirectoryStream<Path> dirStream = Files.newDirectoryStream(ruleChainsPath, path -> path.toString().endsWith(InstallScripts.JSON_EXT))) { return files.toList();
dirStream.forEach(paths::add);
} }
return paths;
} }
public RuleChain createDefaultRuleChain(TenantId tenantId, String ruleChainName) { public RuleChain createDefaultRuleChain(TenantId tenantId, String ruleChainName) {
@ -211,11 +207,11 @@ public class InstallScripts {
return ruleChain; return ruleChain;
} }
public void loadSystemWidgets() throws Exception { public void loadSystemWidgets() {
log.info("Loading system widgets"); log.info("Loading system widgets");
Map<Path, JsonNode> widgetsBundlesMap = new HashMap<>(); Map<Path, JsonNode> widgetsBundlesMap = new HashMap<>();
Path widgetBundlesDir = Paths.get(getDataDir(), JSON_DIR, SYSTEM_DIR, WIDGET_BUNDLES_DIR); Path widgetBundlesDir = Paths.get(getDataDir(), JSON_DIR, SYSTEM_DIR, WIDGET_BUNDLES_DIR);
try (DirectoryStream<Path> dirStream = Files.newDirectoryStream(widgetBundlesDir, path -> path.toString().endsWith(JSON_EXT))) { try (Stream<Path> dirStream = listDir(widgetBundlesDir).filter(path -> path.toString().endsWith(JSON_EXT))) {
dirStream.forEach( dirStream.forEach(
path -> { path -> {
JsonNode widgetsBundleDescriptorJson; JsonNode widgetsBundleDescriptorJson;
@ -247,12 +243,14 @@ public class InstallScripts {
} }
Path widgetTypesDir = Paths.get(getDataDir(), JSON_DIR, SYSTEM_DIR, WIDGET_TYPES_DIR); Path widgetTypesDir = Paths.get(getDataDir(), JSON_DIR, SYSTEM_DIR, WIDGET_TYPES_DIR);
if (Files.exists(widgetTypesDir)) { if (Files.exists(widgetTypesDir)) {
try (DirectoryStream<Path> dirStream = Files.newDirectoryStream(widgetTypesDir, path -> path.toString().endsWith(JSON_EXT))) { try (Stream<Path> dirStream = listDir(widgetTypesDir).filter(path -> path.toString().endsWith(JSON_EXT))) {
dirStream.forEach( dirStream.forEach(
path -> { path -> {
try { try {
JsonNode widgetTypeJson = JacksonUtil.toJsonNode(path.toFile()); String widgetTypeJson = Files.readString(path);
WidgetTypeDetails widgetTypeDetails = JacksonUtil.treeToValue(widgetTypeJson, WidgetTypeDetails.class); widgetTypeJson = resourceService.checkSystemResourcesUsage(widgetTypeJson, ResourceType.JS_MODULE);
WidgetTypeDetails widgetTypeDetails = JacksonUtil.fromString(widgetTypeJson, WidgetTypeDetails.class);
widgetTypeService.saveWidgetType(widgetTypeDetails); widgetTypeService.saveWidgetType(widgetTypeDetails);
} catch (Exception e) { } catch (Exception e) {
log.error("Unable to load widget type from json: [{}]", path.toString()); log.error("Unable to load widget type from json: [{}]", path.toString());
@ -300,12 +298,12 @@ public class InstallScripts {
} }
} }
private void loadSystemScadaSymbols() throws Exception { private void loadSystemScadaSymbols() {
log.info("Loading system SCADA symbols"); log.info("Loading system SCADA symbols");
Path scadaSymbolsDir = Paths.get(getDataDir(), JSON_DIR, SYSTEM_DIR, SCADA_SYMBOLS_DIR); Path scadaSymbolsDir = Paths.get(getDataDir(), JSON_DIR, SYSTEM_DIR, SCADA_SYMBOLS_DIR);
if (Files.exists(scadaSymbolsDir)) { if (Files.exists(scadaSymbolsDir)) {
WidgetTypeDetails scadaSymbolWidgetTemplate = widgetTypeService.findWidgetTypeDetailsByTenantIdAndFqn(TenantId.SYS_TENANT_ID, "scada_symbol"); WidgetTypeDetails scadaSymbolWidgetTemplate = widgetTypeService.findWidgetTypeDetailsByTenantIdAndFqn(TenantId.SYS_TENANT_ID, "scada_symbol");
try (DirectoryStream<Path> dirStream = Files.newDirectoryStream(scadaSymbolsDir, path -> path.toString().endsWith(SVG_EXT))) { try (Stream<Path> dirStream = listDir(scadaSymbolsDir).filter(path -> path.toString().endsWith(SVG_EXT))) {
dirStream.forEach( dirStream.forEach(
path -> { path -> {
try { try {
@ -404,11 +402,10 @@ public class InstallScripts {
imagesUpdater.updateAssetProfilesImages(); imagesUpdater.updateAssetProfilesImages();
} }
@SneakyThrows
public void loadSystemImages() { public void loadSystemImages() {
log.info("Loading system images..."); log.info("Loading system images...");
Stream<Path> dashboardsFiles = Stream.concat(Files.list(Paths.get(getDataDir(), JSON_DIR, DEMO_DIR, DASHBOARDS_DIR)), Stream<Path> dashboardsFiles = Stream.concat(listDir(Paths.get(getDataDir(), JSON_DIR, DEMO_DIR, DASHBOARDS_DIR)),
Files.list(Paths.get(getDataDir(), JSON_DIR, TENANT_DIR, DASHBOARDS_DIR))); listDir(Paths.get(getDataDir(), JSON_DIR, TENANT_DIR, DASHBOARDS_DIR)));
try (dashboardsFiles) { try (dashboardsFiles) {
dashboardsFiles.forEach(file -> { dashboardsFiles.forEach(file -> {
try { try {
@ -431,11 +428,9 @@ public class InstallScripts {
loadDashboardsFromDir(tenantId, customerId, dashboardsDir); loadDashboardsFromDir(tenantId, customerId, dashboardsDir);
} }
@SneakyThrows
private void loadDashboardsFromDir(TenantId tenantId, CustomerId customerId, Path dashboardsDir) { private void loadDashboardsFromDir(TenantId tenantId, CustomerId customerId, Path dashboardsDir) {
try (DirectoryStream<Path> dirStream = Files.newDirectoryStream(dashboardsDir, path -> path.toString().endsWith(JSON_EXT))) { try (Stream<Path> dashboards = listDir(dashboardsDir).filter(path -> path.toString().endsWith(JSON_EXT))) {
dirStream.forEach( dashboards.forEach(path -> {
path -> {
try { try {
JsonNode dashboardJson = JacksonUtil.toJsonNode(path.toFile()); JsonNode dashboardJson = JacksonUtil.toJsonNode(path.toFile());
Dashboard dashboard = JacksonUtil.treeToValue(dashboardJson, Dashboard.class); Dashboard dashboard = JacksonUtil.treeToValue(dashboardJson, Dashboard.class);
@ -448,8 +443,7 @@ public class InstallScripts {
log.error("Unable to load dashboard from json: [{}]", path.toString()); log.error("Unable to load dashboard from json: [{}]", path.toString());
throw new RuntimeException("Unable to load dashboard from json", e); throw new RuntimeException("Unable to load dashboard from json", e);
} }
} });
);
} }
} }
@ -464,9 +458,9 @@ public class InstallScripts {
} }
} }
public void createOAuth2Templates() throws Exception { public void createOAuth2Templates() {
Path oauth2ConfigTemplatesDir = Paths.get(getDataDir(), JSON_DIR, SYSTEM_DIR, OAUTH2_CONFIG_TEMPLATES_DIR); Path oauth2ConfigTemplatesDir = Paths.get(getDataDir(), JSON_DIR, SYSTEM_DIR, OAUTH2_CONFIG_TEMPLATES_DIR);
try (DirectoryStream<Path> dirStream = Files.newDirectoryStream(oauth2ConfigTemplatesDir, path -> path.toString().endsWith(JSON_EXT))) { try (Stream<Path> dirStream = listDir(oauth2ConfigTemplatesDir).filter(path -> path.toString().endsWith(JSON_EXT))) {
dirStream.forEach( dirStream.forEach(
path -> { path -> {
try { try {
@ -489,7 +483,7 @@ public class InstallScripts {
public void loadSystemLwm2mResources() { public void loadSystemLwm2mResources() {
Path resourceLwm2mPath = Paths.get(getDataDir(), MODELS_LWM2M_DIR); Path resourceLwm2mPath = Paths.get(getDataDir(), MODELS_LWM2M_DIR);
try (DirectoryStream<Path> dirStream = Files.newDirectoryStream(resourceLwm2mPath, path -> path.toString().endsWith(InstallScripts.XML_EXT))) { try (Stream<Path> dirStream = listDir(resourceLwm2mPath).filter(path -> path.toString().endsWith(InstallScripts.XML_EXT))) {
dirStream.forEach( dirStream.forEach(
path -> { path -> {
try { try {
@ -539,9 +533,11 @@ public class InstallScripts {
} }
} }
private Stream<Path> listDir(Path resourcesDir) { private Stream<Path> listDir(Path dir) {
try { try {
return Files.list(resourcesDir); return Files.list(dir);
} catch (NoSuchFileException e) {
return Stream.empty();
} catch (IOException e) { } catch (IOException e) {
throw new UncheckedIOException(e); throw new UncheckedIOException(e);
} }

5
application/src/test/java/org/thingsboard/server/service/install/InstallScriptsTest.java
View File

@ -39,7 +39,6 @@ import org.thingsboard.server.dao.widget.WidgetTypeService;
import org.thingsboard.server.dao.widget.WidgetsBundleService; import org.thingsboard.server.dao.widget.WidgetsBundleService;
import org.thingsboard.server.service.install.update.ImagesUpdater; import org.thingsboard.server.service.install.update.ImagesUpdater;
import java.io.IOException;
import java.nio.file.Path; import java.nio.file.Path;
import java.util.Collections; import java.util.Collections;
import java.util.List; import java.util.List;
@ -87,14 +86,14 @@ class InstallScriptsTest {
} }
@Test @Test
void testDefaultRuleChainsTemplates() throws IOException { void testDefaultRuleChainsTemplates() {
Path dir = installScripts.getTenantRuleChainsDir(); Path dir = installScripts.getTenantRuleChainsDir();
installScripts.findRuleChainsFromPath(dir) installScripts.findRuleChainsFromPath(dir)
.forEach(this::validateRuleChainTemplate); .forEach(this::validateRuleChainTemplate);
} }
@Test @Test
void testDefaultEdgeRuleChainsTemplates() throws IOException { void testDefaultEdgeRuleChainsTemplates() {
Path dir = installScripts.getEdgeRuleChainsDir(); Path dir = installScripts.getEdgeRuleChainsDir();
installScripts.findRuleChainsFromPath(dir) installScripts.findRuleChainsFromPath(dir)
.forEach(this::validateRuleChainTemplate); .forEach(this::validateRuleChainTemplate);

15
dao/src/main/java/org/thingsboard/server/dao/resource/BaseResourceService.java
View File

@ -17,15 +17,19 @@ package org.thingsboard.server.dao.resource;
import com.google.common.hash.Hashing; import com.google.common.hash.Hashing;
import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListenableFuture;
import lombok.AllArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.hibernate.exception.ConstraintViolationException; import org.hibernate.exception.ConstraintViolationException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Lazy;
import org.springframework.context.annotation.Primary; import org.springframework.context.annotation.Primary;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.transaction.event.TransactionalEventListener; import org.springframework.transaction.event.TransactionalEventListener;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.common.util.RegexUtils; import org.thingsboard.common.util.RegexUtils;
import org.thingsboard.server.cache.resourceInfo.ResourceInfoCacheKey; import org.thingsboard.server.cache.resourceInfo.ResourceInfoCacheKey;
import org.thingsboard.server.cache.resourceInfo.ResourceInfoEvictEvent; import org.thingsboard.server.cache.resourceInfo.ResourceInfoEvictEvent;
import org.thingsboard.server.common.data.Dashboard;
import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.ResourceType; import org.thingsboard.server.common.data.ResourceType;
import org.thingsboard.server.common.data.TbResource; import org.thingsboard.server.common.data.TbResource;
@ -54,7 +58,7 @@ import static org.thingsboard.server.dao.service.Validator.validateId;
@Service("TbResourceDaoService") @Service("TbResourceDaoService")
@Slf4j @Slf4j
@AllArgsConstructor @RequiredArgsConstructor
@Primary @Primary
public class BaseResourceService extends AbstractCachedEntityService<ResourceInfoCacheKey, TbResourceInfo, ResourceInfoEvictEvent> implements ResourceService { public class BaseResourceService extends AbstractCachedEntityService<ResourceInfoCacheKey, TbResourceInfo, ResourceInfoEvictEvent> implements ResourceService {
@ -62,6 +66,8 @@ public class BaseResourceService extends AbstractCachedEntityService<ResourceInf
protected final TbResourceDao resourceDao; protected final TbResourceDao resourceDao;
protected final TbResourceInfoDao resourceInfoDao; protected final TbResourceInfoDao resourceInfoDao;
protected final ResourceDataValidator resourceValidator; protected final ResourceDataValidator resourceValidator;
@Autowired @Lazy
private ImageService imageService;
@Override @Override
public TbResource saveResource(TbResource resource, boolean doValidate) { public TbResource saveResource(TbResource resource, boolean doValidate) {
@ -243,6 +249,11 @@ public class BaseResourceService extends AbstractCachedEntityService<ResourceInf
public TbResource createOrUpdateSystemResource(ResourceType resourceType, String resourceKey, String data) { public TbResource createOrUpdateSystemResource(ResourceType resourceType, String resourceKey, String data) {
if (resourceType == ResourceType.DASHBOARD) { if (resourceType == ResourceType.DASHBOARD) {
data = checkSystemResourcesUsage(data, ResourceType.JS_MODULE); data = checkSystemResourcesUsage(data, ResourceType.JS_MODULE);
Dashboard dashboard = JacksonUtil.fromString(data, Dashboard.class);
dashboard.setTenantId(TenantId.SYS_TENANT_ID);
imageService.replaceBase64WithImageUrl(dashboard);
data = JacksonUtil.toString(dashboard);
} }
TbResource resource = findResourceByTenantIdAndKey(TenantId.SYS_TENANT_ID, resourceType, resourceKey); TbResource resource = findResourceByTenantIdAndKey(TenantId.SYS_TENANT_ID, resourceType, resourceKey);