Implement version control ssh access with private key. Improve VC settings REST methods. Add tenantId field to admin settings. Add DB upgrade.

This commit is contained in:
Igor Kulikov 2022-05-19 12:01:06 +03:00
parent 0905aff7a0
commit 0b6de71768
25 changed files with 412 additions and 155 deletions

View File

@ -345,6 +345,10 @@
<groupId>org.eclipse.jgit</groupId>
<artifactId>org.eclipse.jgit</artifactId>
</dependency>
<dependency>
<groupId>org.eclipse.jgit</groupId>
<artifactId>org.eclipse.jgit.ssh.apache</artifactId>
</dependency>
</dependencies>
<build>

View File

@ -26,3 +26,6 @@ ALTER TABLE dashboard
ADD COLUMN IF NOT EXISTS external_id UUID;
ALTER TABLE customer
ADD COLUMN IF NOT EXISTS external_id UUID;
ALTER TABLE admin_settings
ADD COLUMN IF NOT EXISTS tenant_id uuid NOT NULL DEFAULT '13814000-1dd2-11b2-8080-808080808080';

View File

@ -20,12 +20,7 @@ import io.swagger.annotations.ApiOperation;
import io.swagger.annotations.ApiParam;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.ResponseBody;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.bind.annotation.*;
import org.thingsboard.rule.engine.api.MailService;
import org.thingsboard.rule.engine.api.SmsService;
import org.thingsboard.server.common.data.AdminSettings;
@ -34,14 +29,17 @@ import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.security.model.SecuritySettings;
import org.thingsboard.server.common.data.sms.config.TestSmsRequest;
import org.thingsboard.server.common.data.vc.EntitiesVersionControlSettings;
import org.thingsboard.server.dao.settings.AdminSettingsService;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.security.permission.Operation;
import org.thingsboard.server.service.security.permission.Resource;
import org.thingsboard.server.service.security.system.SystemSecurityService;
import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService;
import org.thingsboard.server.service.update.UpdateService;
import static org.thingsboard.server.controller.ControllerConstants.SYSTEM_AUTHORITY_PARAGRAPH;
import static org.thingsboard.server.controller.ControllerConstants.TENANT_AUTHORITY_PARAGRAPH;
@RestController
@TbCoreComponent
@ -60,6 +58,9 @@ public class AdminController extends BaseController {
@Autowired
private SystemSecurityService systemSecurityService;
@Autowired
private EntitiesVersionControlService versionControlService;
@Autowired
private UpdateService updateService;
@ -96,6 +97,7 @@ public class AdminController extends BaseController {
@RequestBody AdminSettings adminSettings) throws ThingsboardException {
try {
accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.WRITE);
adminSettings.setTenantId(getTenantId());
adminSettings = checkNotNull(adminSettingsService.saveAdminSettings(TenantId.SYS_TENANT_ID, adminSettings));
if (adminSettings.getKey().equals("mail")) {
mailService.updateMailConfiguration();
@ -180,6 +182,53 @@ public class AdminController extends BaseController {
}
}
@ApiOperation(value = "Get version control settings (getVersionControlSettings)",
notes = "Get the version control settings object. " + TENANT_AUTHORITY_PARAGRAPH)
@PreAuthorize("hasAuthority('TENANT_ADMIN')")
@GetMapping("/vcSettings")
@ResponseBody
public EntitiesVersionControlSettings getVersionControlSettings() throws ThingsboardException {
try {
accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.READ);
EntitiesVersionControlSettings versionControlSettings = checkNotNull(versionControlService.getVersionControlSettings(getTenantId()));
versionControlSettings.setPassword(null);
versionControlSettings.setPrivateKey(null);
versionControlSettings.setPrivateKeyPassword(null);
return versionControlSettings;
} catch (Exception e) {
throw handleException(e);
}
}
@ApiOperation(value = "Creates or Updates the version control settings (saveVersionControlSettings)",
notes = "Creates or Updates the version control settings object. " + TENANT_AUTHORITY_PARAGRAPH)
@PreAuthorize("hasAuthority('TENANT_ADMIN')")
@PostMapping("/vsSettings")
public void saveVersionControlSettings(@RequestBody EntitiesVersionControlSettings settings) throws ThingsboardException {
try {
accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.WRITE);
versionControlService.saveVersionControlSettings(getTenantId(), settings);
} catch (Exception e) {
throw handleException(e);
}
}
@ApiOperation(value = "Check version control access (checkVersionControlAccess)",
notes = "Attempts to check version control access. " + TENANT_AUTHORITY_PARAGRAPH)
@PreAuthorize("hasAuthority('TENANT_ADMIN')")
@RequestMapping(value = "/vcSettings/checkAccess", method = RequestMethod.POST)
public void checkVersionControlAccess(
@ApiParam(value = "A JSON value representing the Entities Version Control Settings.")
@RequestBody EntitiesVersionControlSettings settings) throws ThingsboardException {
try {
accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.READ);
settings = checkNotNull(settings);
versionControlService.checkVersionControlAccess(getTenantId(), settings);
} catch (Exception e) {
throw handleException(e);
}
}
@ApiOperation(value = "Check for new Platform Releases (checkUpdates)",
notes = "Check notifications about new platform releases. "
+ SYSTEM_AUTHORITY_PARAGRAPH)

View File

@ -15,25 +15,18 @@
*/
package org.thingsboard.server.controller;
import io.swagger.annotations.ApiModelProperty;
import io.swagger.annotations.ApiOperation;
import lombok.Data;
import lombok.RequiredArgsConstructor;
import org.apache.commons.lang3.StringUtils;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.bind.annotation.*;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.StringUtils;
import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.EntityIdFactory;
import org.thingsboard.server.service.security.model.SecurityUser;
import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService;
import org.thingsboard.server.service.sync.vc.data.EntitiesVersionControlSettings;
import org.thingsboard.server.service.sync.vc.data.EntityVersion;
import org.thingsboard.server.service.sync.vc.data.VersionCreationResult;
import org.thingsboard.server.service.sync.vc.data.VersionLoadResult;
@ -280,7 +273,7 @@ public class EntitiesVersionControlController extends BaseController {
List<String> remoteBranches = versionControlService.listBranches(getTenantId());
List<BranchInfo> infos = new ArrayList<>();
String defaultBranch = getSettings().getDefaultBranch();
String defaultBranch = versionControlService.getVersionControlSettings(getTenantId()).getDefaultBranch();
if (StringUtils.isNotEmpty(defaultBranch)) {
remoteBranches.remove(defaultBranch);
infos.add(new BranchInfo(defaultBranch, true));
@ -293,40 +286,6 @@ public class EntitiesVersionControlController extends BaseController {
}
}
@ApiOperation(value = "", notes = "" +
"```\n{\n" +
" \"repositoryUri\": \"https://github.com/User/repo.git\",\n" +
" \"username\": \"User\",\n" +
" \"password\": \"api_key\",\n" +
" \"defaultBranch\": \"master\"\n" +
"}\n```")
@GetMapping("/settings")
public EntitiesVersionControlSettings getSettings() throws ThingsboardException {
try {
return versionControlService.getSettings(getTenantId());
} catch (Exception e) {
throw handleException(e);
}
}
@ApiOperation(value = "", notes = "" +
"```\n{\n" +
" \"repositoryUri\": \"https://github.com/User/repo.git\",\n" +
" \"username\": \"User\",\n" +
" \"password\": \"api_key\",\n" +
" \"defaultBranch\": \"master\"\n" +
"}\n```")
@PostMapping("/settings")
public void saveSettings(@RequestBody EntitiesVersionControlSettings settings) throws ThingsboardException {
try {
versionControlService.saveSettings(getTenantId(), settings);
} catch (Exception e) {
throw handleException(e);
}
}
@Data
public static class BranchInfo {
private final String name;

View File

@ -219,6 +219,7 @@ public class ThingsboardInstallService {
databaseEntitiesUpgradeService.upgradeDatabase("3.3.3");
case "3.3.4":
log.info("Upgrading ThingsBoard from version 3.3.4 to 3.4.0 ...");
databaseEntitiesUpgradeService.upgradeDatabase("3.3.4");
log.info("Updating system data...");
systemDataLoaderService.updateSystemWidgets();
break;

View File

@ -83,7 +83,7 @@ public class AdminSettingsEdgeEventFetcher implements EdgeEventFetcher {
result.add(EdgeUtils.constructEdgeEvent(tenantId, edge.getId(), EdgeEventType.ADMIN_SETTINGS,
EdgeEventActionType.UPDATED, null, mapper.valueToTree(systemMailSettings)));
AdminSettings tenantMailSettings = convertToTenantAdminSettings(systemMailSettings.getKey(), (ObjectNode) systemMailSettings.getJsonValue());
AdminSettings tenantMailSettings = convertToTenantAdminSettings(tenantId, systemMailSettings.getKey(), (ObjectNode) systemMailSettings.getJsonValue());
result.add(EdgeUtils.constructEdgeEvent(tenantId, edge.getId(), EdgeEventType.ADMIN_SETTINGS,
EdgeEventActionType.UPDATED, null, mapper.valueToTree(tenantMailSettings)));
@ -91,7 +91,7 @@ public class AdminSettingsEdgeEventFetcher implements EdgeEventFetcher {
result.add(EdgeUtils.constructEdgeEvent(tenantId, edge.getId(), EdgeEventType.ADMIN_SETTINGS,
EdgeEventActionType.UPDATED, null, mapper.valueToTree(systemMailTemplates)));
AdminSettings tenantMailTemplates = convertToTenantAdminSettings(systemMailTemplates.getKey(), (ObjectNode) systemMailTemplates.getJsonValue());
AdminSettings tenantMailTemplates = convertToTenantAdminSettings(tenantId, systemMailTemplates.getKey(), (ObjectNode) systemMailTemplates.getJsonValue());
result.add(EdgeUtils.constructEdgeEvent(tenantId, edge.getId(), EdgeEventType.ADMIN_SETTINGS,
EdgeEventActionType.UPDATED, null, mapper.valueToTree(tenantMailTemplates)));
@ -151,8 +151,9 @@ public class AdminSettingsEdgeEventFetcher implements EdgeEventFetcher {
}
}
private AdminSettings convertToTenantAdminSettings(String key, ObjectNode jsonValue) {
private AdminSettings convertToTenantAdminSettings(TenantId tenantId, String key, ObjectNode jsonValue) {
AdminSettings tenantMailSettings = new AdminSettings();
tenantMailSettings.setTenantId(tenantId);
jsonValue.put("useSystemMailSettings", true);
tenantMailSettings.setJsonValue(jsonValue);
tenantMailSettings.setKey(key);

View File

@ -231,6 +231,7 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService {
@Override
public void createAdminSettings() throws Exception {
AdminSettings generalSettings = new AdminSettings();
generalSettings.setTenantId(TenantId.SYS_TENANT_ID);
generalSettings.setKey("general");
ObjectNode node = objectMapper.createObjectNode();
node.put("baseUrl", "http://localhost:8080");
@ -239,6 +240,7 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService {
adminSettingsService.saveAdminSettings(TenantId.SYS_TENANT_ID, generalSettings);
AdminSettings mailSettings = new AdminSettings();
mailSettings.setTenantId(TenantId.SYS_TENANT_ID);
mailSettings.setKey("mail");
node = objectMapper.createObjectNode();
node.put("mailFrom", "ThingsBoard <sysadmin@localhost.localdomain>");

View File

@ -534,6 +534,18 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService
log.error("Failed updating schema!!!", e);
}
break;
case "3.3.4":
try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) {
log.info("Updating schema ...");
schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "3.3.4", SCHEMA_UPDATE_SQL);
loadSql(schemaUpdateFile, conn);
log.info("Updating schema settings...");
conn.createStatement().execute("UPDATE tb_schema_settings SET schema_version = 3004000;");
log.info("Schema updated.");
} catch (Exception e) {
log.error("Failed updating schema!!!", e);
}
break;
default:
throw new RuntimeException("Unable to upgrade SQL database, unsupported fromVersion: " + fromVersion);
}

View File

@ -28,6 +28,7 @@ public class TenantAdminPermissions extends AbstractPermissions {
public TenantAdminPermissions() {
super();
put(Resource.ADMIN_SETTINGS, PermissionChecker.allowAllPermissionChecker);
put(Resource.ALARM, tenantEntityPermissionChecker);
put(Resource.ASSET, tenantEntityPermissionChecker);
put(Resource.DEVICE, tenantEntityPermissionChecker);

View File

@ -107,6 +107,7 @@ public class DefaultSystemSecurityService implements SystemSecurityService {
AdminSettings adminSettings = adminSettingsService.findAdminSettingsByKey(tenantId, "securitySettings");
if (adminSettings == null) {
adminSettings = new AdminSettings();
adminSettings.setTenantId(tenantId);
adminSettings.setKey("securitySettings");
}
adminSettings.setJsonValue(JacksonUtil.valueToTree(securitySettings));

View File

@ -19,34 +19,29 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectWriter;
import com.fasterxml.jackson.databind.SerializationFeature;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.springframework.transaction.support.TransactionTemplate;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.common.data.DataConstants;
import org.thingsboard.server.common.data.AdminSettings;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.ExportableEntity;
import org.thingsboard.server.common.data.StringUtils;
import org.thingsboard.server.common.data.exception.ThingsboardErrorCode;
import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.CustomerId;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.EntityIdFactory;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry;
import org.thingsboard.server.common.data.kv.JsonDataEntry;
import org.thingsboard.server.common.data.kv.KvEntry;
import org.thingsboard.server.common.data.query.EntityDataPageLink;
import org.thingsboard.server.common.data.query.EntityDataQuery;
import org.thingsboard.server.common.data.query.EntityDataSortOrder;
import org.thingsboard.server.common.data.query.EntityKey;
import org.thingsboard.server.common.data.query.EntityKeyType;
import org.thingsboard.server.common.data.query.EntityTypeFilter;
import org.thingsboard.server.common.data.query.*;
import org.thingsboard.server.common.data.vc.EntitiesVersionControlSettings;
import org.thingsboard.server.common.data.vc.VersionControlAuthMethod;
import org.thingsboard.server.dao.DaoUtil;
import org.thingsboard.server.dao.attributes.AttributesService;
import org.thingsboard.server.dao.entity.EntityService;
import org.thingsboard.server.dao.settings.AdminSettingsService;
import org.thingsboard.server.dao.tenant.TenantDao;
import org.thingsboard.server.queue.util.AfterStartUp;
import org.thingsboard.server.queue.util.TbCoreComponent;
@ -58,17 +53,11 @@ import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExp
import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportSettings;
import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportResult;
import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportSettings;
import org.thingsboard.server.service.sync.vc.data.EntitiesVersionControlSettings;
import org.thingsboard.server.service.sync.vc.data.EntityVersion;
import org.thingsboard.server.service.sync.vc.data.VersionCreationResult;
import org.thingsboard.server.service.sync.vc.data.VersionLoadResult;
import org.thingsboard.server.service.sync.vc.data.VersionedEntityInfo;
import org.thingsboard.server.service.sync.vc.data.request.create.EntityListVersionCreateRequest;
import org.thingsboard.server.service.sync.vc.data.request.create.ComplexVersionCreateRequest;
import org.thingsboard.server.service.sync.vc.data.request.create.SingleEntityVersionCreateRequest;
import org.thingsboard.server.service.sync.vc.data.request.create.SyncStrategy;
import org.thingsboard.server.service.sync.vc.data.request.create.VersionCreateConfig;
import org.thingsboard.server.service.sync.vc.data.request.create.VersionCreateRequest;
import org.thingsboard.server.service.sync.vc.data.request.create.*;
import org.thingsboard.server.service.sync.vc.data.request.load.EntityTypeVersionLoadRequest;
import org.thingsboard.server.service.sync.vc.data.request.load.SingleEntityVersionLoadRequest;
import org.thingsboard.server.service.sync.vc.data.request.load.VersionLoadConfig;
@ -81,13 +70,7 @@ import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
@ -104,7 +87,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
private final EntitiesExportImportService exportImportService;
private final ExportableEntitiesService exportableEntitiesService;
private final AttributesService attributesService;
private final AdminSettingsService adminSettingsService;
private final EntityService entityService;
private final TenantDao tenantDao;
private final TransactionTemplate transactionTemplate;
@ -114,14 +97,14 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
@Value("${java.io.tmpdir}/repositories")
private String repositoriesFolder;
private static final String SETTINGS_KEY = "vc";
private static final String SETTINGS_KEY = "entitiesVersionControl";
private final ObjectWriter jsonWriter = new ObjectMapper().writer(SerializationFeature.INDENT_OUTPUT);
@AfterStartUp
public void init() {
DaoUtil.processInBatches(tenantDao::findTenantsIds, 100, tenantId -> {
EntitiesVersionControlSettings settings = getSettings(tenantId);
EntitiesVersionControlSettings settings = getVersionControlSettings(tenantId);
if (settings != null) {
try {
initRepository(tenantId, settings);
@ -412,6 +395,73 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
return repository.listBranches();
}
@Override
public EntitiesVersionControlSettings getVersionControlSettings(TenantId tenantId) {
AdminSettings adminSettings = adminSettingsService.findAdminSettingsByKey(tenantId, SETTINGS_KEY);
if (adminSettings != null) {
try {
return JacksonUtil.convertValue(adminSettings.getJsonValue(), EntitiesVersionControlSettings.class);
} catch (Exception e) {
throw new RuntimeException("Failed to load version control settings!", e);
}
}
return null;
}
@Override
public EntitiesVersionControlSettings saveVersionControlSettings(TenantId tenantId, EntitiesVersionControlSettings versionControlSettings) {
EntitiesVersionControlSettings storedSettings = getVersionControlSettings(tenantId);
versionControlSettings = this.restoreCredentials(versionControlSettings, storedSettings);
AdminSettings adminSettings = new AdminSettings();
adminSettings.setTenantId(tenantId);
adminSettings.setKey(SETTINGS_KEY);
adminSettings.setJsonValue(JacksonUtil.valueToTree(versionControlSettings));
AdminSettings savedAdminSettings = adminSettingsService.saveAdminSettings(tenantId, adminSettings);
EntitiesVersionControlSettings savedVersionControlSettings;
try {
savedVersionControlSettings = JacksonUtil.convertValue(savedAdminSettings.getJsonValue(), EntitiesVersionControlSettings.class);
} catch (Exception e) {
throw new RuntimeException("Failed to load version control settings!", e);
}
try {
clearRepository(tenantId);
initRepository(tenantId, savedVersionControlSettings);
} catch (Exception e) {
throw new RuntimeException("Failed to init repository!", e);
}
return savedVersionControlSettings;
}
@Override
public void checkVersionControlAccess(TenantId tenantId, EntitiesVersionControlSettings settings) throws ThingsboardException {
EntitiesVersionControlSettings storedSettings = getVersionControlSettings(tenantId);
settings = this.restoreCredentials(settings, storedSettings);
Path repositoryDirectory = Path.of(repositoriesFolder, tenantId.getId().toString());
try {
GitRepository.test(settings, repositoryDirectory.toFile());
} catch (GitAPIException e) {
throw new ThingsboardException(String.format("Unable to access repository: %s", e.getMessage()),
ThingsboardErrorCode.GENERAL);
}
}
private EntitiesVersionControlSettings restoreCredentials(EntitiesVersionControlSettings settings, EntitiesVersionControlSettings storedSettings) {
VersionControlAuthMethod authMethod = settings.getAuthMethod();
if (VersionControlAuthMethod.USERNAME_PASSWORD.equals(authMethod) && settings.getPassword() == null) {
if (storedSettings != null) {
settings.setPassword(storedSettings.getPassword());
}
} else if (VersionControlAuthMethod.PRIVATE_KEY.equals(authMethod) && settings.getPrivateKey() == null) {
if (storedSettings != null) {
settings.setPrivateKey(storedSettings.getPrivateKey());
if (StringUtils.isEmpty(settings.getPrivateKeyPassword()) &&
StringUtils.isNotEmpty(storedSettings.getPrivateKeyPassword())) {
settings.setPrivateKeyPassword(storedSettings.getPrivateKeyPassword());
}
}
}
return settings;
}
private EntityVersion checkVersion(TenantId tenantId, String branch, String versionId) throws Exception {
return listVersions(tenantId, branch, null).stream()
@ -426,11 +476,12 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
private void initRepository(TenantId tenantId, EntitiesVersionControlSettings settings) throws Exception {
Path repositoryDirectory = Path.of(repositoriesFolder, tenantId.getId().toString());
GitRepository repository;
FileUtils.forceDelete(repositoryDirectory.toFile());
FileUtils.forceDelete(repositoryDirectory.toFile());
Files.createDirectories(repositoryDirectory);
repository = GitRepository.clone(settings.getRepositoryUri(), settings.getUsername(), settings.getPassword(), repositoryDirectory.toFile());
GitRepository repository = GitRepository.clone(settings, repositoryDirectory.toFile());
repositories.put(tenantId, repository);
}
@ -442,34 +493,6 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
}
}
@SneakyThrows
@Override
public void saveSettings(TenantId tenantId, EntitiesVersionControlSettings settings) {
attributesService.save(tenantId, tenantId, DataConstants.SERVER_SCOPE, List.of(
new BaseAttributeKvEntry(System.currentTimeMillis(), new JsonDataEntry(SETTINGS_KEY, JacksonUtil.toString(settings)))
)).get();
clearRepository(tenantId);
initRepository(tenantId, settings);
}
@SneakyThrows
@Override
public EntitiesVersionControlSettings getSettings(TenantId tenantId) {
return attributesService.find(tenantId, tenantId, DataConstants.SERVER_SCOPE, SETTINGS_KEY).get()
.flatMap(KvEntry::getJsonValue)
.map(json -> {
try {
return JacksonUtil.fromString(json, EntitiesVersionControlSettings.class);
} catch (IllegalArgumentException e) {
return null;
}
})
.orElse(null);
}
private EntityVersion toVersion(GitRepository.Commit commit) {
return new EntityVersion(commit.getId(), commit.getMessage());
}

View File

@ -16,16 +16,17 @@
package org.thingsboard.server.service.sync.vc;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.vc.EntitiesVersionControlSettings;
import org.thingsboard.server.service.security.model.SecurityUser;
import org.thingsboard.server.service.sync.vc.data.EntitiesVersionControlSettings;
import org.thingsboard.server.service.sync.vc.data.EntityVersion;
import org.thingsboard.server.service.sync.vc.data.VersionCreationResult;
import org.thingsboard.server.service.sync.vc.data.VersionLoadResult;
import org.thingsboard.server.service.sync.vc.data.VersionedEntityInfo;
import org.thingsboard.server.service.sync.vc.data.request.load.VersionLoadRequest;
import org.thingsboard.server.service.sync.vc.data.request.create.VersionCreateRequest;
import org.thingsboard.server.service.sync.vc.data.request.load.VersionLoadRequest;
import java.util.List;
@ -51,9 +52,12 @@ public interface EntitiesVersionControlService {
List<String> listBranches(TenantId tenantId) throws Exception;
EntitiesVersionControlSettings getVersionControlSettings(TenantId tenantId);
EntitiesVersionControlSettings saveVersionControlSettings(TenantId tenantId, EntitiesVersionControlSettings versionControlSettings);
void checkVersionControlAccess(TenantId tenantId, EntitiesVersionControlSettings settings) throws ThingsboardException;
void saveSettings(TenantId tenantId, EntitiesVersionControlSettings settings);
EntitiesVersionControlSettings getSettings(TenantId tenantId);
}

View File

@ -18,14 +18,8 @@ package org.thingsboard.server.utils;
import com.google.common.collect.Streams;
import lombok.Data;
import lombok.Getter;
import org.apache.commons.lang3.StringUtils;
import org.eclipse.jgit.api.Git;
import org.eclipse.jgit.api.GitCommand;
import org.eclipse.jgit.api.ListBranchCommand;
import org.eclipse.jgit.api.LogCommand;
import org.eclipse.jgit.api.RmCommand;
import org.eclipse.jgit.api.Status;
import org.eclipse.jgit.api.TransportCommand;
import org.apache.sshd.common.util.security.SecurityUtils;
import org.eclipse.jgit.api.*;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.eclipse.jgit.lib.Constants;
import org.eclipse.jgit.lib.ObjectId;
@ -34,50 +28,88 @@ import org.eclipse.jgit.lib.ObjectReader;
import org.eclipse.jgit.revwalk.RevCommit;
import org.eclipse.jgit.revwalk.filter.RevFilter;
import org.eclipse.jgit.transport.CredentialsProvider;
import org.eclipse.jgit.transport.SshTransport;
import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider;
import org.eclipse.jgit.transport.sshd.JGitKeyCache;
import org.eclipse.jgit.transport.sshd.ServerKeyDatabase;
import org.eclipse.jgit.transport.sshd.SshdSessionFactory;
import org.eclipse.jgit.transport.sshd.SshdSessionFactoryBuilder;
import org.eclipse.jgit.treewalk.TreeWalk;
import org.eclipse.jgit.treewalk.filter.PathFilter;
import org.thingsboard.server.common.data.StringUtils;
import org.thingsboard.server.common.data.vc.EntitiesVersionControlSettings;
import org.thingsboard.server.common.data.vc.VersionControlAuthMethod;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.nio.charset.StandardCharsets;
import java.security.KeyPair;
import java.security.PublicKey;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Set;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.stream.Collectors;
public class GitRepository {
private final Git git;
private final CredentialsProvider credentialsProvider;
private final SshdSessionFactory sshSessionFactory;
@Getter
private final String directory;
private GitRepository(Git git, CredentialsProvider credentialsProvider, String directory) {
private GitRepository(Git git, CredentialsProvider credentialsProvider, SshdSessionFactory sshSessionFactory, String directory) {
this.git = git;
this.credentialsProvider = credentialsProvider;
this.sshSessionFactory = sshSessionFactory;
this.directory = directory;
}
public static GitRepository clone(String uri, String username, String password, File directory) throws GitAPIException {
CredentialsProvider credentialsProvider = newCredentialsProvider(username, password);
Git git = Git.cloneRepository()
.setURI(uri)
public static GitRepository clone(EntitiesVersionControlSettings settings, File directory) throws GitAPIException {
CredentialsProvider credentialsProvider = null;
SshdSessionFactory sshSessionFactory = null;
if (VersionControlAuthMethod.USERNAME_PASSWORD.equals(settings.getAuthMethod())) {
credentialsProvider = newCredentialsProvider(settings.getUsername(), settings.getPassword());
} else if (VersionControlAuthMethod.PRIVATE_KEY.equals(settings.getAuthMethod())) {
sshSessionFactory = newSshdSessionFactory(settings.getPrivateKey(), settings.getPrivateKeyPassword(), directory);
}
CloneCommand cloneCommand = Git.cloneRepository()
.setURI(settings.getRepositoryUri())
.setDirectory(directory)
.setNoCheckout(true)
.setCredentialsProvider(credentialsProvider)
.call();
return new GitRepository(git, credentialsProvider, directory.getAbsolutePath());
.setNoCheckout(true);
configureTransportCommand(cloneCommand, credentialsProvider, sshSessionFactory);
Git git = cloneCommand.call();
return new GitRepository(git, credentialsProvider, sshSessionFactory, directory.getAbsolutePath());
}
public static GitRepository open(File directory, String username, String password) throws IOException {
public static GitRepository open(File directory, EntitiesVersionControlSettings settings) throws IOException {
Git git = Git.open(directory);
return new GitRepository(git, newCredentialsProvider(username, password), directory.getAbsolutePath());
CredentialsProvider credentialsProvider = null;
SshdSessionFactory sshSessionFactory = null;
if (VersionControlAuthMethod.USERNAME_PASSWORD.equals(settings.getAuthMethod())) {
credentialsProvider = newCredentialsProvider(settings.getUsername(), settings.getPassword());
} else if (VersionControlAuthMethod.PRIVATE_KEY.equals(settings.getAuthMethod())) {
sshSessionFactory = newSshdSessionFactory(settings.getPrivateKey(), settings.getPrivateKeyPassword(), directory);
}
return new GitRepository(git, credentialsProvider, sshSessionFactory, directory.getAbsolutePath());
}
public static void test(EntitiesVersionControlSettings settings, File directory) throws GitAPIException {
CredentialsProvider credentialsProvider = null;
SshdSessionFactory sshSessionFactory = null;
if (VersionControlAuthMethod.USERNAME_PASSWORD.equals(settings.getAuthMethod())) {
credentialsProvider = newCredentialsProvider(settings.getUsername(), settings.getPassword());
} else if (VersionControlAuthMethod.PRIVATE_KEY.equals(settings.getAuthMethod())) {
sshSessionFactory = newSshdSessionFactory(settings.getPrivateKey(), settings.getPrivateKeyPassword(), directory);
}
LsRemoteCommand lsRemoteCommand = Git.lsRemoteRepository().setRemote(settings.getRepositoryUri());
configureTransportCommand(lsRemoteCommand, credentialsProvider, sshSessionFactory);
lsRemoteCommand.call();
}
public void fetch() throws GitAPIException {
execute(git.fetch()
@ -108,7 +140,6 @@ public class GitRepository {
.distinct().collect(Collectors.toList());
}
public List<Commit> listCommits(String branch, int limit) throws IOException, GitAPIException {
return listCommits(branch, null, limit);
}
@ -246,16 +277,66 @@ public class GitRepository {
}
private <C extends GitCommand<T>, T> T execute(C command) throws GitAPIException {
if (command instanceof TransportCommand && credentialsProvider != null) {
((TransportCommand<?, ?>) command).setCredentialsProvider(credentialsProvider);
if (command instanceof TransportCommand) {
configureTransportCommand((TransportCommand) command, credentialsProvider, sshSessionFactory);
}
return command.call();
}
private static CredentialsProvider newCredentialsProvider(String username, String password) {
return new UsernamePasswordCredentialsProvider(username, password);
private static void configureTransportCommand(TransportCommand transportCommand, CredentialsProvider credentialsProvider, SshdSessionFactory sshSessionFactory) {
if (credentialsProvider != null) {
transportCommand.setCredentialsProvider(credentialsProvider);
}
if (sshSessionFactory != null) {
transportCommand.setTransportConfigCallback(transport -> {
if (transport instanceof SshTransport) {
SshTransport sshTransport = (SshTransport) transport;
sshTransport.setSshSessionFactory(sshSessionFactory);
}
});
}
}
private static CredentialsProvider newCredentialsProvider(String username, String password) {
return new UsernamePasswordCredentialsProvider(username, password == null ? "" : password);
}
private static SshdSessionFactory newSshdSessionFactory(String privateKey, String password, File directory) {
SshdSessionFactory sshSessionFactory = null;
if (StringUtils.isNotBlank(privateKey)) {
Iterable<KeyPair> keyPairs = loadKeyPairs(privateKey, password);
sshSessionFactory = new SshdSessionFactoryBuilder()
.setPreferredAuthentications("publickey")
.setDefaultKeysProvider(file -> keyPairs)
.setHomeDirectory(directory)
.setSshDirectory(directory)
.setServerKeyDatabase((file, file2) -> new ServerKeyDatabase() {
@Override
public List<PublicKey> lookup(String connectAddress, InetSocketAddress remoteAddress, Configuration config) {
return Collections.emptyList();
}
@Override
public boolean accept(String connectAddress, InetSocketAddress remoteAddress, PublicKey serverKey, Configuration config, CredentialsProvider provider) {
return true;
}
})
.build(new JGitKeyCache());
}
return sshSessionFactory;
}
private static Iterable<KeyPair> loadKeyPairs(String privateKeyContent, String password) {
Iterable<KeyPair> keyPairs = null;
try {
keyPairs = SecurityUtils.loadKeyPairIdentities(null,
null, new ByteArrayInputStream(privateKeyContent.getBytes()), (session, resourceKey, retryIndex) -> password);
} catch (Exception e) {}
if (keyPairs == null) {
throw new IllegalArgumentException("Failed to load ssh private key");
}
return keyPairs;
}
@Data
public static class Commit {

View File

@ -21,14 +21,17 @@ import org.thingsboard.server.common.data.id.AdminSettingsId;
import com.fasterxml.jackson.databind.JsonNode;
import org.thingsboard.server.common.data.id.DeviceId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.validation.Length;
import org.thingsboard.server.common.data.validation.NoXss;
@ApiModel
public class AdminSettings extends BaseData<AdminSettingsId> {
public class AdminSettings extends BaseData<AdminSettingsId> implements HasTenantId {
private static final long serialVersionUID = -7670322981725511892L;
private TenantId tenantId;
@NoXss
@Length(fieldName = "key")
private String key;
@ -44,6 +47,7 @@ public class AdminSettings extends BaseData<AdminSettingsId> {
public AdminSettings(AdminSettings adminSettings) {
super(adminSettings);
this.tenantId = adminSettings.getTenantId();
this.key = adminSettings.getKey();
this.jsonValue = adminSettings.getJsonValue();
}
@ -60,7 +64,16 @@ public class AdminSettings extends BaseData<AdminSettingsId> {
return super.getCreatedTime();
}
@ApiModelProperty(position = 3, value = "The Administration Settings key, (e.g. 'general' or 'mail')", example = "mail")
@ApiModelProperty(position = 3, value = "JSON object with Tenant Id.", readOnly = true)
public TenantId getTenantId() {
return tenantId;
}
public void setTenantId(TenantId tenantId) {
this.tenantId = tenantId;
}
@ApiModelProperty(position = 4, value = "The Administration Settings key, (e.g. 'general' or 'mail')", example = "mail")
public String getKey() {
return key;
}
@ -69,7 +82,7 @@ public class AdminSettings extends BaseData<AdminSettingsId> {
this.key = key;
}
@ApiModelProperty(position = 4, value = "JSON representation of the Administration Settings value")
@ApiModelProperty(position = 5, value = "JSON representation of the Administration Settings value")
public JsonNode getJsonValue() {
return jsonValue;
}

View File

@ -17,6 +17,10 @@ package org.thingsboard.server.common.data;
public class StringUtils {
public static final String EMPTY = "";
public static final int INDEX_NOT_FOUND = -1;
public static boolean isEmpty(String source) {
return source == null || source.isEmpty();
}
@ -32,4 +36,42 @@ public class StringUtils {
public static boolean isNotBlank(String source) {
return source != null && !source.isEmpty() && !source.trim().isEmpty();
}
public static String removeStart(final String str, final String remove) {
if (isEmpty(str) || isEmpty(remove)) {
return str;
}
if (str.startsWith(remove)){
return str.substring(remove.length());
}
return str;
}
public static String substringBefore(final String str, final String separator) {
if (isEmpty(str) || separator == null) {
return str;
}
if (separator.isEmpty()) {
return EMPTY;
}
final int pos = str.indexOf(separator);
if (pos == INDEX_NOT_FOUND) {
return str;
}
return str.substring(0, pos);
}
public static String substringBetween(final String str, final String open, final String close) {
if (str == null || open == null || close == null) {
return null;
}
final int start = str.indexOf(open);
if (start != INDEX_NOT_FOUND) {
final int end = str.indexOf(close, start + open.length());
if (end != INDEX_NOT_FOUND) {
return str.substring(start + open.length(), end);
}
}
return null;
}
}

View File

@ -13,14 +13,18 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.sync.vc.data;
package org.thingsboard.server.common.data.vc;
import lombok.Data;
@Data
public class EntitiesVersionControlSettings {
private String repositoryUri;
private VersionControlAuthMethod authMethod;
private String username;
private String password;
private String privateKeyFileName;
private String privateKey;
private String privateKeyPassword;
private String defaultBranch;
}

View File

@ -0,0 +1,21 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.data.vc;
public enum VersionControlAuthMethod {
USERNAME_PASSWORD,
PRIVATE_KEY
}

View File

@ -90,6 +90,8 @@ public class ModelConstants {
* Cassandra admin_settings constants.
*/
public static final String ADMIN_SETTINGS_COLUMN_FAMILY_NAME = "admin_settings";
public static final String ADMIN_SETTINGS_TENANT_ID_PROPERTY = TENANT_ID_PROPERTY;
public static final String ADMIN_SETTINGS_KEY_PROPERTY = "key";
public static final String ADMIN_SETTINGS_JSON_VALUE_PROPERTY = "json_value";

View File

@ -22,14 +22,18 @@ import org.hibernate.annotations.Type;
import org.hibernate.annotations.TypeDef;
import org.thingsboard.server.common.data.AdminSettings;
import org.thingsboard.server.common.data.id.AdminSettingsId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.dao.model.BaseEntity;
import org.thingsboard.server.dao.model.BaseSqlEntity;
import org.thingsboard.server.dao.model.ModelConstants;
import org.thingsboard.server.dao.util.mapping.JsonStringType;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.Table;
import java.util.UUID;
import static org.thingsboard.server.dao.model.ModelConstants.ADMIN_SETTINGS_COLUMN_FAMILY_NAME;
import static org.thingsboard.server.dao.model.ModelConstants.ADMIN_SETTINGS_JSON_VALUE_PROPERTY;
import static org.thingsboard.server.dao.model.ModelConstants.ADMIN_SETTINGS_KEY_PROPERTY;
@ -41,6 +45,9 @@ import static org.thingsboard.server.dao.model.ModelConstants.ADMIN_SETTINGS_KEY
@Table(name = ADMIN_SETTINGS_COLUMN_FAMILY_NAME)
public final class AdminSettingsEntity extends BaseSqlEntity<AdminSettings> implements BaseEntity<AdminSettings> {
@Column(name = ModelConstants.ADMIN_SETTINGS_TENANT_ID_PROPERTY)
private UUID tenantId;
@Column(name = ADMIN_SETTINGS_KEY_PROPERTY)
private String key;
@ -57,6 +64,7 @@ public final class AdminSettingsEntity extends BaseSqlEntity<AdminSettings> impl
this.setUuid(adminSettings.getId().getId());
}
this.setCreatedTime(adminSettings.getCreatedTime());
this.tenantId = adminSettings.getTenantId().getId();
this.key = adminSettings.getKey();
this.jsonValue = adminSettings.getJsonValue();
}
@ -65,6 +73,7 @@ public final class AdminSettingsEntity extends BaseSqlEntity<AdminSettings> impl
public AdminSettings toData() {
AdminSettings adminSettings = new AdminSettings(new AdminSettingsId(id));
adminSettings.setCreatedTime(createdTime);
adminSettings.setTenantId(TenantId.fromUUID(tenantId));
adminSettings.setKey(key);
adminSettings.setJsonValue(jsonValue);
return adminSettings;

View File

@ -19,6 +19,8 @@ import org.thingsboard.server.common.data.AdminSettings;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.dao.Dao;
import java.util.UUID;
public interface AdminSettingsDao extends Dao<AdminSettings> {
/**
@ -35,6 +37,6 @@ public interface AdminSettingsDao extends Dao<AdminSettings> {
* @param key the key
* @return the admin settings object
*/
AdminSettings findByKey(TenantId tenantId, String key);
AdminSettings findByTenantIdAndKey(UUID tenantId, String key);
}

View File

@ -22,6 +22,7 @@ import org.springframework.stereotype.Service;
import org.thingsboard.server.common.data.AdminSettings;
import org.thingsboard.server.common.data.id.AdminSettingsId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.vc.VersionControlAuthMethod;
import org.thingsboard.server.dao.service.DataValidator;
import org.thingsboard.server.dao.service.Validator;
@ -46,20 +47,35 @@ public class AdminSettingsServiceImpl implements AdminSettingsService {
public AdminSettings findAdminSettingsByKey(TenantId tenantId, String key) {
log.trace("Executing findAdminSettingsByKey [{}]", key);
Validator.validateString(key, "Incorrect key " + key);
return adminSettingsDao.findByKey(tenantId, key);
return adminSettingsDao.findByTenantIdAndKey(tenantId.getId(), key);
}
@Override
public AdminSettings saveAdminSettings(TenantId tenantId, AdminSettings adminSettings) {
log.trace("Executing saveAdminSettings [{}]", adminSettings);
adminSettingsValidator.validate(adminSettings, data -> tenantId);
if(adminSettings.getKey().equals("mail") && !adminSettings.getJsonValue().has("password")) {
if (adminSettings.getKey().equals("mail") && !adminSettings.getJsonValue().has("password")) {
AdminSettings mailSettings = findAdminSettingsByKey(tenantId, "mail");
if (mailSettings != null) {
((ObjectNode) adminSettings.getJsonValue()).put("password", mailSettings.getJsonValue().get("password").asText());
}
} else if (adminSettings.getKey().equals("entitiesVersionControl")) {
VersionControlAuthMethod authMethod = VersionControlAuthMethod.valueOf(adminSettings.getJsonValue().get("authMethod").asText());
if (VersionControlAuthMethod.USERNAME_PASSWORD.equals(authMethod) && !adminSettings.getJsonValue().has("password")) {
AdminSettings vcSettings = findAdminSettingsByKey(tenantId, "entitiesVersionControl");
if (vcSettings != null) {
((ObjectNode) adminSettings.getJsonValue()).put("password", vcSettings.getJsonValue().get("password").asText());
}
} else if (VersionControlAuthMethod.PRIVATE_KEY.equals(authMethod) && !adminSettings.getJsonValue().has("privateKey")) {
AdminSettings vcSettings = findAdminSettingsByKey(tenantId, "entitiesVersionControl");
if (vcSettings != null) {
((ObjectNode) adminSettings.getJsonValue()).put("privateKey", vcSettings.getJsonValue().get("privateKey").asText());
if (!adminSettings.getJsonValue().has("privateKeyPassword") && vcSettings.getJsonValue().has("privateKeyPassword")) {
((ObjectNode) adminSettings.getJsonValue()).put("privateKeyPassword", vcSettings.getJsonValue().get("privateKeyPassword").asText());
}
}
}
}
return adminSettingsDao.save(tenantId, adminSettings);
}

View File

@ -25,5 +25,6 @@ import java.util.UUID;
*/
public interface AdminSettingsRepository extends JpaRepository<AdminSettingsEntity, UUID> {
AdminSettingsEntity findByKey(String key);
AdminSettingsEntity findByTenantIdAndKey(UUID tenantId, String key);
}

View File

@ -46,7 +46,7 @@ public class JpaAdminSettingsDao extends JpaAbstractDao<AdminSettingsEntity, Adm
}
@Override
public AdminSettings findByKey(TenantId tenantId, String key) {
return DaoUtil.getData(adminSettingsRepository.findByKey(key));
public AdminSettings findByTenantIdAndKey(UUID tenantId, String key) {
return DaoUtil.getData(adminSettingsRepository.findByTenantIdAndKey(tenantId, key));
}
}

View File

@ -34,6 +34,7 @@ call insert_tb_schema_settings();
CREATE TABLE IF NOT EXISTS admin_settings (
id uuid NOT NULL CONSTRAINT admin_settings_pkey PRIMARY KEY,
tenant_id uuid NOT NULL,
created_time bigint NOT NULL,
json_value varchar,
key varchar(255)

View File

@ -1887,6 +1887,11 @@
<artifactId>org.eclipse.jgit</artifactId>
<version>${jgit.version}</version>
</dependency>
<dependency>
<groupId>org.eclipse.jgit</groupId>
<artifactId>org.eclipse.jgit.ssh.apache</artifactId>
<version>${jgit.version}</version>
</dependency>
</dependencies>
</dependencyManagement>