dendi-wang/thingsboard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Introduce Git sync service

Browse Source
This commit is contained in:
ViacheslavKlimov 2024-10-09 14:43:31 +03:00
parent c88a05ef5c
commit 7248279a88
4 changed files with 237 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

164
application/src/main/java/org/thingsboard/server/service/sync/DefaultGitSyncService.java Normal file
View File

@ -0,0 +1,164 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.sync;
import jakarta.annotation.PreDestroy;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.thingsboard.common.util.ThingsBoardThreadFactory;
import org.thingsboard.server.common.data.sync.vc.RepositorySettings;
import org.thingsboard.server.service.sync.vc.GitRepository;
import org.thingsboard.server.service.sync.vc.GitRepository.FileType;
import org.thingsboard.server.service.sync.vc.GitRepository.RepoFile;
import java.net.URI;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
@Service
@RequiredArgsConstructor
@Slf4j
public class DefaultGitSyncService implements GitSyncService {
@Value("${vc.git.repositories-folder:${java.io.tmpdir}/repositories}")
private String repositoriesFolder;
private final ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("git-sync"));
private final Map<String, GitRepository> repositories = new ConcurrentHashMap<>();
private final Map<String, Runnable> updateListeners = new ConcurrentHashMap<>();
@Override
public void registerSync(String key, String repoUri, String branch, long fetchFrequencyMs, Runnable onUpdate) {
RepositorySettings settings = new RepositorySettings();
settings.setRepositoryUri(repoUri);
settings.setDefaultBranch(branch);
if (onUpdate != null) {
updateListeners.put(key, onUpdate);
}
executor.execute(() -> {
initRepository(key, settings);
});
executor.scheduleWithFixedDelay(() -> {
GitRepository repository = repositories.get(key);
if (repository == null || Files.notExists(Path.of(repository.getDirectory()))) {
initRepository(key, settings);
return;
}
try {
log.debug("Fetching {} repository", key);
repository.fetch();
onUpdate(key);
} catch (Throwable e) {
log.error("Failed to fetch {} repository", key, e);
}
}, fetchFrequencyMs, fetchFrequencyMs, TimeUnit.MILLISECONDS);
}
@Override
public List<RepoFile> listFiles(String key, String path, int depth, FileType type) {
GitRepository repository = getRepository(key);
return repository.listFilesAtCommit(getBranchRef(repository), path, depth).stream()
.filter(file -> type == null || file.type() == type)
.toList();
}
@Override
public String getFileContent(String key, String path) {
GitRepository repository = getRepository(key);
try {
return repository.getFileContentAtCommit(path, getBranchRef(repository));
} catch (Exception e) {
log.warn("Failed to get file content for path {} for {} repository: {}", path, key, e.getMessage());
return "{}";
}
}
@Override
public String getGithubRawContentUrl(String key, String path) {
if (path == null) {
return "";
}
RepositorySettings settings = getRepository(key).getSettings();
return StringUtils.removeEnd(settings.getRepositoryUri(), ".git") + "/blob/" + settings.getDefaultBranch() + "/" + path + "?raw=true";
}
private GitRepository getRepository(String key) {
GitRepository repository = repositories.get(key);
if (repository != null) {
if (Files.notExists(Path.of(repository.getDirectory()))) {
// reinitializing the repository because folder was deleted
initRepository(key, repository.getSettings());
}
}
repository = repositories.get(key);
if (repository == null) {
throw new IllegalStateException("{} repository is not initialized");
}
return repository;
}
private void initRepository(String key, RepositorySettings settings) {
try {
repositories.remove(key);
Path directory = getRepoDirectory(settings);
GitRepository repository = GitRepository.openOrClone(directory, settings, true);
repositories.put(key, repository);
log.info("Initialized {} repository", key);
onUpdate(key);
} catch (Throwable e) {
log.error("Failed to init {} repository for settings {}", key, settings, e);
}
}
private void onUpdate(String key) {
Runnable listener = updateListeners.get(key);
if (listener != null) {
listener.run();
}
}
private Path getRepoDirectory(RepositorySettings settings) {
// using uri to define folder name in case repo url is changed
String name = URI.create(settings.getRepositoryUri()).getPath().replaceAll("[^a-zA-Z]", "");
return Path.of(repositoriesFolder, name);
}
private String getBranchRef(GitRepository repository) {
return "refs/remotes/origin/" + repository.getSettings().getDefaultBranch();
}
@PreDestroy
private void preDestroy() {
executor.shutdownNow();
}
}

33
application/src/main/java/org/thingsboard/server/service/sync/GitSyncService.java Normal file
View File

@ -0,0 +1,33 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.sync;
import org.thingsboard.server.service.sync.vc.GitRepository.FileType;
import org.thingsboard.server.service.sync.vc.GitRepository.RepoFile;
import java.util.List;
public interface GitSyncService {
void registerSync(String key, String repoUri, String branch, long fetchFrequencyMs, Runnable onUpdate);
List<RepoFile> listFiles(String key, String path, int depth, FileType type);
String getFileContent(String key, String path);
String getGithubRawContentUrl(String key, String path);
}

17
common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java
View File

@ -283,22 +283,7 @@ public class DefaultGitRepositoryService implements GitRepositoryService {
private GitRepository openOrCloneRepository(TenantId tenantId, RepositorySettings settings, boolean fetch) throws Exception {
log.debug("[{}] Init tenant repository started.", tenantId);
Path repositoryDirectory = Path.of(repositoriesFolder, settings.isLocalOnly() ? "local_" + settings.getRepositoryUri() : tenantId.getId().toString());
GitRepository repository;
if (Files.exists(repositoryDirectory)) {
repository = GitRepository.open(repositoryDirectory.toFile(), settings);
if (fetch) {
repository.fetch();
}
} else {
Files.createDirectories(repositoryDirectory);
if (settings.isLocalOnly()) {
repository = GitRepository.create(settings, repositoryDirectory.toFile());
} else {
repository = GitRepository.clone(settings, repositoryDirectory.toFile());
}
}
GitRepository repository = GitRepository.openOrClone(repositoryDirectory, settings, fetch);
repositories.put(tenantId, repository);
log.debug("[{}] Init tenant repository completed.", tenantId);
return repository;

45
common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java
View File

@ -21,6 +21,7 @@ import com.google.common.collect.Streams;
import lombok.Data;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
@ -75,6 +76,7 @@ import java.io.IOException;
import java.net.InetSocketAddress;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.security.KeyPair;
import java.security.PublicKey;
import java.util.ArrayList;
@ -136,6 +138,24 @@ public class GitRepository {
return new GitRepository(git, settings, authHandler, directory.getAbsolutePath());
}
public static GitRepository openOrClone(Path directory, RepositorySettings settings, boolean fetch) throws IOException, GitAPIException {
GitRepository repository;
if (Files.exists(directory)) {
repository = GitRepository.open(directory.toFile(), settings);
if (fetch) {
repository.fetch();
}
} else {
Files.createDirectories(directory);
if (settings.isLocalOnly()) {
repository = GitRepository.create(settings, directory.toFile());
} else {
repository = GitRepository.clone(settings, directory.toFile());
}
}
return repository;
}
public static void test(RepositorySettings settings, File directory) throws Exception {
if (settings.isLocalOnly()) {
return;
@ -233,22 +253,29 @@ public class GitRepository {
return iterableToPageData(commits, this::toCommit, pageLink, revCommitComparatorFunction);
}
public List<String> listFilesAtCommit(String commitId) throws IOException {
return listFilesAtCommit(commitId, null);
public List<String> listFilesAtCommit(String commitId, String path) {
return listFilesAtCommit(commitId, path, -1).stream().map(RepoFile::path).toList();
}
public List<String> listFilesAtCommit(String commitId, String path) throws IOException {
@SneakyThrows
public List<RepoFile> listFilesAtCommit(String commitId, String path, int depth) {
log.debug("Executing listFilesAtCommit [{}][{}][{}]", settings.getRepositoryUri(), commitId, path);
List<String> files = new ArrayList<>();
List<RepoFile> files = new ArrayList<>();
RevCommit revCommit = resolveCommit(commitId);
try (TreeWalk treeWalk = new TreeWalk(git.getRepository())) {
treeWalk.reset(revCommit.getTree().getId());
if (StringUtils.isNotEmpty(path)) {
treeWalk.setFilter(PathFilter.create(path));
}
treeWalk.setRecursive(true);
boolean fixedDepth = depth != -1;
treeWalk.setRecursive(!fixedDepth);
while (treeWalk.next()) {
files.add(treeWalk.getPathString());
if (!fixedDepth || treeWalk.getDepth() == depth) {
files.add(new RepoFile(treeWalk.getPathString(), treeWalk.getNameString(), treeWalk.isSubtree() ? FileType.DIRECTORY : FileType.FILE));
}
if (fixedDepth && treeWalk.getDepth() < depth) {
treeWalk.enterSubtree();
}
}
}
return files;
@ -584,4 +611,10 @@ public class GitRepository {
private String diffStringValue;
}
public record RepoFile(String path, String name, FileType type) {}
public enum FileType {
FILE, DIRECTORY
}
}