Introduce Git sync service
This commit is contained in:
ViacheslavKlimov
parent
c88a05ef5c
commit
7248279a88
@ -0,0 +1,164 @@
|
||||
/**
|
||||
* Copyright © 2016-2024 The Thingsboard Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.thingsboard.server.service.sync;
|
||||
|
||||
import jakarta.annotation.PreDestroy;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.thingsboard.common.util.ThingsBoardThreadFactory;
|
||||
import org.thingsboard.server.common.data.sync.vc.RepositorySettings;
|
||||
import org.thingsboard.server.service.sync.vc.GitRepository;
|
||||
import org.thingsboard.server.service.sync.vc.GitRepository.FileType;
|
||||
import org.thingsboard.server.service.sync.vc.GitRepository.RepoFile;
|
||||
|
||||
import java.net.URI;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.ScheduledExecutorService;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@Slf4j
|
||||
public class DefaultGitSyncService implements GitSyncService {
|
||||
|
||||
@Value("${vc.git.repositories-folder:${java.io.tmpdir}/repositories}")
|
||||
private String repositoriesFolder;
|
||||
|
||||
private final ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("git-sync"));
|
||||
private final Map<String, GitRepository> repositories = new ConcurrentHashMap<>();
|
||||
private final Map<String, Runnable> updateListeners = new ConcurrentHashMap<>();
|
||||
|
||||
@Override
|
||||
public void registerSync(String key, String repoUri, String branch, long fetchFrequencyMs, Runnable onUpdate) {
|
||||
RepositorySettings settings = new RepositorySettings();
|
||||
settings.setRepositoryUri(repoUri);
|
||||
settings.setDefaultBranch(branch);
|
||||
if (onUpdate != null) {
|
||||
updateListeners.put(key, onUpdate);
|
||||
}
|
||||
|
||||
executor.execute(() -> {
|
||||
initRepository(key, settings);
|
||||
});
|
||||
|
||||
executor.scheduleWithFixedDelay(() -> {
|
||||
GitRepository repository = repositories.get(key);
|
||||
if (repository == null || Files.notExists(Path.of(repository.getDirectory()))) {
|
||||
initRepository(key, settings);
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
log.debug("Fetching {} repository", key);
|
||||
repository.fetch();
|
||||
onUpdate(key);
|
||||
} catch (Throwable e) {
|
||||
log.error("Failed to fetch {} repository", key, e);
|
||||
}
|
||||
}, fetchFrequencyMs, fetchFrequencyMs, TimeUnit.MILLISECONDS);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<RepoFile> listFiles(String key, String path, int depth, FileType type) {
|
||||
GitRepository repository = getRepository(key);
|
||||
return repository.listFilesAtCommit(getBranchRef(repository), path, depth).stream()
|
||||
.filter(file -> type == null || file.type() == type)
|
||||
.toList();
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public String getFileContent(String key, String path) {
|
||||
GitRepository repository = getRepository(key);
|
||||
try {
|
||||
return repository.getFileContentAtCommit(path, getBranchRef(repository));
|
||||
} catch (Exception e) {
|
||||
log.warn("Failed to get file content for path {} for {} repository: {}", path, key, e.getMessage());
|
||||
return "{}";
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getGithubRawContentUrl(String key, String path) {
|
||||
if (path == null) {
|
||||
return "";
|
||||
}
|
||||
RepositorySettings settings = getRepository(key).getSettings();
|
||||
return StringUtils.removeEnd(settings.getRepositoryUri(), ".git") + "/blob/" + settings.getDefaultBranch() + "/" + path + "?raw=true";
|
||||
}
|
||||
|
||||
private GitRepository getRepository(String key) {
|
||||
GitRepository repository = repositories.get(key);
|
||||
if (repository != null) {
|
||||
if (Files.notExists(Path.of(repository.getDirectory()))) {
|
||||
// reinitializing the repository because folder was deleted
|
||||
initRepository(key, repository.getSettings());
|
||||
}
|
||||
}
|
||||
|
||||
repository = repositories.get(key);
|
||||
if (repository == null) {
|
||||
throw new IllegalStateException("{} repository is not initialized");
|
||||
}
|
||||
return repository;
|
||||
}
|
||||
|
||||
private void initRepository(String key, RepositorySettings settings) {
|
||||
try {
|
||||
repositories.remove(key);
|
||||
Path directory = getRepoDirectory(settings);
|
||||
|
||||
GitRepository repository = GitRepository.openOrClone(directory, settings, true);
|
||||
repositories.put(key, repository);
|
||||
log.info("Initialized {} repository", key);
|
||||
|
||||
onUpdate(key);
|
||||
} catch (Throwable e) {
|
||||
log.error("Failed to init {} repository for settings {}", key, settings, e);
|
||||
}
|
||||
}
|
||||
|
||||
private void onUpdate(String key) {
|
||||
Runnable listener = updateListeners.get(key);
|
||||
if (listener != null) {
|
||||
listener.run();
|
||||
}
|
||||
}
|
||||
|
||||
private Path getRepoDirectory(RepositorySettings settings) {
|
||||
// using uri to define folder name in case repo url is changed
|
||||
String name = URI.create(settings.getRepositoryUri()).getPath().replaceAll("[^a-zA-Z]", "");
|
||||
return Path.of(repositoriesFolder, name);
|
||||
}
|
||||
|
||||
private String getBranchRef(GitRepository repository) {
|
||||
return "refs/remotes/origin/" + repository.getSettings().getDefaultBranch();
|
||||
}
|
||||
|
||||
@PreDestroy
|
||||
private void preDestroy() {
|
||||
executor.shutdownNow();
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,33 @@
|
||||
/**
|
||||
* Copyright © 2016-2024 The Thingsboard Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.thingsboard.server.service.sync;
|
||||
|
||||
import org.thingsboard.server.service.sync.vc.GitRepository.FileType;
|
||||
import org.thingsboard.server.service.sync.vc.GitRepository.RepoFile;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
public interface GitSyncService {
|
||||
|
||||
void registerSync(String key, String repoUri, String branch, long fetchFrequencyMs, Runnable onUpdate);
|
||||
|
||||
List<RepoFile> listFiles(String key, String path, int depth, FileType type);
|
||||
|
||||
String getFileContent(String key, String path);
|
||||
|
||||
String getGithubRawContentUrl(String key, String path);
|
||||
|
||||
}
|
||||
@ -283,22 +283,7 @@ public class DefaultGitRepositoryService implements GitRepositoryService {
|
||||
private GitRepository openOrCloneRepository(TenantId tenantId, RepositorySettings settings, boolean fetch) throws Exception {
|
||||
log.debug("[{}] Init tenant repository started.", tenantId);
|
||||
Path repositoryDirectory = Path.of(repositoriesFolder, settings.isLocalOnly() ? "local_" + settings.getRepositoryUri() : tenantId.getId().toString());
|
||||
|
||||
GitRepository repository;
|
||||
if (Files.exists(repositoryDirectory)) {
|
||||
repository = GitRepository.open(repositoryDirectory.toFile(), settings);
|
||||
if (fetch) {
|
||||
repository.fetch();
|
||||
}
|
||||
} else {
|
||||
Files.createDirectories(repositoryDirectory);
|
||||
if (settings.isLocalOnly()) {
|
||||
repository = GitRepository.create(settings, repositoryDirectory.toFile());
|
||||
} else {
|
||||
repository = GitRepository.clone(settings, repositoryDirectory.toFile());
|
||||
}
|
||||
}
|
||||
|
||||
GitRepository repository = GitRepository.openOrClone(repositoryDirectory, settings, fetch);
|
||||
repositories.put(tenantId, repository);
|
||||
log.debug("[{}] Init tenant repository completed.", tenantId);
|
||||
return repository;
|
||||
|
||||
@ -21,6 +21,7 @@ import com.google.common.collect.Streams;
|
||||
import lombok.Data;
|
||||
import lombok.Getter;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.SneakyThrows;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
@ -75,6 +76,7 @@ import java.io.IOException;
|
||||
import java.net.InetSocketAddress;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.security.KeyPair;
|
||||
import java.security.PublicKey;
|
||||
import java.util.ArrayList;
|
||||
@ -136,6 +138,24 @@ public class GitRepository {
|
||||
return new GitRepository(git, settings, authHandler, directory.getAbsolutePath());
|
||||
}
|
||||
|
||||
public static GitRepository openOrClone(Path directory, RepositorySettings settings, boolean fetch) throws IOException, GitAPIException {
|
||||
GitRepository repository;
|
||||
if (Files.exists(directory)) {
|
||||
repository = GitRepository.open(directory.toFile(), settings);
|
||||
if (fetch) {
|
||||
repository.fetch();
|
||||
}
|
||||
} else {
|
||||
Files.createDirectories(directory);
|
||||
if (settings.isLocalOnly()) {
|
||||
repository = GitRepository.create(settings, directory.toFile());
|
||||
} else {
|
||||
repository = GitRepository.clone(settings, directory.toFile());
|
||||
}
|
||||
}
|
||||
return repository;
|
||||
}
|
||||
|
||||
public static void test(RepositorySettings settings, File directory) throws Exception {
|
||||
if (settings.isLocalOnly()) {
|
||||
return;
|
||||
@ -233,22 +253,29 @@ public class GitRepository {
|
||||
return iterableToPageData(commits, this::toCommit, pageLink, revCommitComparatorFunction);
|
||||
}
|
||||
|
||||
public List<String> listFilesAtCommit(String commitId) throws IOException {
|
||||
return listFilesAtCommit(commitId, null);
|
||||
public List<String> listFilesAtCommit(String commitId, String path) {
|
||||
return listFilesAtCommit(commitId, path, -1).stream().map(RepoFile::path).toList();
|
||||
}
|
||||
|
||||
public List<String> listFilesAtCommit(String commitId, String path) throws IOException {
|
||||
@SneakyThrows
|
||||
public List<RepoFile> listFilesAtCommit(String commitId, String path, int depth) {
|
||||
log.debug("Executing listFilesAtCommit [{}][{}][{}]", settings.getRepositoryUri(), commitId, path);
|
||||
List<String> files = new ArrayList<>();
|
||||
List<RepoFile> files = new ArrayList<>();
|
||||
RevCommit revCommit = resolveCommit(commitId);
|
||||
try (TreeWalk treeWalk = new TreeWalk(git.getRepository())) {
|
||||
treeWalk.reset(revCommit.getTree().getId());
|
||||
if (StringUtils.isNotEmpty(path)) {
|
||||
treeWalk.setFilter(PathFilter.create(path));
|
||||
}
|
||||
treeWalk.setRecursive(true);
|
||||
boolean fixedDepth = depth != -1;
|
||||
treeWalk.setRecursive(!fixedDepth);
|
||||
while (treeWalk.next()) {
|
||||
files.add(treeWalk.getPathString());
|
||||
if (!fixedDepth || treeWalk.getDepth() == depth) {
|
||||
files.add(new RepoFile(treeWalk.getPathString(), treeWalk.getNameString(), treeWalk.isSubtree() ? FileType.DIRECTORY : FileType.FILE));
|
||||
}
|
||||
if (fixedDepth && treeWalk.getDepth() < depth) {
|
||||
treeWalk.enterSubtree();
|
||||
}
|
||||
}
|
||||
}
|
||||
return files;
|
||||
@ -584,4 +611,10 @@ public class GitRepository {
|
||||
private String diffStringValue;
|
||||
}
|
||||
|
||||
public record RepoFile(String path, String name, FileType type) {}
|
||||
|
||||
public enum FileType {
|
||||
FILE, DIRECTORY
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user