diff --git a/application/src/main/java/org/thingsboard/server/service/sync/DefaultGitSyncService.java b/application/src/main/java/org/thingsboard/server/service/sync/DefaultGitSyncService.java new file mode 100644 index 0000000000..3480a5c9c8 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/DefaultGitSyncService.java @@ -0,0 +1,164 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync; + +import jakarta.annotation.PreDestroy; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.common.data.sync.vc.RepositorySettings; +import org.thingsboard.server.service.sync.vc.GitRepository; +import org.thingsboard.server.service.sync.vc.GitRepository.FileType; +import org.thingsboard.server.service.sync.vc.GitRepository.RepoFile; + +import java.net.URI; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +@Service +@RequiredArgsConstructor +@Slf4j +public class DefaultGitSyncService implements GitSyncService { + + @Value("${vc.git.repositories-folder:${java.io.tmpdir}/repositories}") + private String repositoriesFolder; + + private final ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("git-sync")); + private final Map repositories = new ConcurrentHashMap<>(); + private final Map updateListeners = new ConcurrentHashMap<>(); + + @Override + public void registerSync(String key, String repoUri, String branch, long fetchFrequencyMs, Runnable onUpdate) { + RepositorySettings settings = new RepositorySettings(); + settings.setRepositoryUri(repoUri); + settings.setDefaultBranch(branch); + if (onUpdate != null) { + updateListeners.put(key, onUpdate); + } + + executor.execute(() -> { + initRepository(key, settings); + }); + + executor.scheduleWithFixedDelay(() -> { + GitRepository repository = repositories.get(key); + if (repository == null || Files.notExists(Path.of(repository.getDirectory()))) { + initRepository(key, settings); + return; + } + + try { + log.debug("Fetching {} repository", key); + repository.fetch(); + onUpdate(key); + } catch (Throwable e) { + log.error("Failed to fetch {} repository", key, e); + } + }, fetchFrequencyMs, fetchFrequencyMs, TimeUnit.MILLISECONDS); + } + + @Override + public List listFiles(String key, String path, int depth, FileType type) { + GitRepository repository = getRepository(key); + return repository.listFilesAtCommit(getBranchRef(repository), path, depth).stream() + .filter(file -> type == null || file.type() == type) + .toList(); + } + + + @Override + public String getFileContent(String key, String path) { + GitRepository repository = getRepository(key); + try { + return repository.getFileContentAtCommit(path, getBranchRef(repository)); + } catch (Exception e) { + log.warn("Failed to get file content for path {} for {} repository: {}", path, key, e.getMessage()); + return "{}"; + } + } + + @Override + public String getGithubRawContentUrl(String key, String path) { + if (path == null) { + return ""; + } + RepositorySettings settings = getRepository(key).getSettings(); + return StringUtils.removeEnd(settings.getRepositoryUri(), ".git") + "/blob/" + settings.getDefaultBranch() + "/" + path + "?raw=true"; + } + + private GitRepository getRepository(String key) { + GitRepository repository = repositories.get(key); + if (repository != null) { + if (Files.notExists(Path.of(repository.getDirectory()))) { + // reinitializing the repository because folder was deleted + initRepository(key, repository.getSettings()); + } + } + + repository = repositories.get(key); + if (repository == null) { + throw new IllegalStateException("{} repository is not initialized"); + } + return repository; + } + + private void initRepository(String key, RepositorySettings settings) { + try { + repositories.remove(key); + Path directory = getRepoDirectory(settings); + + GitRepository repository = GitRepository.openOrClone(directory, settings, true); + repositories.put(key, repository); + log.info("Initialized {} repository", key); + + onUpdate(key); + } catch (Throwable e) { + log.error("Failed to init {} repository for settings {}", key, settings, e); + } + } + + private void onUpdate(String key) { + Runnable listener = updateListeners.get(key); + if (listener != null) { + listener.run(); + } + } + + private Path getRepoDirectory(RepositorySettings settings) { + // using uri to define folder name in case repo url is changed + String name = URI.create(settings.getRepositoryUri()).getPath().replaceAll("[^a-zA-Z]", ""); + return Path.of(repositoriesFolder, name); + } + + private String getBranchRef(GitRepository repository) { + return "refs/remotes/origin/" + repository.getSettings().getDefaultBranch(); + } + + @PreDestroy + private void preDestroy() { + executor.shutdownNow(); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/GitSyncService.java b/application/src/main/java/org/thingsboard/server/service/sync/GitSyncService.java new file mode 100644 index 0000000000..d1a09757f7 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/GitSyncService.java @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync; + +import org.thingsboard.server.service.sync.vc.GitRepository.FileType; +import org.thingsboard.server.service.sync.vc.GitRepository.RepoFile; + +import java.util.List; + +public interface GitSyncService { + + void registerSync(String key, String repoUri, String branch, long fetchFrequencyMs, Runnable onUpdate); + + List listFiles(String key, String path, int depth, FileType type); + + String getFileContent(String key, String path); + + String getGithubRawContentUrl(String key, String path); + +} diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java index 0788143030..07edfedf21 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java @@ -283,22 +283,7 @@ public class DefaultGitRepositoryService implements GitRepositoryService { private GitRepository openOrCloneRepository(TenantId tenantId, RepositorySettings settings, boolean fetch) throws Exception { log.debug("[{}] Init tenant repository started.", tenantId); Path repositoryDirectory = Path.of(repositoriesFolder, settings.isLocalOnly() ? "local_" + settings.getRepositoryUri() : tenantId.getId().toString()); - - GitRepository repository; - if (Files.exists(repositoryDirectory)) { - repository = GitRepository.open(repositoryDirectory.toFile(), settings); - if (fetch) { - repository.fetch(); - } - } else { - Files.createDirectories(repositoryDirectory); - if (settings.isLocalOnly()) { - repository = GitRepository.create(settings, repositoryDirectory.toFile()); - } else { - repository = GitRepository.clone(settings, repositoryDirectory.toFile()); - } - } - + GitRepository repository = GitRepository.openOrClone(repositoryDirectory, settings, fetch); repositories.put(tenantId, repository); log.debug("[{}] Init tenant repository completed.", tenantId); return repository; diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java index aaa4c22289..015eab1492 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java @@ -21,6 +21,7 @@ import com.google.common.collect.Streams; import lombok.Data; import lombok.Getter; import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; @@ -75,6 +76,7 @@ import java.io.IOException; import java.net.InetSocketAddress; import java.nio.charset.StandardCharsets; import java.nio.file.Files; +import java.nio.file.Path; import java.security.KeyPair; import java.security.PublicKey; import java.util.ArrayList; @@ -136,6 +138,24 @@ public class GitRepository { return new GitRepository(git, settings, authHandler, directory.getAbsolutePath()); } + public static GitRepository openOrClone(Path directory, RepositorySettings settings, boolean fetch) throws IOException, GitAPIException { + GitRepository repository; + if (Files.exists(directory)) { + repository = GitRepository.open(directory.toFile(), settings); + if (fetch) { + repository.fetch(); + } + } else { + Files.createDirectories(directory); + if (settings.isLocalOnly()) { + repository = GitRepository.create(settings, directory.toFile()); + } else { + repository = GitRepository.clone(settings, directory.toFile()); + } + } + return repository; + } + public static void test(RepositorySettings settings, File directory) throws Exception { if (settings.isLocalOnly()) { return; @@ -233,22 +253,29 @@ public class GitRepository { return iterableToPageData(commits, this::toCommit, pageLink, revCommitComparatorFunction); } - public List listFilesAtCommit(String commitId) throws IOException { - return listFilesAtCommit(commitId, null); + public List listFilesAtCommit(String commitId, String path) { + return listFilesAtCommit(commitId, path, -1).stream().map(RepoFile::path).toList(); } - public List listFilesAtCommit(String commitId, String path) throws IOException { + @SneakyThrows + public List listFilesAtCommit(String commitId, String path, int depth) { log.debug("Executing listFilesAtCommit [{}][{}][{}]", settings.getRepositoryUri(), commitId, path); - List files = new ArrayList<>(); + List files = new ArrayList<>(); RevCommit revCommit = resolveCommit(commitId); try (TreeWalk treeWalk = new TreeWalk(git.getRepository())) { treeWalk.reset(revCommit.getTree().getId()); if (StringUtils.isNotEmpty(path)) { treeWalk.setFilter(PathFilter.create(path)); } - treeWalk.setRecursive(true); + boolean fixedDepth = depth != -1; + treeWalk.setRecursive(!fixedDepth); while (treeWalk.next()) { - files.add(treeWalk.getPathString()); + if (!fixedDepth || treeWalk.getDepth() == depth) { + files.add(new RepoFile(treeWalk.getPathString(), treeWalk.getNameString(), treeWalk.isSubtree() ? FileType.DIRECTORY : FileType.FILE)); + } + if (fixedDepth && treeWalk.getDepth() < depth) { + treeWalk.enterSubtree(); + } } } return files; @@ -584,4 +611,10 @@ public class GitRepository { private String diffStringValue; } + public record RepoFile(String path, String name, FileType type) {} + + public enum FileType { + FILE, DIRECTORY + } + }