Merge pull request #9362 from dashevchenko/github_action_yml_check

Added github action to check all yml parameter description exist

.github/workflows/main.yml

@@ -0,0 +1,40 @@
+#
+# Copyright © 2016-2023 The Thingsboard Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+name: check_yml_file
+on:
+  pull_request:
+    paths:
+      - 'application/src/main/resources/thingsboard.yml'
+jobs:
+  build:
+    name: check_yml_file
+    runs-on: ubuntu-20.04
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v3
+        with:
+          python-version: "3.10.2"
+          architecture: "x64"
+        env:
+          AGENT_TOOLSDIRECTORY: /opt/hostedtoolcache
+
+      - name: Run Verification Script
+        run: python3 tools/src/main/python/check_yml_file.py

pull_request_template.md

@@ -26,6 +26,7 @@ Put your PR description here instead of this sentence.
 - [ ] If new dependency was added: the dependency tree is checked for conflicts.
 - [ ] If new service was added: the service is marked with corresponding @TbCoreComponent, @TbRuleEngineComponent, @TbTransportComponent, etc.
 - [ ] If new REST API was added: the RestClient.java was updated, issue for [Python REST client](https://github.com/thingsboard/thingsboard-python-rest-client) is created.
+- [ ] If new yml property was added: make sure a description is added (above or near the property).
 
 
 

tools/src/main/python/check_yml_file.py

@@ -0,0 +1,122 @@
+#
+# Copyright © 2016-2023 The Thingsboard Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import sys
+import re
+
+def extract_properties_with_comments(yaml_file_path):
+    properties = {}
+
+    with open(yaml_file_path, 'r') as file:
+        lines = file.readlines()
+        index = 0
+        key_level_map = {0 : ''}
+        parse_line('', '', key_level_map, 0, index, lines, properties)
+
+    return properties
+
+
+def parse_line(table_name, comment, key_level_map, parent_line_level, index, lines, properties):
+    if index >= len(lines):
+        return
+    line = lines[index]
+    line_level = (len(line) - len(line.lstrip())) if line.strip() else 0
+    line = line.strip()
+    # if line is empty - parse next line
+    if not line:
+        index = index + 1
+        parse_line(table_name, comment, key_level_map, line_level, index, lines, properties)
+    # if line is a comment - save comment and parse next line
+    else:
+        if line_level == 0:
+            key_level_map = {0 : ''}
+        if line.startswith('#'):
+            if line_level == 0:
+                table_name = line.lstrip('#')
+            elif line_level == parent_line_level:
+                comment = comment + '\n' + line.lstrip('#')
+            else:
+                comment = line.lstrip('#')
+            index = index + 1
+            parse_line(table_name, comment, key_level_map, line_level, index, lines, properties)
+        else:
+            # Check if it's a property line
+            if ':' in line:
+                # clean comment if level was changed
+                if line_level != parent_line_level:
+                    comment = ''
+                key, value = line.split(':', 1)
+                if key.startswith('- '):
+                    key = key.lstrip('- ')
+                key_level_map[line_level] = key
+                value = value.strip()
+                if value.split('#')[0]:
+                    current_key = ''
+                    for k in key_level_map.keys():
+                        if k <= line_level:
+                            current_key = ((current_key + '.') if current_key else '') + key_level_map[k]
+                    properties[current_key] = (value, comment, table_name)
+                    comment = ''
+                index = index + 1
+                parse_line(table_name, comment, key_level_map, line_level, index, lines, properties)
+
+def extract_property_info(properties):
+    rows = []
+    for property_name, value in properties.items():
+        if '#' in value[0]:
+            value_parts = value[0].split('#')
+            comment = value_parts[1]
+        else:
+            comment = value[1]
+        pattern = r'\"\$\{(.*?)\:(.*?)\}\"'
+        match = re.match(pattern, value[0])
+        if match is not None:
+            rows.append((property_name, match.group(1), match.group(2), comment, value[2]))
+        else:
+            rows.append((property_name, "", value[0].split('#')[0], comment, value[2]))
+    return rows
+
+def check_descriptions(properties):
+    variables_without_description = []
+    for row in properties:
+        # Extract information from the tuple
+        property_name, env_variable, default_value, comment, table_name = row
+        if comment == '' or len(comment) < 5 :
+            variables_without_description.append(property_name)
+
+    return variables_without_description
+
+
+if __name__ == '__main__':
+    sys. setrecursionlimit(10000)
+    # path to the YAML file
+    input_yaml_file = "application/src/main/resources/thingsboard.yml"
+
+    # Parse yml file to map where key is property key path with '.' separator
+    # and value is an object (env_name_with_default_value, comment, table_name)
+    properties = extract_properties_with_comments(input_yaml_file)
+
+    # Extract property information (extract env name, default value and comment nearby property)
+    property_info = extract_property_info(properties)
+
+    # Check all properies have descriptions
+    variables_without_desc = check_descriptions(property_info)
+
+    if len(variables_without_desc) > 0:
+        print(f"There are some yml properties without valid description: (total {len(variables_without_desc)}) {variables_without_desc}.")
+        exit(1)
+    else:
+        print("All yml properties have valid description.")