From f38f2d5b6a878f0e3751afa7d7b75f7af13c9ef7 Mon Sep 17 00:00:00 2001 From: Alexander Wilms Date: Fri, 15 Sep 2023 20:55:48 +0000 Subject: [PATCH 1/4] Add JSON validation as a CI stage --- .github/validate_json.py | 25 +++++++++++++++++++++++++ .github/workflows/github.yml | 5 +++++ 2 files changed, 30 insertions(+) create mode 100644 .github/validate_json.py diff --git a/.github/validate_json.py b/.github/validate_json.py new file mode 100644 index 000000000..0c95557bc --- /dev/null +++ b/.github/validate_json.py @@ -0,0 +1,25 @@ +import jstyleson +from pathlib import Path +from pprint import pprint + +errors = [] +for path in sorted(Path('.').glob('**/*.json')): + # because path is an object and not a string + path_str = str(path) + try: + with open(path_str, 'r') as file: + jstyleson.load(file) + print(f"Validation of {path_str} succeeded") + except Exception as exc: + print(f"Validation of {path_str} failed") + pprint(exc) + # https://stackoverflow.com/a/72850269/2278742 + if hasattr(exc, 'pos'): + position_msg = f"{path_str}:{exc.lineno}:{exc.colno}" + print(position_msg) + errors.append({"position": position_msg, "exception": exc}) + +if errors: + print("Summary of errors:") + pprint(errors) + raise Exception("Not all JSON files are valid") diff --git a/.github/workflows/github.yml b/.github/workflows/github.yml index 29e3db01d..321e715c8 100644 --- a/.github/workflows/github.yml +++ b/.github/workflows/github.yml @@ -145,6 +145,11 @@ jobs: with: submodules: recursive + - name: Validate JSON + run: | + pip install jstyleson + python3 .github/validate_json.py + - name: Dependencies run: source '${{github.workspace}}/CI/${{matrix.platform}}/before_install.sh' env: From 2394612e284f1924fafd43187fc255b6d06e3be0 Mon Sep 17 00:00:00 2001 From: Alexander Wilms Date: Fri, 15 Sep 2023 23:31:43 +0000 Subject: [PATCH 2/4] JSON validation: Support various levels of strictness --- .github/validate_json.py | 40 +++++++++++++++++++++++++++++++----- .github/workflows/github.yml | 2 +- 2 files changed, 36 insertions(+), 6 deletions(-) diff --git a/.github/validate_json.py b/.github/validate_json.py index 0c95557bc..8152c838f 100644 --- a/.github/validate_json.py +++ b/.github/validate_json.py @@ -1,6 +1,15 @@ import jstyleson from pathlib import Path from pprint import pprint +import yaml +import json5 +import re + +# 'json', 'json5' or 'yaml' +# json: strict, but doesn't preserve line numbers necessarily, since it strips comments before parsing +# json5: strict and preserves line numbers even for files will with line comments +# yaml: less strict, allows e.g. leading zeros +VALIDATION_TYPE = 'json5' errors = [] for path in sorted(Path('.').glob('**/*.json')): @@ -8,16 +17,37 @@ for path in sorted(Path('.').glob('**/*.json')): path_str = str(path) try: with open(path_str, 'r') as file: - jstyleson.load(file) + if VALIDATION_TYPE == 'json': + jstyleson.load(file) + if VALIDATION_TYPE == 'json5': + + json5.load(file) + elif VALIDATION_TYPE == 'yaml': + + file = file.read().replace("\t", " ") + file = file.replace("//", "#") + yaml.safe_load(file) print(f"Validation of {path_str} succeeded") except Exception as exc: print(f"Validation of {path_str} failed") pprint(exc) - # https://stackoverflow.com/a/72850269/2278742 + + error_pos = path_str + if hasattr(exc, 'pos'): - position_msg = f"{path_str}:{exc.lineno}:{exc.colno}" - print(position_msg) - errors.append({"position": position_msg, "exception": exc}) + # https://stackoverflow.com/a/72850269/2278742 + error_pos = f"{path_str}:{exc.lineno}:{exc.colno}" + print(error_pos) + # error_msg = "position": position_msg, "exception": exc + if hasattr(exc, 'problem_mark'): + mark = exc.problem_mark + error_pos = f"{path_str}:{mark.line+1}:{mark.column+1}" + print(error_pos) + if VALIDATION_TYPE == 'json5': + pos = re.findall(r'\d+', str(exc)) + error_pos = f"{path_str}:{pos[0]}:{pos[-1]}" + + errors.append({"error_pos": error_pos, "error_msg": exc}) if errors: print("Summary of errors:") diff --git a/.github/workflows/github.yml b/.github/workflows/github.yml index 321e715c8..03bb4c0a1 100644 --- a/.github/workflows/github.yml +++ b/.github/workflows/github.yml @@ -147,7 +147,7 @@ jobs: - name: Validate JSON run: | - pip install jstyleson + pip3 install json5 jstyleson python3 .github/validate_json.py - name: Dependencies From 512816b2232dc96c2ca63bbdd6c0bb56ff6b0f20 Mon Sep 17 00:00:00 2001 From: Alexander Wilms Date: Fri, 15 Sep 2023 23:52:04 +0000 Subject: [PATCH 3/4] Only validate JSON on Linux --- .github/validate_json.py | 12 +++++------- .github/workflows/github.yml | 2 ++ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/validate_json.py b/.github/validate_json.py index 8152c838f..e78d0601b 100644 --- a/.github/validate_json.py +++ b/.github/validate_json.py @@ -1,13 +1,14 @@ -import jstyleson +import re from pathlib import Path from pprint import pprint -import yaml + import json5 -import re +import jstyleson +import yaml # 'json', 'json5' or 'yaml' # json: strict, but doesn't preserve line numbers necessarily, since it strips comments before parsing -# json5: strict and preserves line numbers even for files will with line comments +# json5: strict and preserves line numbers even for files with line comments # yaml: less strict, allows e.g. leading zeros VALIDATION_TYPE = 'json5' @@ -20,10 +21,8 @@ for path in sorted(Path('.').glob('**/*.json')): if VALIDATION_TYPE == 'json': jstyleson.load(file) if VALIDATION_TYPE == 'json5': - json5.load(file) elif VALIDATION_TYPE == 'yaml': - file = file.read().replace("\t", " ") file = file.replace("//", "#") yaml.safe_load(file) @@ -38,7 +37,6 @@ for path in sorted(Path('.').glob('**/*.json')): # https://stackoverflow.com/a/72850269/2278742 error_pos = f"{path_str}:{exc.lineno}:{exc.colno}" print(error_pos) - # error_msg = "position": position_msg, "exception": exc if hasattr(exc, 'problem_mark'): mark = exc.problem_mark error_pos = f"{path_str}:{mark.line+1}:{mark.column+1}" diff --git a/.github/workflows/github.yml b/.github/workflows/github.yml index 03bb4c0a1..4242f8e12 100644 --- a/.github/workflows/github.yml +++ b/.github/workflows/github.yml @@ -146,6 +146,8 @@ jobs: submodules: recursive - name: Validate JSON + # the Python yaml module doesn't seem to work on mac-arm + if: ${{ startsWith(matrix.preset, 'linux') }} run: | pip3 install json5 jstyleson python3 .github/validate_json.py From e0089c76ae17d51e94b162c8dd6573aaac5de086 Mon Sep 17 00:00:00 2001 From: Alexander Wilms Date: Sat, 16 Sep 2023 11:18:09 +0000 Subject: [PATCH 4/4] Add shebang, use elif and only run on linux-clang-test --- .github/validate_json.py | 16 +++++++++++----- .github/workflows/github.yml | 3 ++- 2 files changed, 13 insertions(+), 6 deletions(-) mode change 100644 => 100755 .github/validate_json.py diff --git a/.github/validate_json.py b/.github/validate_json.py old mode 100644 new mode 100755 index e78d0601b..82dab94ec --- a/.github/validate_json.py +++ b/.github/validate_json.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python3 + import re from pathlib import Path from pprint import pprint @@ -20,7 +22,7 @@ for path in sorted(Path('.').glob('**/*.json')): with open(path_str, 'r') as file: if VALIDATION_TYPE == 'json': jstyleson.load(file) - if VALIDATION_TYPE == 'json5': + elif VALIDATION_TYPE == 'json5': json5.load(file) elif VALIDATION_TYPE == 'yaml': file = file.read().replace("\t", " ") @@ -33,17 +35,21 @@ for path in sorted(Path('.').glob('**/*.json')): error_pos = path_str + # create error position strings for each type of parser if hasattr(exc, 'pos'): + # 'json' # https://stackoverflow.com/a/72850269/2278742 error_pos = f"{path_str}:{exc.lineno}:{exc.colno}" print(error_pos) - if hasattr(exc, 'problem_mark'): + elif VALIDATION_TYPE == 'json5': + # 'json5' + pos = re.findall(r'\d+', str(exc)) + error_pos = f"{path_str}:{pos[0]}:{pos[-1]}" + elif hasattr(exc, 'problem_mark'): + # 'yaml' mark = exc.problem_mark error_pos = f"{path_str}:{mark.line+1}:{mark.column+1}" print(error_pos) - if VALIDATION_TYPE == 'json5': - pos = re.findall(r'\d+', str(exc)) - error_pos = f"{path_str}:{pos[0]}:{pos[-1]}" errors.append({"error_pos": error_pos, "error_msg": exc}) diff --git a/.github/workflows/github.yml b/.github/workflows/github.yml index 4242f8e12..eecb0f737 100644 --- a/.github/workflows/github.yml +++ b/.github/workflows/github.yml @@ -147,7 +147,8 @@ jobs: - name: Validate JSON # the Python yaml module doesn't seem to work on mac-arm - if: ${{ startsWith(matrix.preset, 'linux') }} + # also, running it on multiple presets is redundant and slightly increases already long CI built times + if: ${{ startsWith(matrix.preset, 'linux-clang-test') }} run: | pip3 install json5 jstyleson python3 .github/validate_json.py