From 27049797c1d11ed60cea5bcdf1304f23dcf13735 Mon Sep 17 00:00:00 2001 From: Hans Johnson Date: Mon, 4 Nov 2019 07:57:01 -0600 Subject: [PATCH 1/2] ENH: Unconditionally format python files with `black` pip install black black --target-version py36 . Black is the uncompromising Python code formatter. By using it, you agree to cede control over minutiae of hand-formatting. In return, Black gives you speed, determinism, and freedom from pycodestyle nagging about formatting. You will save time and mental energy for more important matters. Blackened code looks the same regardless of the project you're reading. Formatting becomes transparent after a while and you can focus on the content instead. Black makes code review faster by producing the smallest diffs possible. https://github.com/python/black --- test/cleantests.py | 4 +- test/generate_expected.py | 14 +-- test/pyjsontestrunner.py | 52 +++++----- test/runjsontests.py | 210 +++++++++++++++++++++++++------------- test/rununittests.py | 48 ++++++--- 5 files changed, 207 insertions(+), 121 deletions(-) diff --git a/test/cleantests.py b/test/cleantests.py index 36d5b9b46..3a3e1f430 100644 --- a/test/cleantests.py +++ b/test/cleantests.py @@ -9,8 +9,8 @@ import os paths = [] -for pattern in [ '*.actual', '*.actual-rewrite', '*.rewrite', '*.process-output' ]: - paths += glob.glob('data/' + pattern) +for pattern in ["*.actual", "*.actual-rewrite", "*.rewrite", "*.process-output"]: + paths += glob.glob("data/" + pattern) for path in paths: os.unlink(path) diff --git a/test/generate_expected.py b/test/generate_expected.py index e049ab5f4..f43baf87c 100644 --- a/test/generate_expected.py +++ b/test/generate_expected.py @@ -6,12 +6,12 @@ from __future__ import print_function import glob import os.path -for path in glob.glob('*.json'): - text = file(path,'rt').read() - target = os.path.splitext(path)[0] + '.expected' + +for path in glob.glob("*.json"): + text = file(path, "rt").read() + target = os.path.splitext(path)[0] + ".expected" if os.path.exists(target): - print('skipping:', target) + print("skipping:", target) else: - print('creating:', target) - file(target,'wt').write(text) - + print("creating:", target) + file(target, "wt").write(text) diff --git a/test/pyjsontestrunner.py b/test/pyjsontestrunner.py index bd749b530..67734661a 100644 --- a/test/pyjsontestrunner.py +++ b/test/pyjsontestrunner.py @@ -15,55 +15,59 @@ if len(sys.argv) != 2: print("Usage: %s input-json-file", sys.argv[0]) sys.exit(3) - + input_path = sys.argv[1] base_path = os.path.splitext(input_path)[0] -actual_path = base_path + '.actual' -rewrite_path = base_path + '.rewrite' -rewrite_actual_path = base_path + '.actual-rewrite' +actual_path = base_path + ".actual" +rewrite_path = base_path + ".rewrite" +rewrite_actual_path = base_path + ".actual-rewrite" + -def valueTreeToString(fout, value, path = '.'): - ty = type(value) - if ty is types.DictType: - fout.write('%s={}\n' % path) - suffix = path[-1] != '.' and '.' or '' +def valueTreeToString(fout, value, path="."): + ty = type(value) + if ty is types.DictType: + fout.write("%s={}\n" % path) + suffix = path[-1] != "." and "." or "" names = value.keys() names.sort() for name in names: valueTreeToString(fout, value[name], path + suffix + name) elif ty is types.ListType: - fout.write('%s=[]\n' % path) - for index, childValue in zip(xrange(0,len(value)), value): - valueTreeToString(fout, childValue, path + '[%d]' % index) + fout.write("%s=[]\n" % path) + for index, childValue in zip(xrange(0, len(value)), value): + valueTreeToString(fout, childValue, path + "[%d]" % index) elif ty is types.StringType: - fout.write('%s="%s"\n' % (path,value)) + fout.write('%s="%s"\n' % (path, value)) elif ty is types.IntType: - fout.write('%s=%d\n' % (path,value)) + fout.write("%s=%d\n" % (path, value)) elif ty is types.FloatType: - fout.write('%s=%.16g\n' % (path,value)) + fout.write("%s=%.16g\n" % (path, value)) elif value is True: - fout.write('%s=true\n' % path) + fout.write("%s=true\n" % path) elif value is False: - fout.write('%s=false\n' % path) + fout.write("%s=false\n" % path) elif value is None: - fout.write('%s=null\n' % path) + fout.write("%s=null\n" % path) else: assert False and "Unexpected value type" - + + def parseAndSaveValueTree(input, actual_path): root = json.loads(input) - fout = file(actual_path, 'wt') + fout = file(actual_path, "wt") valueTreeToString(fout, root) fout.close() return root + def rewriteValueTree(value, rewrite_path): rewrite = json.dumps(value) - #rewrite = rewrite[1:-1] # Somehow the string is quoted ! jsonpy bug ? - file(rewrite_path, 'wt').write(rewrite + '\n') + # rewrite = rewrite[1:-1] # Somehow the string is quoted ! jsonpy bug ? + file(rewrite_path, "wt").write(rewrite + "\n") return rewrite - -input = file(input_path, 'rt').read() + + +input = file(input_path, "rt").read() root = parseAndSaveValueTree(input, actual_path) rewrite = rewriteValueTree(json.write(root), rewrite_path) rewrite_root = parseAndSaveValueTree(rewrite, rewrite_actual_path) diff --git a/test/runjsontests.py b/test/runjsontests.py index dfdeca3ea..d83717ca8 100644 --- a/test/runjsontests.py +++ b/test/runjsontests.py @@ -12,7 +12,8 @@ import os.path import optparse -VALGRIND_CMD = 'valgrind --tool=memcheck --leak-check=yes --undef-value-errors=yes ' +VALGRIND_CMD = "valgrind --tool=memcheck --leak-check=yes --undef-value-errors=yes " + def getStatusOutput(cmd): """ @@ -26,50 +27,63 @@ def getStatusOutput(cmd): # We have been using os.popen(). When we read() the result # we get 'str' (bytes) in py2, and 'str' (unicode) in py3. # Ugh! There must be a better way to handle this. - process_output = process_output.decode('utf-8') + process_output = process_output.decode("utf-8") except AttributeError: pass # python3 status = pipe.close() return status, process_output + + def compareOutputs(expected, actual, message): - expected = expected.strip().replace('\r','').split('\n') - actual = actual.strip().replace('\r','').split('\n') + expected = expected.strip().replace("\r", "").split("\n") + actual = actual.strip().replace("\r", "").split("\n") diff_line = 0 max_line_to_compare = min(len(expected), len(actual)) - for index in range(0,max_line_to_compare): + for index in range(0, max_line_to_compare): if expected[index].strip() != actual[index].strip(): diff_line = index + 1 break if diff_line == 0 and len(expected) != len(actual): - diff_line = max_line_to_compare+1 + diff_line = max_line_to_compare + 1 if diff_line == 0: return None + def safeGetLine(lines, index): index += -1 if index >= len(lines): - return '' + return "" return lines[index].strip() + return """ Difference in %s at line %d: Expected: '%s' Actual: '%s' -""" % (message, diff_line, - safeGetLine(expected,diff_line), - safeGetLine(actual,diff_line)) +""" % ( + message, + diff_line, + safeGetLine(expected, diff_line), + safeGetLine(actual, diff_line), + ) + def safeReadFile(path): try: - return open(path, 'rt', encoding = 'utf-8').read() + return open(path, "rt", encoding="utf-8").read() except IOError as e: - return '' % (path,e) + return '' % (path, e) + -def runAllTests(jsontest_executable_path, input_dir = None, - use_valgrind=False, with_json_checker=False, - writerClass='StyledWriter'): +def runAllTests( + jsontest_executable_path, + input_dir=None, + use_valgrind=False, + with_json_checker=False, + writerClass="StyledWriter", +): if not input_dir: - input_dir = os.path.join(os.getcwd(), 'data') - tests = glob(os.path.join(input_dir, '*.json')) + input_dir = os.path.join(os.getcwd(), "data") + tests = glob(os.path.join(input_dir, "*.json")) if with_json_checker: - all_test_jsonchecker = glob(os.path.join(input_dir, '../jsonchecker', '*.json')) + all_test_jsonchecker = glob(os.path.join(input_dir, "../jsonchecker", "*.json")) # These tests fail with strict json support, but pass with jsoncpp extra lieniency """ Failure details: @@ -102,84 +116,126 @@ def runAllTests(jsontest_executable_path, input_dir = None, ["line break"] """ - known_differences_withjsonchecker = [ "fail25.json", "fail13.json", "fail18.json", "fail8.json", - "fail7.json", "fail10.json", "fail27.json" ] - test_jsonchecker = [ test for test in all_test_jsonchecker if os.path.basename(test) not in known_differences_withjsonchecker ] + known_differences_withjsonchecker = [ + "fail25.json", + "fail13.json", + "fail18.json", + "fail8.json", + "fail7.json", + "fail10.json", + "fail27.json", + ] + test_jsonchecker = [ + test + for test in all_test_jsonchecker + if os.path.basename(test) not in known_differences_withjsonchecker + ] else: test_jsonchecker = [] failed_tests = [] - valgrind_path = use_valgrind and VALGRIND_CMD or '' + valgrind_path = use_valgrind and VALGRIND_CMD or "" for input_path in tests + test_jsonchecker: - expect_failure = os.path.basename(input_path).startswith('fail') + expect_failure = os.path.basename(input_path).startswith("fail") is_json_checker_test = (input_path in test_jsonchecker) or expect_failure - print('TESTING:', input_path, end=' ') - options = is_json_checker_test and '--json-checker' or '' - options += ' --json-writer %s'%writerClass - cmd = '%s%s %s "%s"' % ( valgrind_path, jsontest_executable_path, options, - input_path) + print("TESTING:", input_path, end=" ") + options = is_json_checker_test and "--json-checker" or "" + options += " --json-writer %s" % writerClass + cmd = '%s%s %s "%s"' % ( + valgrind_path, + jsontest_executable_path, + options, + input_path, + ) status, process_output = getStatusOutput(cmd) if is_json_checker_test: if expect_failure: if not status: - print('FAILED') - failed_tests.append((input_path, 'Parsing should have failed:\n%s' % - safeReadFile(input_path))) + print("FAILED") + failed_tests.append( + ( + input_path, + "Parsing should have failed:\n%s" + % safeReadFile(input_path), + ) + ) else: - print('OK') + print("OK") else: if status: - print('FAILED') - failed_tests.append((input_path, 'Parsing failed:\n' + process_output)) + print("FAILED") + failed_tests.append( + (input_path, "Parsing failed:\n" + process_output) + ) else: - print('OK') + print("OK") else: base_path = os.path.splitext(input_path)[0] - actual_output = safeReadFile(base_path + '.actual') - actual_rewrite_output = safeReadFile(base_path + '.actual-rewrite') - open(base_path + '.process-output', 'wt', encoding = 'utf-8').write(process_output) + actual_output = safeReadFile(base_path + ".actual") + actual_rewrite_output = safeReadFile(base_path + ".actual-rewrite") + open(base_path + ".process-output", "wt", encoding="utf-8").write( + process_output + ) if status: - print('parsing failed') - failed_tests.append((input_path, 'Parsing failed:\n' + process_output)) + print("parsing failed") + failed_tests.append((input_path, "Parsing failed:\n" + process_output)) else: - expected_output_path = os.path.splitext(input_path)[0] + '.expected' - expected_output = open(expected_output_path, 'rt', encoding = 'utf-8').read() - detail = (compareOutputs(expected_output, actual_output, 'input') - or compareOutputs(expected_output, actual_rewrite_output, 'rewrite')) + expected_output_path = os.path.splitext(input_path)[0] + ".expected" + expected_output = open( + expected_output_path, "rt", encoding="utf-8" + ).read() + detail = compareOutputs( + expected_output, actual_output, "input" + ) or compareOutputs(expected_output, actual_rewrite_output, "rewrite") if detail: - print('FAILED') + print("FAILED") failed_tests.append((input_path, detail)) else: - print('OK') + print("OK") if failed_tests: print() - print('Failure details:') + print("Failure details:") for failed_test in failed_tests: - print('* Test', failed_test[0]) + print("* Test", failed_test[0]) print(failed_test[1]) print() - print('Test results: %d passed, %d failed.' % (len(tests)-len(failed_tests), - len(failed_tests))) + print( + "Test results: %d passed, %d failed." + % (len(tests) - len(failed_tests), len(failed_tests)) + ) return 1 else: - print('All %d tests passed.' % len(tests)) + print("All %d tests passed." % len(tests)) return 0 + def main(): from optparse import OptionParser - parser = OptionParser(usage="%prog [options] [test case directory]") - parser.add_option("--valgrind", - action="store_true", dest="valgrind", default=False, - help="run all the tests using valgrind to detect memory leaks") - parser.add_option("-c", "--with-json-checker", - action="store_true", dest="with_json_checker", default=False, - help="run all the tests from the official JSONChecker test suite of json.org") + + parser = OptionParser( + usage="%prog [options] [test case directory]" + ) + parser.add_option( + "--valgrind", + action="store_true", + dest="valgrind", + default=False, + help="run all the tests using valgrind to detect memory leaks", + ) + parser.add_option( + "-c", + "--with-json-checker", + action="store_true", + dest="with_json_checker", + default=False, + help="run all the tests from the official JSONChecker test suite of json.org", + ) parser.enable_interspersed_args() options, args = parser.parse_args() if len(args) < 1 or len(args) > 2: - parser.error('Must provides at least path to jsontestrunner executable.') + parser.error("Must provides at least path to jsontestrunner executable.") sys.exit(1) jsontest_executable_path = os.path.normpath(os.path.abspath(args[0])) @@ -187,24 +243,34 @@ def main(): input_path = os.path.normpath(os.path.abspath(args[1])) else: input_path = None - status = runAllTests(jsontest_executable_path, input_path, - use_valgrind=options.valgrind, - with_json_checker=options.with_json_checker, - writerClass='StyledWriter') + status = runAllTests( + jsontest_executable_path, + input_path, + use_valgrind=options.valgrind, + with_json_checker=options.with_json_checker, + writerClass="StyledWriter", + ) if status: sys.exit(status) - status = runAllTests(jsontest_executable_path, input_path, - use_valgrind=options.valgrind, - with_json_checker=options.with_json_checker, - writerClass='StyledStreamWriter') + status = runAllTests( + jsontest_executable_path, + input_path, + use_valgrind=options.valgrind, + with_json_checker=options.with_json_checker, + writerClass="StyledStreamWriter", + ) if status: sys.exit(status) - status = runAllTests(jsontest_executable_path, input_path, - use_valgrind=options.valgrind, - with_json_checker=options.with_json_checker, - writerClass='BuiltStyledStreamWriter') + status = runAllTests( + jsontest_executable_path, + input_path, + use_valgrind=options.valgrind, + with_json_checker=options.with_json_checker, + writerClass="BuiltStyledStreamWriter", + ) if status: sys.exit(status) -if __name__ == '__main__': + +if __name__ == "__main__": main() diff --git a/test/rununittests.py b/test/rununittests.py index 6634e7241..8b86e1db3 100644 --- a/test/rununittests.py +++ b/test/rununittests.py @@ -13,7 +13,8 @@ import subprocess import optparse -VALGRIND_CMD = 'valgrind --tool=memcheck --leak-check=yes --undef-value-errors=yes' +VALGRIND_CMD = "valgrind --tool=memcheck --leak-check=yes --undef-value-errors=yes" + class TestProxy(object): def __init__(self, test_exe_path, use_valgrind=False): @@ -25,9 +26,11 @@ def run(self, options): cmd = VALGRIND_CMD.split() else: cmd = [] - cmd.extend([self.test_exe_path, '--test-auto'] + options) + cmd.extend([self.test_exe_path, "--test-auto"] + options) try: - process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + process = subprocess.Popen( + cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT + ) except: print(cmd) raise @@ -36,49 +39,62 @@ def run(self, options): return False, stdout return True, stdout + def runAllTests(exe_path, use_valgrind=False): test_proxy = TestProxy(exe_path, use_valgrind=use_valgrind) - status, test_names = test_proxy.run(['--list-tests']) + status, test_names = test_proxy.run(["--list-tests"]) if not status: print("Failed to obtain unit tests list:\n" + test_names, file=sys.stderr) return 1 - test_names = [name.strip() for name in test_names.decode('utf-8').strip().split('\n')] + test_names = [ + name.strip() for name in test_names.decode("utf-8").strip().split("\n") + ] failures = [] for name in test_names: - print('TESTING %s:' % name, end=' ') - succeed, result = test_proxy.run(['--test', name]) + print("TESTING %s:" % name, end=" ") + succeed, result = test_proxy.run(["--test", name]) if succeed: - print('OK') + print("OK") else: failures.append((name, result)) - print('FAILED') + print("FAILED") failed_count = len(failures) pass_count = len(test_names) - failed_count if failed_count: print() for name, result in failures: print(result) - print('%d/%d tests passed (%d failure(s))' % ( pass_count, len(test_names), failed_count)) + print( + "%d/%d tests passed (%d failure(s))" + % (pass_count, len(test_names), failed_count) + ) return 1 else: - print('All %d tests passed' % len(test_names)) + print("All %d tests passed" % len(test_names)) return 0 + def main(): from optparse import OptionParser + parser = OptionParser(usage="%prog [options] ") - parser.add_option("--valgrind", - action="store_true", dest="valgrind", default=False, - help="run all the tests using valgrind to detect memory leaks") + parser.add_option( + "--valgrind", + action="store_true", + dest="valgrind", + default=False, + help="run all the tests using valgrind to detect memory leaks", + ) parser.enable_interspersed_args() options, args = parser.parse_args() if len(args) != 1: - parser.error('Must provides at least path to test_lib_json executable.') + parser.error("Must provides at least path to test_lib_json executable.") sys.exit(1) exit_code = runAllTests(args[0], use_valgrind=options.valgrind) sys.exit(exit_code) -if __name__ == '__main__': + +if __name__ == "__main__": main() From 7ff840dd8eabd4ce6c80e371656848a7d41ba4dc Mon Sep 17 00:00:00 2001 From: Hans Johnson Date: Mon, 4 Nov 2019 08:01:40 -0600 Subject: [PATCH 2/2] STYLE: Prefer python 3.6 language styles ```bash pip install pyupgrade pyupgrade --py36-plus *.py ``` --- test/generate_expected.py | 1 - test/pyjsontestrunner.py | 5 ++--- test/runjsontests.py | 8 +++----- test/rununittests.py | 4 +--- 4 files changed, 6 insertions(+), 12 deletions(-) diff --git a/test/generate_expected.py b/test/generate_expected.py index f43baf87c..9918527fb 100644 --- a/test/generate_expected.py +++ b/test/generate_expected.py @@ -3,7 +3,6 @@ # recognized in your jurisdiction. # See file LICENSE for detail or copy at http://jsoncpp.sourceforge.net/LICENSE -from __future__ import print_function import glob import os.path diff --git a/test/pyjsontestrunner.py b/test/pyjsontestrunner.py index 67734661a..2224fb7b8 100644 --- a/test/pyjsontestrunner.py +++ b/test/pyjsontestrunner.py @@ -6,7 +6,6 @@ """Simple implementation of a json test runner to run the test against json-py.""" -from __future__ import print_function import sys import os.path import json @@ -37,11 +36,11 @@ def valueTreeToString(fout, value, path="."): for index, childValue in zip(xrange(0, len(value)), value): valueTreeToString(fout, childValue, path + "[%d]" % index) elif ty is types.StringType: - fout.write('%s="%s"\n' % (path, value)) + fout.write(f'{path}="{value}"\n') elif ty is types.IntType: fout.write("%s=%d\n" % (path, value)) elif ty is types.FloatType: - fout.write("%s=%.16g\n" % (path, value)) + fout.write(f"{path}={value:.16g}\n") elif value is True: fout.write("%s=true\n" % path) elif value is False: diff --git a/test/runjsontests.py b/test/runjsontests.py index d83717ca8..7353ebd14 100644 --- a/test/runjsontests.py +++ b/test/runjsontests.py @@ -3,8 +3,6 @@ # recognized in your jurisdiction. # See file LICENSE for detail or copy at http://jsoncpp.sourceforge.net/LICENSE -from __future__ import print_function -from __future__ import unicode_literals from io import open from glob import glob import sys @@ -68,8 +66,8 @@ def safeGetLine(lines, index): def safeReadFile(path): try: return open(path, "rt", encoding="utf-8").read() - except IOError as e: - return '' % (path, e) + except OSError as e: + return f'' def runAllTests( @@ -141,7 +139,7 @@ def runAllTests( print("TESTING:", input_path, end=" ") options = is_json_checker_test and "--json-checker" or "" options += " --json-writer %s" % writerClass - cmd = '%s%s %s "%s"' % ( + cmd = '{}{} {} "{}"'.format( valgrind_path, jsontest_executable_path, options, diff --git a/test/rununittests.py b/test/rununittests.py index 8b86e1db3..02b011465 100644 --- a/test/rununittests.py +++ b/test/rununittests.py @@ -3,8 +3,6 @@ # recognized in your jurisdiction. # See file LICENSE for detail or copy at http://jsoncpp.sourceforge.net/LICENSE -from __future__ import print_function -from __future__ import unicode_literals from io import open from glob import glob import sys @@ -16,7 +14,7 @@ VALGRIND_CMD = "valgrind --tool=memcheck --leak-check=yes --undef-value-errors=yes" -class TestProxy(object): +class TestProxy: def __init__(self, test_exe_path, use_valgrind=False): self.test_exe_path = os.path.normpath(os.path.abspath(test_exe_path)) self.use_valgrind = use_valgrind