From 5f55f485c8e3b23d4eb2ceb63111d53a0ff4a3ed Mon Sep 17 00:00:00 2001 From: Stan Ulbrych Date: Sat, 26 Apr 2025 19:41:05 +0100 Subject: [PATCH 1/5] commit --- Lib/test/test_tools/i18n_data/messages.pot | 10 +++++----- Lib/test/test_tools/i18n_data/messages.py | 3 +++ .../test_tools/i18n_data/skipdocstrings.py | 9 +++++++++ .../test_tools/i18n_data/skipdocstrings.txt | 5 +++++ Lib/test/test_tools/test_i18n.py | 19 ++++++++++-------- ...4-26-19-26-00.gh-issues-130197.sjdhsja.rst | 1 + Tools/i18n/pygettext.py | 20 ++++++++----------- 7 files changed, 42 insertions(+), 25 deletions(-) create mode 100644 Lib/test/test_tools/i18n_data/skipdocstrings.py create mode 100644 Lib/test/test_tools/i18n_data/skipdocstrings.txt create mode 100644 Misc/NEWS.d/next/Library/2025-04-26-19-26-00.gh-issues-130197.sjdhsja.rst diff --git a/Lib/test/test_tools/i18n_data/messages.pot b/Lib/test/test_tools/i18n_data/messages.pot index e8167acfc0742b..d5c368e3d82f01 100644 --- a/Lib/test/test_tools/i18n_data/messages.pot +++ b/Lib/test/test_tools/i18n_data/messages.pot @@ -33,8 +33,8 @@ msgid "" " multiline!\n" msgstr "" -#: messages.py:46 messages.py:89 messages.py:90 messages.py:93 messages.py:94 -#: messages.py:99 messages.py:100 messages.py:101 +#: messages.py:46 messages.py:92 messages.py:93 messages.py:96 messages.py:97 +#: messages.py:102 messages.py:103 messages.py:104 msgid "foo" msgid_plural "foos" msgstr[0] "" @@ -80,18 +80,18 @@ msgstr "" msgid "default value" msgstr "" -#: messages.py:91 messages.py:92 messages.py:95 messages.py:96 +#: messages.py:94 messages.py:95 messages.py:98 messages.py:99 msgctxt "context" msgid "foo" msgid_plural "foos" msgstr[0] "" msgstr[1] "" -#: messages.py:102 +#: messages.py:105 msgid "domain foo" msgstr "" -#: messages.py:118 messages.py:119 +#: messages.py:121 messages.py:122 msgid "world" msgid_plural "worlds" msgstr[0] "" diff --git a/Lib/test/test_tools/i18n_data/messages.py b/Lib/test/test_tools/i18n_data/messages.py index 9457bcb8611020..fe7c2c304ee8ee 100644 --- a/Lib/test/test_tools/i18n_data/messages.py +++ b/Lib/test/test_tools/i18n_data/messages.py @@ -85,6 +85,9 @@ def _(x="don't extract me"): pass +def func(): + """Docstring...""" + # Other gettext functions gettext("foo") ngettext("foo", "foos", 1) diff --git a/Lib/test/test_tools/i18n_data/skipdocstrings.py b/Lib/test/test_tools/i18n_data/skipdocstrings.py new file mode 100644 index 00000000000000..bdf498b506dc49 --- /dev/null +++ b/Lib/test/test_tools/i18n_data/skipdocstrings.py @@ -0,0 +1,9 @@ +def test(x): + """'I think he’s had a heart attack!' + 'IF THERE’S ONE THING I CAN’T STAND IS PEOPLE HAVING HEART ATTACKS!!!' + 'I’m fine, now!' + 'Good. Now we’re getting somewhere.' + """ + +class Foo: + """Docstring""" diff --git a/Lib/test/test_tools/i18n_data/skipdocstrings.txt b/Lib/test/test_tools/i18n_data/skipdocstrings.txt new file mode 100644 index 00000000000000..d0dfb84e78ab09 --- /dev/null +++ b/Lib/test/test_tools/i18n_data/skipdocstrings.txt @@ -0,0 +1,5 @@ +fileloc.py + +skipdocstrings.py +messages.py +messages.py \ No newline at end of file diff --git a/Lib/test/test_tools/test_i18n.py b/Lib/test/test_tools/test_i18n.py index 8416b1bad825eb..676048a8cb32bc 100644 --- a/Lib/test/test_tools/test_i18n.py +++ b/Lib/test/test_tools/test_i18n.py @@ -589,6 +589,7 @@ def extract_from_snapshots(): 'messages.py': (), 'fileloc.py': ('--docstrings',), 'docstrings.py': ('--docstrings',), + ('docstrings.py', 'skipdocstrings.py', 'docstrings.pot'): ('--docstrings', f'--no-docstrings={DATA_DIR}{os.path.sep}skipdocstrings.txt'), 'comments.py': ('--add-comments=i18n:',), 'custom_keywords.py': ('--keyword=foo', '--keyword=nfoo:1,2', '--keyword=pfoo:1c,2', @@ -606,18 +607,20 @@ def extract_from_snapshots(): for filename, args in snapshots.items(): if isinstance(filename, tuple): - filename, output_file = filename + *filenames, output_file = filename output_file = DATA_DIR / output_file - input_file = DATA_DIR / filename + input_files = [DATA_DIR / file for file in filenames] else: - input_file = DATA_DIR / filename - output_file = input_file.with_suffix('.pot') - contents = input_file.read_bytes() + input_files = [DATA_DIR / filename] + output_file = input_files[0].with_suffix('.pot') + with temp_cwd(None): - Path(input_file.name).write_bytes(contents) + for input_file in input_files: + contents = input_file.read_bytes() + Path(input_file.name).write_bytes(contents) assert_python_ok('-Xutf8', Test_pygettext.script, *args, - input_file.name) - yield (input_file, output_file, + *[file.name for file in input_files]) + yield (input_files, output_file, Path('messages.pot').read_text(encoding='utf-8')) diff --git a/Misc/NEWS.d/next/Library/2025-04-26-19-26-00.gh-issues-130197.sjdhsja.rst b/Misc/NEWS.d/next/Library/2025-04-26-19-26-00.gh-issues-130197.sjdhsja.rst new file mode 100644 index 00000000000000..31c79e595e8bc6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-04-26-19-26-00.gh-issues-130197.sjdhsja.rst @@ -0,0 +1 @@ +Fix and test the ``--no-docstrings`` option in :program:`pygettext`. diff --git a/Tools/i18n/pygettext.py b/Tools/i18n/pygettext.py index f46b05067d7fde..394a19fda18e6b 100755 --- a/Tools/i18n/pygettext.py +++ b/Tools/i18n/pygettext.py @@ -483,7 +483,7 @@ def visit_Call(self, node): def _extract_docstring(self, node): if (not self.options.docstrings or - self.options.nodocstrings.get(self.filename)): + os.path.basename(self.filename) in self.options.nodocstrings): return docstring = ast.get_docstring(node) @@ -692,7 +692,7 @@ def main(): 'help', 'keyword=', 'no-default-keywords', 'add-location', 'no-location', 'output=', 'output-dir=', 'style=', 'verbose', 'version', 'width=', 'exclude-file=', - 'docstrings', 'no-docstrings', + 'docstrings', 'no-docstrings=', ]) except getopt.error as msg: usage(1, msg) @@ -714,7 +714,7 @@ class Options: width = 78 excludefilename = '' docstrings = 0 - nodocstrings = {} + nodocstrings = [] comment_tags = set() options = Options() @@ -767,15 +767,11 @@ class Options: elif opt in ('-x', '--exclude-file'): options.excludefilename = arg elif opt in ('-X', '--no-docstrings'): - fp = open(arg) - try: - while 1: - line = fp.readline() - if not line: - break - options.nodocstrings[line[:-1]] = 1 - finally: - fp.close() + with open(arg, 'r') as nodocstrings_file: + for line in nodocstrings_file: + filename = os.path.basename(line.strip()) + if filename not in options.nodocstrings: + options.nodocstrings.append(filename) options.comment_tags = tuple(options.comment_tags) From 6d62ea4ee03dedd645206f49040d824e47b0cd10 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych Date: Sat, 26 Apr 2025 19:45:46 +0100 Subject: [PATCH 2/5] Fix NEWS name --- ...jdhsja.rst => 2025-04-26-19-26-00.gh-issue-130197.sjdhsja.rst} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename Misc/NEWS.d/next/Library/{2025-04-26-19-26-00.gh-issues-130197.sjdhsja.rst => 2025-04-26-19-26-00.gh-issue-130197.sjdhsja.rst} (100%) diff --git a/Misc/NEWS.d/next/Library/2025-04-26-19-26-00.gh-issues-130197.sjdhsja.rst b/Misc/NEWS.d/next/Library/2025-04-26-19-26-00.gh-issue-130197.sjdhsja.rst similarity index 100% rename from Misc/NEWS.d/next/Library/2025-04-26-19-26-00.gh-issues-130197.sjdhsja.rst rename to Misc/NEWS.d/next/Library/2025-04-26-19-26-00.gh-issue-130197.sjdhsja.rst From e64847f20729a77edf1b41bc9a6e702d5e219196 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych Date: Sun, 27 Apr 2025 10:47:53 +0100 Subject: [PATCH 3/5] Rename to *exclude-docstrings* --- Lib/test/test_tools/test_i18n.py | 3 ++- .../Library/2025-04-26-19-26-00.gh-issue-130197.sjdhsja.rst | 1 - .../2025-04-26-19-26-00.gh-issue-130197.sjdhsja.rst | 2 ++ Tools/i18n/pygettext.py | 6 +++--- 4 files changed, 7 insertions(+), 5 deletions(-) delete mode 100644 Misc/NEWS.d/next/Library/2025-04-26-19-26-00.gh-issue-130197.sjdhsja.rst create mode 100644 Misc/NEWS.d/next/Tools-Demos/2025-04-26-19-26-00.gh-issue-130197.sjdhsja.rst diff --git a/Lib/test/test_tools/test_i18n.py b/Lib/test/test_tools/test_i18n.py index 676048a8cb32bc..c9731324fb7f0b 100644 --- a/Lib/test/test_tools/test_i18n.py +++ b/Lib/test/test_tools/test_i18n.py @@ -589,7 +589,8 @@ def extract_from_snapshots(): 'messages.py': (), 'fileloc.py': ('--docstrings',), 'docstrings.py': ('--docstrings',), - ('docstrings.py', 'skipdocstrings.py', 'docstrings.pot'): ('--docstrings', f'--no-docstrings={DATA_DIR}{os.path.sep}skipdocstrings.txt'), + ('docstrings.py', 'skipdocstrings.py', 'docstrings.pot'): ('--docstrings', + f'--exclude-docstrings={DATA_DIR}{os.path.sep}skipdocstrings.txt'), 'comments.py': ('--add-comments=i18n:',), 'custom_keywords.py': ('--keyword=foo', '--keyword=nfoo:1,2', '--keyword=pfoo:1c,2', diff --git a/Misc/NEWS.d/next/Library/2025-04-26-19-26-00.gh-issue-130197.sjdhsja.rst b/Misc/NEWS.d/next/Library/2025-04-26-19-26-00.gh-issue-130197.sjdhsja.rst deleted file mode 100644 index 31c79e595e8bc6..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-04-26-19-26-00.gh-issue-130197.sjdhsja.rst +++ /dev/null @@ -1 +0,0 @@ -Fix and test the ``--no-docstrings`` option in :program:`pygettext`. diff --git a/Misc/NEWS.d/next/Tools-Demos/2025-04-26-19-26-00.gh-issue-130197.sjdhsja.rst b/Misc/NEWS.d/next/Tools-Demos/2025-04-26-19-26-00.gh-issue-130197.sjdhsja.rst new file mode 100644 index 00000000000000..0e1d69409c30bc --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2025-04-26-19-26-00.gh-issue-130197.sjdhsja.rst @@ -0,0 +1,2 @@ +Fix and test the ``--exclude-docstrings`` option in :program:`pygettext`. +Note: the option was also renamed from ``--no-docstrings`` for clarity. diff --git a/Tools/i18n/pygettext.py b/Tools/i18n/pygettext.py index 394a19fda18e6b..6ae72adac1f57d 100755 --- a/Tools/i18n/pygettext.py +++ b/Tools/i18n/pygettext.py @@ -131,7 +131,7 @@ appear on a line by itself in the file. -X filename - --no-docstrings=filename + --exclude-docstrings=filename Specify a file that contains a list of files (one per line) that should not have their docstrings extracted. This is only useful in conjunction with the -D option above. @@ -692,7 +692,7 @@ def main(): 'help', 'keyword=', 'no-default-keywords', 'add-location', 'no-location', 'output=', 'output-dir=', 'style=', 'verbose', 'version', 'width=', 'exclude-file=', - 'docstrings', 'no-docstrings=', + 'docstrings', 'exclude-docstrings=', ]) except getopt.error as msg: usage(1, msg) @@ -766,7 +766,7 @@ class Options: usage(1, f'--width argument must be an integer: {arg}') elif opt in ('-x', '--exclude-file'): options.excludefilename = arg - elif opt in ('-X', '--no-docstrings'): + elif opt in ('-X', '--exclude-docstrings'): with open(arg, 'r') as nodocstrings_file: for line in nodocstrings_file: filename = os.path.basename(line.strip()) From c52e785ba43b2a8b12e68e0c3838ba709283431c Mon Sep 17 00:00:00 2001 From: Stan Ulbrych Date: Sun, 27 Apr 2025 16:09:16 +0100 Subject: [PATCH 4/5] Use fnmatch --- Tools/i18n/pygettext.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/Tools/i18n/pygettext.py b/Tools/i18n/pygettext.py index 6ae72adac1f57d..a3c3657cfb4cda 100755 --- a/Tools/i18n/pygettext.py +++ b/Tools/i18n/pygettext.py @@ -96,7 +96,7 @@ -o filename --output=filename Rename the default output file from messages.pot to filename. If - filename is `-' then the output is sent to standard out. + filename is '-' then the output is sent to standard out. -p dir --output-dir=dir @@ -110,7 +110,7 @@ Solaris # File: filename, line: line-number GNU #: filename:line - The style name is case insensitive. GNU style is the default. + The style name is case-insensitive. GNU style is the default. -v --verbose @@ -136,10 +136,11 @@ should not have their docstrings extracted. This is only useful in conjunction with the -D option above. -If `inputfile' is -, standard input is read. +If 'inputfile' is -, standard input is read. """ import ast +import fnmatch import getopt import glob import importlib.machinery @@ -188,7 +189,7 @@ def make_escapes(pass_nonascii): global escapes, escape if pass_nonascii: # Allow non-ascii characters to pass through so that e.g. 'msgid - # "Höhe"' would not result in 'msgid "H\366he"'. Otherwise we + # "Höhe"' would not result in 'msgid "H\366he"'. Otherwise, we # escape any character outside the 32..126 range. escape = escape_ascii else: @@ -483,7 +484,8 @@ def visit_Call(self, node): def _extract_docstring(self, node): if (not self.options.docstrings or - os.path.basename(self.filename) in self.options.nodocstrings): + any(fnmatch.fnmatch(self.filename, pattern) + for pattern in self.options.nodocstrings)): return docstring = ast.get_docstring(node) @@ -769,9 +771,9 @@ class Options: elif opt in ('-X', '--exclude-docstrings'): with open(arg, 'r') as nodocstrings_file: for line in nodocstrings_file: - filename = os.path.basename(line.strip()) - if filename not in options.nodocstrings: - options.nodocstrings.append(filename) + line = line.strip() + if line and line not in options.nodocstrings: + options.nodocstrings.append(line) options.comment_tags = tuple(options.comment_tags) From 1ad4ae94552e02894054527cba1807d300d05ea1 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych Date: Sun, 27 Apr 2025 16:37:00 +0100 Subject: [PATCH 5/5] Support fnmatch --- Lib/test/test_tools/i18n_data/skipdocstrings.txt | 3 ++- Lib/test/test_tools/i18n_data/testfile.py | 2 ++ Lib/test/test_tools/test_i18n.py | 2 +- Tools/i18n/pygettext.py | 15 ++++++++------- 4 files changed, 13 insertions(+), 9 deletions(-) create mode 100644 Lib/test/test_tools/i18n_data/testfile.py diff --git a/Lib/test/test_tools/i18n_data/skipdocstrings.txt b/Lib/test/test_tools/i18n_data/skipdocstrings.txt index d0dfb84e78ab09..3e545dc7999de4 100644 --- a/Lib/test/test_tools/i18n_data/skipdocstrings.txt +++ b/Lib/test/test_tools/i18n_data/skipdocstrings.txt @@ -2,4 +2,5 @@ fileloc.py skipdocstrings.py messages.py -messages.py \ No newline at end of file +messages.py +test*.py \ No newline at end of file diff --git a/Lib/test/test_tools/i18n_data/testfile.py b/Lib/test/test_tools/i18n_data/testfile.py new file mode 100644 index 00000000000000..9bbf8849508ca5 --- /dev/null +++ b/Lib/test/test_tools/i18n_data/testfile.py @@ -0,0 +1,2 @@ +def func(): + """Get the Holy Hand Grenade!""" diff --git a/Lib/test/test_tools/test_i18n.py b/Lib/test/test_tools/test_i18n.py index c9731324fb7f0b..625bb1ce738ade 100644 --- a/Lib/test/test_tools/test_i18n.py +++ b/Lib/test/test_tools/test_i18n.py @@ -589,7 +589,7 @@ def extract_from_snapshots(): 'messages.py': (), 'fileloc.py': ('--docstrings',), 'docstrings.py': ('--docstrings',), - ('docstrings.py', 'skipdocstrings.py', 'docstrings.pot'): ('--docstrings', + ('docstrings.py', 'skipdocstrings.py', 'testfile.py', 'docstrings.pot'): ('--docstrings', f'--exclude-docstrings={DATA_DIR}{os.path.sep}skipdocstrings.txt'), 'comments.py': ('--add-comments=i18n:',), 'custom_keywords.py': ('--keyword=foo', '--keyword=nfoo:1,2', diff --git a/Tools/i18n/pygettext.py b/Tools/i18n/pygettext.py index a3c3657cfb4cda..4f276aca3f2f96 100755 --- a/Tools/i18n/pygettext.py +++ b/Tools/i18n/pygettext.py @@ -131,10 +131,12 @@ appear on a line by itself in the file. -X filename - --exclude-docstrings=filename + --exclude-docstrings= + This is only useful in conjunction with the -D option above. Specify a file that contains a list of files (one per line) that - should not have their docstrings extracted. This is only useful in - conjunction with the -D option above. + should not have their docstrings extracted. fnmatch-style patterns are + supported. + If 'inputfile' is -, standard input is read. """ @@ -716,7 +718,7 @@ class Options: width = 78 excludefilename = '' docstrings = 0 - nodocstrings = [] + nodocstrings = set() comment_tags = set() options = Options() @@ -769,11 +771,10 @@ class Options: elif opt in ('-x', '--exclude-file'): options.excludefilename = arg elif opt in ('-X', '--exclude-docstrings'): - with open(arg, 'r') as nodocstrings_file: + with open(arg) as nodocstrings_file: for line in nodocstrings_file: line = line.strip() - if line and line not in options.nodocstrings: - options.nodocstrings.append(line) + options.nodocstrings.add(line) options.comment_tags = tuple(options.comment_tags)