feat(changelog): add incremental flag

woile · woile · commit bfd814e1ba5c · 2020-05-02T10:51:14.000+02:00
diff --git a/commitizen/changelog.py b/commitizen/changelog.py
@@ -20,6 +20,8 @@
 Extra:
 - Generate full or partial changelog
 - Include in tree from file all the extra comments added manually
+- hook after message is parsed (add extra information like hyperlinks)
+- hook after changelog is generated (api calls)
 """
 import re
 from collections import defaultdict
@@ -29,7 +31,7 @@
 from jinja2 import Template
 
 from commitizen import defaults
-from commitizen.git import GitCommit, GitProtocol, GitTag
+from commitizen.git import GitCommit, GitTag
 
 CATEGORIES = [
     ("fix", "fix"),
@@ -55,15 +57,9 @@ def transform_change_type(change_type: str) -> str:
         raise ValueError(f"Could not match a change_type with {change_type}")
 
 
-def get_commit_tag(commit: GitProtocol, tags: List[GitProtocol]) -> Optional[GitTag]:
+def get_commit_tag(commit: GitCommit, tags: List[GitTag]) -> Optional[GitTag]:
     """"""
-    try:
-        tag_index = tags.index(commit)
-    except ValueError:
-        return None
-    else:
-        tag = tags[tag_index]
-        return tag
+    return next((tag for tag in tags if tag.rev == commit.rev), None)
 
 
 def generate_tree_from_commits(
@@ -109,6 +105,7 @@ def generate_tree_from_commits(
         message = map_pat.match(commit.message)
         message_body = map_pat.match(commit.body)
         if message:
+            # TODO: add a post hook coming from a rule (CzBase)
             parsed_message: Dict = message.groupdict()
             # change_type becomes optional by providing None
             change_type = parsed_message.pop("change_type", None)
@@ -132,3 +129,109 @@ def render_changelog(tree: Iterable) -> str:
     jinja_template = Template(template_file, trim_blocks=True)
     changelog: str = jinja_template.render(tree=tree)
     return changelog
+
+
+def parse_version_from_markdown(value: str) -> Optional[str]:
+    if not value.startswith("#"):
+        return None
+    m = re.search(defaults.version_parser, value)
+    if not m:
+        return None
+    return m.groupdict().get("version")
+
+
+def parse_title_type_of_line(value: str) -> Optional[str]:
+    md_title_parser = r"^(?P<title>#+)"
+    m = re.search(md_title_parser, value)
+    if not m:
+        return None
+    return m.groupdict().get("title")
+
+
+def get_metadata(filepath: str) -> Dict:
+    unreleased_start: Optional[int] = None
+    unreleased_end: Optional[int] = None
+    unreleased_title: Optional[str] = None
+    latest_version: Optional[str] = None
+    latest_version_position: Optional[int] = None
+    with open(filepath, "r") as changelog_file:
+        for index, line in enumerate(changelog_file):
+            line = line.strip().lower()
+
+            unreleased: Optional[str] = None
+            if "unreleased" in line:
+                unreleased = parse_title_type_of_line(line)
+            # Try to find beginning and end lines of the unreleased block
+            if unreleased:
+                unreleased_start = index
+                unreleased_title = unreleased
+                continue
+            elif (
+                isinstance(unreleased_title, str)
+                and parse_title_type_of_line(line) == unreleased_title
+            ):
+                unreleased_end = index
+
+            # Try to find the latest release done
+            version = parse_version_from_markdown(line)
+            if version:
+                latest_version = version
+                latest_version_position = index
+                break  # there's no need for more info
+        if unreleased_start is not None and unreleased_end is None:
+            unreleased_end = index
+    return {
+        "unreleased_start": unreleased_start,
+        "unreleased_end": unreleased_end,
+        "latest_version": latest_version,
+        "latest_version_position": latest_version_position,
+    }
+
+
+def incremental_build(new_content: str, lines: List, metadata: Dict) -> List:
+    """Takes the original lines and updates with new_content.
+
+    The metadata holds information enough to remove the old unreleased and
+    where to place the new content
+
+    Arguments:
+        lines -- the lines from the changelog
+        new_content -- this should be placed somewhere in the lines
+        metadata -- information about the changelog
+
+    Returns:
+        List -- updated lines
+    """
+    unreleased_start = metadata.get("unreleased_start")
+    unreleased_end = metadata.get("unreleased_end")
+    latest_version_position = metadata.get("latest_version_position")
+    skip = False
+    output_lines: List = []
+    for index, line in enumerate(lines):
+        if index == unreleased_start:
+            skip = True
+        elif index == unreleased_end:
+            skip = False
+            if (
+                latest_version_position is None
+                or isinstance(latest_version_position, int)
+                and isinstance(unreleased_end, int)
+                and latest_version_position > unreleased_end
+            ):
+                continue
+
+        if skip:
+            continue
+
+        if (
+            isinstance(latest_version_position, int)
+            and index == latest_version_position
+        ):
+
+            output_lines.append(new_content)
+            output_lines.append("\n")
+
+        output_lines.append(line)
+    if not isinstance(latest_version_position, int):
+        output_lines.append(new_content)
+    return output_lines
diff --git a/commitizen/cli.py b/commitizen/cli.py
@@ -131,6 +131,10 @@
                         "default": False,
                         "help": "show changelog to stdout",
                     },
+                    {
+                        "name": "--file-name",
+                        "help": "file name of changelog (default: 'CHANGELOG.md')",
+                    },
                     {
                         "name": "--incremental",
                         "action": "store_true",
@@ -140,10 +144,6 @@
                             "useful if the changelog has been manually modified"
                         ),
                     },
-                    {
-                        "name": "--file-name",
-                        "help": "file name of changelog (default: 'CHANGELOG.md')",
-                    },
                     {
                         "name": "--start-rev",
                         "default": None,
diff --git a/commitizen/commands/changelog.py b/commitizen/commands/changelog.py
@@ -1,12 +1,16 @@
-import re
-from collections import OrderedDict
-
-import pkg_resources
-from jinja2 import Template
+import os.path
+from difflib import SequenceMatcher
+from operator import itemgetter
+from typing import Dict, List
 
 from commitizen import changelog, factory, git, out
 from commitizen.config import BaseConfig
-from commitizen.error_codes import NO_COMMITS_FOUND, NO_PATTERN_MAP, TAG_FAILED
+from commitizen.error_codes import NO_COMMITS_FOUND, NO_PATTERN_MAP, NO_REVISION
+from commitizen.git import GitTag
+
+
+def similar(a, b):
+    return SequenceMatcher(None, a, b).ratio()
 
 
 class Changelog:
@@ -21,69 +25,78 @@ def __init__(self, config: BaseConfig, args):
         self.dry_run = args["dry_run"]
         self.start_rev = args["start_rev"]
 
+    def _find_incremental_rev(self, latest_version: str, tags: List[GitTag]) -> str:
+        """Try to find the 'start_rev'.
+
+        We use a similarity approach. We know how to parse the version from the markdown
+        changelog, but not the whole tag, we don't even know how's the tag made.
+
+        This 'smart' function tries to find a similarity between the found version number
+        and the available tag.
+
+        The SIMILARITY_THRESHOLD is an empirical value, it may have to be adjusted based
+        on our experience.
+        """
+        SIMILARITY_THRESHOLD = 0.89
+        tag_ratio = map(
+            lambda tag: (SequenceMatcher(None, latest_version, tag.name).ratio(), tag),
+            tags,
+        )
+        try:
+            score, tag = max(tag_ratio, key=itemgetter(0))
+        except ValueError:
+            raise SystemExit(NO_REVISION)
+        if score < SIMILARITY_THRESHOLD:
+            raise SystemExit(NO_REVISION)
+        start_rev = tag.name
+        return start_rev
+
     def __call__(self):
-        # changelog_map = self.cz.changelog_map
         commit_parser = self.cz.commit_parser
         changelog_pattern = self.cz.changelog_pattern
+        start_rev = self.start_rev
+        changelog_meta: Dict = {}
 
         if not changelog_pattern or not commit_parser:
             out.error(
                 f"'{self.config.settings['name']}' rule does not support changelog"
             )
             raise SystemExit(NO_PATTERN_MAP)
-        # pat = re.compile(changelog_pattern)
-
-        commits = git.get_commits(start=self.start_rev)
-        if not commits:
-            out.error("No commits found")
-            raise SystemExit(NO_COMMITS_FOUND)
 
         tags = git.get_tags()
         if not tags:
             tags = []
 
+        if self.incremental:
+            changelog_meta = changelog.get_metadata(self.file_name)
+            latest_version = changelog_meta.get("latest_version")
+            if latest_version:
+                start_rev = self._find_incremental_rev(latest_version, tags)
+
+        commits = git.get_commits(start=start_rev)
+        if not commits:
+            out.error("No commits found")
+            raise SystemExit(NO_COMMITS_FOUND)
+
         tree = changelog.generate_tree_from_commits(
             commits, tags, commit_parser, changelog_pattern
         )
         changelog_out = changelog.render_changelog(tree)
-        # tag_map = {tag.rev: tag.name for tag in git.get_tags()}
-
-        # entries = OrderedDict()
-        # # The latest commit is not tagged
-        # latest_commit = commits[0]
-        # if latest_commit.rev not in tag_map:
-        #     current_key = "Unreleased"
-        #     entries[current_key] = OrderedDict(
-        #         {value: [] for value in changelog_map.values()}
-        #     )
-        # else:
-        #     current_key = tag_map[latest_commit.rev]
-
-        # for commit in commits:
-        #     if commit.rev in tag_map:
-        #         current_key = tag_map[commit.rev]
-        #         entries[current_key] = OrderedDict(
-        #             {value: [] for value in changelog_map.values()}
-        #         )
-
-        #     matches = pat.match(commit.message)
-        #     if not matches:
-        #         continue
-
-        #     processed_commit = self.cz.process_commit(commit.message)
-        #     for group_name, commit_type in changelog_map.items():
-        #         if matches.group(group_name):
-        #             entries[current_key][commit_type].append(processed_commit)
-        #             break
-
-        # template_file = pkg_resources.resource_string(
-        #     __name__, "../templates/keep_a_changelog_template.j2"
-        # ).decode("utf-8")
-        # jinja_template = Template(template_file)
-        # changelog_str = jinja_template.render(entries=entries)
+
         if self.dry_run:
             out.write(changelog_out)
             raise SystemExit(0)
 
+        lines = []
+        if self.incremental and os.path.isfile(self.file_name):
+            with open(self.file_name, "r") as changelog_file:
+                lines = changelog_file.readlines()
+
         with open(self.file_name, "w") as changelog_file:
-            changelog_file.write(changelog_out)
+            if self.incremental:
+                new_lines = changelog.incremental_build(
+                    changelog_out, lines, changelog_meta
+                )
+                changelog_file.writelines(new_lines)
+            else:
+                changelog_file.write(changelog_out)
diff --git a/commitizen/defaults.py b/commitizen/defaults.py
@@ -34,3 +34,4 @@
 bump_message = "bump: version $current_version → $new_version"
 
 commit_parser = r"^(?P<change_type>feat|fix|refactor|perf|BREAKING CHANGE)(?:\((?P<scope>[^()\r\n]*)\)|\()?(?P<breaking>!)?:\s(?P<message>.*)?"  # noqa
+version_parser = r"(?P<version>([0-9]+)\.([0-9]+)\.([0-9]+)(?:-([0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*))?(?:\+[0-9A-Za-z-]+)?)"
diff --git a/commitizen/error_codes.py b/commitizen/error_codes.py
@@ -22,3 +22,6 @@
 # Check
 NO_COMMIT_MSG = 13
 INVALID_COMMIT_MSG = 14
+
+# Changelog
+NO_REVISION = 16
diff --git a/commitizen/git.py b/commitizen/git.py
@@ -15,6 +15,8 @@ class GitProtocol(Protocol):
 
 class GitObject:
     rev: str
+    name: str
+    date: str
 
     def __eq__(self, other) -> bool:
         if not hasattr(other, "rev"):
diff --git a/docs/changelog.md b/docs/changelog.md
@@ -14,6 +14,16 @@ cz changelog
 cz ch
 ```
 
+It has support for incremental changelog:
+
+- Build from latest version found in changelog, this is useful if you have a different changelog and want to use chommitizen
+- Update unreleased area
+- Allows users to manually touch the changelog without being rewritten.
+
+## Constrains
+
+At the moment this features is constrained only to markdown files.
+
 ## Description
 
 These are the variables used by the changelog generator.
@@ -26,7 +36,7 @@ These are the variables used by the changelog generator.
 - **<scope>**: <message>
 ```
 
-It will create a full of the above block per version found in the tags.
+It will create a full block like above per version found in the tags.
 And it will create a list of the commits found.
 The `change_type` and the `scope` are optional, they don't need to be provided,
 but if your regex does they will be rendered.
@@ -45,5 +55,10 @@ and the following variables are expected:
 
 - **required**: is the only one required to be parsed by the regex
 
+## TODO
+
+- [ ] support for hooks: this would allow introduction of custom information in the commiter, like a github or jira url. Eventually we could build a `CzConventionalGithub`, which would add links to commits
+- [ ] support for map: allow the usage of a `change_type` mapper, to convert from feat to feature for example.
+
 [keepachangelog]: https://keepachangelog.com/
 [semver]: https://semver.org/
diff --git a/pyproject.toml b/pyproject.toml
@@ -66,6 +66,7 @@ mypy = "^0.770"
 mkdocs = "^1.0"
 mkdocs-material = "^4.1"
 isort = "^4.3.21"
+freezegun = "^0.3.15"
 
 [tool.poetry.scripts]
 cz = "commitizen.cli:main"
diff --git a/tests/commands/test_changelog_command.py b/tests/commands/test_changelog_command.py
diff --git a/tests/test_changelog_meta.py b/tests/test_changelog_meta.py
diff --git a/tests/test_changelog_parser.py b/tests/test_changelog_parser.py

Original file line number	Diff line number	Diff line change
`@@ -34,3 +34,4 @@`
`34`	`34`	`bump_message = "bump: version $current_version → $new_version"`
`35`	`35`
`36`	`36`	`commit_parser = r"^(?P<change_type>feat\|fix\|refactor\|perf\|BREAKING CHANGE)(?:\((?P<scope>[^()\r\n])\)\|\()?(?P<breaking>!)?:\s(?P<message>.)?" # noqa`
	`37`	`+version_parser = r"(?P<version>([0-9]+)\.([0-9]+)\.([0-9]+)(?:-([0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*))?(?:\+[0-9A-Za-z-]+)?)"`