Emit chunks of lines instead of individual lines

This should allow us to get smaller diff chunks from Black than by
diffing and analyzing Black's string output for whole files.

Author: akaihola

src/darker/black_diff.py

@@ -49,8 +49,8 @@
 from black.const import DEFAULT_EXCLUDES, DEFAULT_INCLUDES
 from black.files import gen_python_files
 from black.report import Report
-from darker.linewise_black import format_str_to_lines
 
+from darker.linewise_black import format_str_to_chunks
 from darker.utils import TextDocument
 
 if sys.version_info >= (3, 8):
@@ -181,8 +181,8 @@ def run_black(src_contents: TextDocument, black_config: BlackConfig) -> TextDocu
     # https://github.com/psf/black/pull/2484 lands in Black.
     contents_for_black = src_contents.string_with_newline("\n")
     if contents_for_black.strip():
-        dst_lines = format_str_to_lines(contents_for_black, mode=Mode(**mode))
-        dst_contents = "".join(dst_lines)
+        dst_chunks = format_str_to_chunks(contents_for_black, mode=Mode(**mode))
+        dst_contents = "".join(line for chunk in dst_chunks for line in chunk)
     else:
         dst_contents = "\n" if "\n" in src_contents.string else ""
     return TextDocument.from_str(

src/darker/linewise_black.py

@@ -1,22 +1,23 @@
 """Re-implementation of :func:`black.format_str` as a line generator"""
 
-from typing import Generator
-from black import get_future_imports, detect_target_versions, decode_bytes
-from black.lines import Line, EmptyLineTracker
-from black.linegen import transform_line, LineGenerator
+from typing import Generator, List
+
+from black import decode_bytes, detect_target_versions, get_future_imports
 from black.comments import normalize_fmt_off
-from black.mode import Mode
-from black.mode import Feature, supports_feature
+from black.linegen import LineGenerator, transform_line
+from black.lines import EmptyLineTracker, Line
+from black.mode import Feature, Mode, supports_feature
 from black.parsing import lib2to3_parse
 
 
-def format_str_to_lines(
+def format_str_to_chunks(  # pylint: disable=too-many-locals
     src_contents: str, *, mode: Mode
-) -> Generator[str, None, None]:  # pylint: disable=too-many-locals
+) -> Generator[List[str], None, None]:
     """Reformat a string and yield each line of new contents
 
     This is a re-implementation of :func:`black.format_str` modified to be a generator
-    which yields each resulting line instead of concatenating them into a single string.
+    which yields each resulting chunk as a list of lines instead of concatenating them
+    into a single string.
 
     """
     src_node = lib2to3_parse(src_contents.lstrip(), mode.target_versions)
@@ -42,20 +43,22 @@ def format_str_to_lines(
     }
     num_chars = 0
     for current_line in lines.visit(src_node):
-        for _ in range(after):
-            yield empty_line
-        num_chars += after * empty_line_len
+        if after:
+            yield after * [empty_line]
+            num_chars += after * empty_line_len
         before, after = elt.maybe_empty_lines(current_line)
-        for _ in range(before):
-            yield empty_line
-        num_chars += before * empty_line_len
-        for line in transform_line(
-            current_line, mode=mode, features=split_line_features
-        ):
-            line_str = str(line)
-            yield line_str
-            num_chars += len(line_str)
+        if before:
+            yield before * [empty_line]
+            num_chars += before * empty_line_len
+        lines = [
+            str(line)
+            for line in transform_line(
+                current_line, mode=mode, features=split_line_features
+            )
+        ]
+        yield lines
+        num_chars += sum(len(line) for line in lines)
     if not num_chars:
         normalized_content, _, newline = decode_bytes(src_contents.encode("utf-8"))
         if "\n" in normalized_content:
-            yield newline
+            yield [newline]

src/darker/tests/test_black_diff.py

@@ -196,12 +196,12 @@ def test_run_black(encoding, newline):
 def test_run_black_always_uses_unix_newlines(newline):
     """Content is always passed to Black with Unix newlines"""
     src = TextDocument.from_str(f"print ( 'touché' ){newline}")
-    with patch.object(black_diff, "format_str_to_lines") as format_str_to_lines:
-        format_str_to_lines.return_value = ['print("touché")\n']
+    with patch.object(black_diff, "format_str_to_chunks") as format_str_to_chunks:
+        format_str_to_chunks.return_value = [['print("touché")\n']]
 
         _ = run_black(src, BlackConfig())
 
-    format_str_to_lines.assert_called_once_with("print ( 'touché' )\n", mode=ANY)
+    format_str_to_chunks.assert_called_once_with("print ( 'touché' )\n", mode=ANY)
 
 
 def test_run_black_ignores_excludes():

src/darker/tests/test_command_line.py

@@ -470,9 +470,9 @@ def test_black_options_skip_string_normalization(git_repo, config, options, expe
     added_files["main.py"].write_bytes(b"bar")
     mode_class_mock = Mock(wraps=black_diff.Mode)
     # Speed up tests by mocking `format_str` to skip running Black
-    format_str_to_lines = Mock(return_value=["bar"])
+    format_str_to_chunks = Mock(return_value=[["bar"]])
     with patch.multiple(
-        black_diff, Mode=mode_class_mock, format_str_to_lines=format_str_to_lines
+        black_diff, Mode=mode_class_mock, format_str_to_chunks=format_str_to_chunks
     ):
 
         main(options + [str(path) for path in added_files.values()])
@@ -498,9 +498,9 @@ def test_black_options_skip_magic_trailing_comma(git_repo, config, options, expe
     added_files["main.py"].write_bytes(b"a = [1, 2,]")
     mode_class_mock = Mock(wraps=black_diff.Mode)
     # Speed up tests by mocking `format_str` to skip running Black
-    format_str_to_lines = Mock(return_value=["a = [1, 2,]"])
+    format_str_to_chunks = Mock(return_value=[["a = [1, 2,]"]])
     with patch.multiple(
-        black_diff, Mode=mode_class_mock, format_str_to_lines=format_str_to_lines
+        black_diff, Mode=mode_class_mock, format_str_to_chunks=format_str_to_chunks
     ):
 
         main(options + [str(path) for path in added_files.values()])