remark: rework list fields

* remark-parse: remove `loose` fields from `list`, `listItem`
* remark-parse: add support for empty list-items in any non-pedantic mode
* remark-parse: add `spread` to `list`, to signal that its items should have blank
  lines between them, set to true if any blank line exists between items
* remark-parse: add `spread` to `listItem`, to signal that its children should have
  blank lines between them
* remark-parse: fix bug where two lists, if they are both ordered or both not
  ordered, was seen in commonmark as one list
* remark-stringify: add support for missing `ordered` field
* remark-stringify: add support for missing `start` field
* remark-stringify: add support for `list.spread` (default: false)
* remark-stringify: add support for `listItem.spread` (default: true)
* remark-stringify: fix bug where two lists, if they are both ordered or both not
  ordered, and in commonmark, were not properly stringified, resulting in a
  new parse seeing them as one list
* remark-stringify: fix bug where a list followed by an indented code block, in
  commonmark, was not correctly stringified, resulting in a new parse seeing
  the code as content of the list
* remark-stringify: remove trailing white-space on empty list-items
* remark-stringify: add support for stringifying list-items without parent

Related to syntax-tree/mdast#4.
Related to syntax-tree/mdast-util-to-hast#23.
Related to GH-349.

Closes GH-332.
Closes GH-350.
Closes GH-358.
Closes GH-364.

Reviewed-by: Elijah Hamovitz <elijahhamovitz@gmail.com>
Reviewed-by: Ika <ikatyang@gmail.com>

Author: wooorm

contributing.md

@@ -15,6 +15,7 @@ next to their day job: you are not entitled to free customer service.
 *   [Ecosystem](#ecosystem)
 *   [Questions](#questions)
 *   [Contributions](#contributions)
+
     *   [Improve documentation](#improve-documentation)
     *   [Improve issues](#improve-issues)
     *   [Give feedback on issues](#give-feedback-on-issues)

packages/remark-parse/lib/tokenize/list.js

@@ -80,7 +80,7 @@ function list(eat, value, silent) {
   var item;
   var enterTop;
   var exitBlockquote;
-  var isLoose;
+  var spread = false;
   var node;
   var now;
   var end;
@@ -143,7 +143,7 @@ function list(eat, value, silent) {
   if (
     character !== C_SPACE &&
     character !== C_TAB &&
-    (!commonmark || (character !== C_NEWLINE && character !== C_EMPTY))
+    (pedantic || (character !== C_NEWLINE && character !== C_EMPTY))
   ) {
     return;
   }
@@ -282,14 +282,15 @@ function list(eat, value, silent) {
     }
 
     prevEmpty = empty;
-    empty = !trim(content).length;
+    empty = !prefixed && !trim(content).length;
 
     if (indented && item) {
       item.value = item.value.concat(emptyLines, line);
       allLines = allLines.concat(emptyLines, line);
       emptyLines = [];
     } else if (prefixed) {
       if (emptyLines.length !== 0) {
+        spread = true;
         item.value.push('');
         item.trail = emptyLines.concat();
       }
@@ -304,7 +305,7 @@ function list(eat, value, silent) {
       allLines = allLines.concat(emptyLines, line);
       emptyLines = [];
     } else if (empty) {
-      if (prevEmpty) {
+      if (prevEmpty && !commonmark) {
         break;
       }
 
@@ -330,25 +331,20 @@ function list(eat, value, silent) {
     type: 'list',
     ordered: ordered,
     start: start,
-    loose: null,
+    spread: spread,
     children: []
   });
 
   enterTop = self.enterList();
   exitBlockquote = self.enterBlock();
-  isLoose = false;
   index = -1;
   length = items.length;
 
   while (++index < length) {
     item = items[index].value.join(C_NEWLINE);
     now = eat.now();
 
-    item = eat(item)(listItem(self, item, now), node);
-
-    if (item.loose) {
-      isLoose = true;
-    }
+    eat(item)(listItem(self, item, now), node);
 
     item = items[index].trail.join(C_NEWLINE);
 
@@ -362,8 +358,6 @@ function list(eat, value, silent) {
   enterTop();
   exitBlockquote();
 
-  node.loose = isLoose;
-
   return node;
 }
 
@@ -389,8 +383,7 @@ function listItem(ctx, value, position) {
 
   return {
     type: 'listItem',
-    loose: EXPRESSION_LOOSE_LIST_ITEM.test(value) ||
-      value.charAt(value.length - 1) === C_NEWLINE,
+    spread: EXPRESSION_LOOSE_LIST_ITEM.test(value),
     checked: checked,
     children: ctx.tokenizeBlock(value, position)
   };

packages/remark-parse/readme.md

@@ -33,9 +33,11 @@ process.stdin
 ## Table of Contents
 
 *   [API](#api)
+
     *   [processor.use(parse\[, options\])](#processoruseparse-options)
     *   [parse.Parser](#parseparser)
 *   [Extending the Parser](#extending-the-parser)
+
     *   [Parser#blockTokenizers](#parserblocktokenizers)
     *   [Parser#blockMethods](#parserblockmethods)
     *   [Parser#inlineTokenizers](#parserinlinetokenizers)

packages/remark-stringify/lib/macro/block.js

@@ -2,43 +2,55 @@
 
 module.exports = block;
 
+var newline = '\n';
+var double = newline + newline;
+var triple = double + newline;
+var comment = double + '<!---->' + double;
+
 /* Stringify a block node with block children (e.g., `root`
  * or `blockquote`).
  * Knows about code following a list, or adjacent lists
  * with similar bullets, and places an extra newline
  * between them. */
 function block(node) {
   var self = this;
+  var options = self.options;
+  var fences = options.fences;
+  var gap = options.commonmark ? comment : triple;
   var values = [];
   var children = node.children;
   var length = children.length;
   var index = -1;
-  var child;
   var prev;
+  var child;
 
   while (++index < length) {
+    prev = child;
     child = children[index];
 
     if (prev) {
-      /* Duplicate nodes, such as a list
-       * directly following another list,
-       * often need multiple new lines.
-       *
-       * Additionally, code blocks following a list
-       * might easily be mistaken for a paragraph
-       * in the list itself. */
-      if (child.type === prev.type && prev.type === 'list') {
-        values.push(prev.ordered === child.ordered ? '\n\n\n' : '\n\n');
-      } else if (prev.type === 'list' && child.type === 'code' && !child.lang) {
-        values.push('\n\n\n');
+      // A list preceding another list that are equally ordered, or a
+      // list preceding an indented code block, need a gap between them,
+      // so as not to see them as one list, or content of the list,
+      // respectively.
+      //
+      // In commonmark, only something that breaks both up can do that,
+      // so we opt for an empty, invisible comment.  In other flavours,
+      // two blank lines are fine.
+      if (
+        prev.type === 'list' &&
+        (
+          (child.type === 'list' && prev.ordered === child.ordered) ||
+          (child.type === 'code' && (!child.lang && !fences))
+        )
+      ) {
+        values.push(gap);
       } else {
-        values.push('\n\n');
+        values.push(double);
       }
     }
 
     values.push(self.visit(child, node));
-
-    prev = child;
   }
 
   return values.join('');

packages/remark-stringify/lib/macro/ordered-items.js

@@ -28,10 +28,12 @@ function orderedItems(node) {
   var index = -1;
   var bullet;
 
+  start = start == null ? 1 : start;
+
   while (++index < length) {
     bullet = (increment ? start + index : start) + '.';
     values[index] = fn.call(self, children[index], node, index, bullet);
   }
 
-  return values.join('\n');
+  return values.join(node.spread ? '\n\n' : '\n');
 }

packages/remark-stringify/lib/macro/unordered-items.js

@@ -18,5 +18,5 @@ function unorderedItems(node) {
     values[index] = fn.call(self, children[index], node, index, bullet);
   }
 
-  return values.join('\n');
+  return values.join(node.spread ? '\n\n' : '\n');
 }

packages/remark-stringify/lib/visitors/list-item.js

@@ -5,14 +5,6 @@ var pad = require('../util/pad');
 
 module.exports = listItem;
 
-/* Which checkbox to use. */
-var CHECKBOX_MAP = {
-  undefined: '',
-  null: '',
-  true: '[x] ',
-  false: '[ ] '
-};
-
 /* Stringify a list item.
  *
  * Prefixes the content with a checked checkbox when
@@ -28,7 +20,9 @@ var CHECKBOX_MAP = {
 function listItem(node, parent, position, bullet) {
   var self = this;
   var style = self.options.listItemIndent;
-  var loose = node.loose;
+  var marker = bullet || self.options.bullet;
+  var spread = node.spread == null ? true : node.spread;
+  var checked = node.checked;
   var children = node.children;
   var length = children.length;
   var values = [];
@@ -41,21 +35,22 @@ function listItem(node, parent, position, bullet) {
     values[index] = self.visit(children[index], node);
   }
 
-  value = CHECKBOX_MAP[node.checked] + values.join(loose ? '\n\n' : '\n');
+  value = values.join(spread ? '\n\n' : '\n');
+
+  if (typeof checked === 'boolean') {
+    // Note: I’d like to be able to only add the space between the check and
+    // the value, but unfortunately github does not support empty list-items
+    // with a checkbox :(
+    value = '[' + (checked ? 'x' : ' ') + '] ' + value;
+  }
 
   if (style === '1' || (style === 'mixed' && value.indexOf('\n') === -1)) {
-    indent = bullet.length + 1;
+    indent = marker.length + 1;
     spacing = ' ';
   } else {
-    indent = Math.ceil((bullet.length + 1) / 4) * 4;
-    spacing = repeat(' ', indent - bullet.length);
-  }
-
-  value = bullet + spacing + pad(value, indent / 4).slice(indent);
-
-  if (loose && parent.children.length - 1 !== position) {
-    value += '\n';
+    indent = Math.ceil((marker.length + 1) / 4) * 4;
+    spacing = repeat(' ', indent - marker.length);
   }
 
-  return value;
+  return value ? marker + spacing + pad(value, indent / 4).slice(indent) : marker;
 }

packages/remark-stringify/lib/visitors/list.js

@@ -9,5 +9,6 @@ var ORDERED_MAP = {
 };
 
 function list(node) {
-  return this[ORDERED_MAP[node.ordered]](node);
+  var ordered = node.ordered;
+  return this[ORDERED_MAP[ordered == null ? false : ordered]](node);
 }

packages/remark-stringify/readme.md

@@ -40,9 +40,11 @@ process.stdin
 ## Table of Contents
 
 *   [API](#api)
+
     *   [processor.use(stringify\[, options\])](#processorusestringify-options)
     *   [stringify.Compiler](#stringifycompiler)
 *   [Extending the Compiler](#extending-the-compiler)
+
     *   [Compiler#visitors](#compilervisitors)
     *   [function visitor(node\[, parent\])](#function-visitornode-parent)
 *   [License](#license)

test/fixtures/input/list-item-empty-without-white-space-eof.text

@@ -0,0 +1 @@
+-

test/fixtures/input/list-item-empty-without-white-space.text

@@ -0,0 +1 @@
+-

test/fixtures/input/ordered-different-types.text renamed to test/fixtures/input/ordered-different-types.nooutput.text

@@ -344,11 +344,11 @@
       "type": "list",
       "ordered": false,
       "start": null,
-      "loose": false,
+      "spread": false,
       "children": [
         {
           "type": "listItem",
-          "loose": false,
+          "spread": false,
           "checked": null,
           "children": [
             {
@@ -403,7 +403,7 @@
         },
         {
           "type": "listItem",
-          "loose": false,
+          "spread": false,
           "checked": null,
           "children": [
             {
@@ -478,7 +478,7 @@
         },
         {
           "type": "listItem",
-          "loose": false,
+          "spread": false,
           "checked": null,
           "children": [
             {