fix: headings after implicitly-terminated codeblock

Problem:
`block` has no way to terminate just after an implicitly-terminated
`codeblock`, so the next line is always parsed as a plain `line` instead
of a heading (h1/h2/h3). Because headings and `block` both are top-level
forms, this prevents headings from being recognized in this case.

Solution:
Change headings (h1/h2/h3) to be contained by `line`. They are no longer
top-level.

Author: justinmk

README.md

@@ -18,10 +18,13 @@ Overview
 - `block` is the main top-level node which contains `line` nodes.
   - ends at blank line(s) or a line starting with `<`.
 - `line`:
-  - contains atoms (words, tags, taglinks, …).
+  - contains atoms (words, tags, taglinks, …)
   - contains `codeblock` because `>` can start a codeblock at the end of a line.
+  - contains headings (`h1`, `h2`, `h3`) because `codeblock` terminated by
+    "implicit stop" (no terminating `<`) consumes blank lines, so `block` has
+    no way to end.
   - contains `column_heading` because `<` (the `codeblock` terminating char)
-    can appear at the start of a `column_heading`.
+    can appear at the start of `column_heading`.
 - `codeblock`:
   - contains `line` nodes which do not contain `word` nodes, it's just the full
     raw text line including whitespace. This is somewhat dictated by its
@@ -39,8 +42,6 @@ Known issues
 - Spec requires that `codeblock` delimiter ">" must be preceded by a space
   (" >"), not a tab. But currently the grammar doesn't enforce this. Example:
   `:help lcs-tab`.
-- `codeblock` terminated by an "implicit stop" (no terminating `<`) consumes
-  blank lines, preventing top-level forms like `h1` from being recognized.
 - `url` doesn't handle _surrounding_ parens. E.g. `(https://example.com/#yay)` yields `word`
 - `url` doesn't handle _nested_ parens. E.g. `(https://example.com/(foo)#yay)`
 

corpus/codeblock.txt

@@ -93,20 +93,28 @@ text
       (word))))
 
 ================================================================================
-codeblock with implicit stop (FIXME)
+codeblock with implicit stop
 ================================================================================
 >
   line1
   line2
 
--------------------------------
+===============================
 h1-headline *foo*
 line1
 
 line2
 
+>
+  line1
+
 -------------------------------
-h1-headline *foo*
+h2-headline *foo*
+
+>
+  line1
+
+H3 HEADLINE *foo*
 
 --------------------------------------------------------------------------------
 
@@ -118,20 +126,35 @@ h1-headline *foo*
         (line)
         (line)))
     (line
-      (word))
-    (line
-      (word)
-      (tag
-        (word)))
+      (h1
+        (word)
+        (tag
+          (word))))
     (line
       (word)))
   (block
     (line
       (word)))
-  (h2
-    (word)
-    (tag
-      (word))))
+  (block
+    (line
+      (codeblock
+        (line)
+        (line)))
+    (line
+      (h2
+        (word)
+        (tag
+          (word)))))
+  (block
+    (line
+      (codeblock
+        (line)
+        (line)))
+    (line
+      (h3
+        (uppercase_name)
+        (tag
+          (word))))))
 
 ================================================================================
 codeblock with empty lines

corpus/heading1_2.txt

@@ -15,20 +15,24 @@ Text2
 --------------------------------------------------------------------------------
 
 (help_file
-  (h1
-    (word)
-    (tag
-      (word)))
+  (block
+    (line
+      (h1
+        (word)
+        (tag
+          (word)))))
   (block
     (line
       (word)))
-  (h2
-    (word)
-    (word)
-    (tag
-      (word))
-    (tag
-      (word)))
+  (block
+    (line
+      (h2
+        (word)
+        (word)
+        (tag
+          (word))
+        (tag
+          (word)))))
   (block
     (line
       (word))))
@@ -50,19 +54,23 @@ Text
 --------------------------------------------------------------------------------
 
 (help_file
-  (h1
-    (tag
-      (word))
-    (word)
-    (word))
+  (block
+    (line
+      (h1
+        (tag
+          (word))
+        (word)
+        (word))))
   (block
     (line
       (word)))
-  (h2
-    (tag
-      (word))
-    (word)
-    (word))
+  (block
+    (line
+      (h2
+        (tag
+          (word))
+        (word)
+        (word))))
   (block
     (line
       (word))))

corpus/heading3-column_heading.txt

@@ -10,8 +10,10 @@ Text
 --------------------------------------------------------------------------------
 
 (help_file
-  (h3
-    (uppercase_name))
+  (block
+    (line
+      (h3
+        (uppercase_name))))
   (block
     (line
       (word)))
@@ -31,8 +33,10 @@ HELLO WORLD
 --------------------------------------------------------------------------------
 
 (help_file
-  (h3
-    (uppercase_name))
+  (block
+    (line
+      (h3
+        (uppercase_name))))
   (block
     (line
       (word))))
@@ -46,10 +50,12 @@ Test
 --------------------------------------------------------------------------------
 
 (help_file
-  (h3
-    (uppercase_name)
-    (tag
-      (word)))
+  (block
+    (line
+      (h3
+        (uppercase_name)
+        (tag
+          (word)))))
   (block
     (line
       (word))))

corpus/optionlink.txt

@@ -87,12 +87,29 @@ number: '04' 'ISO-10646-1' 'python3'
       (word))))
 
 ================================================================================
-NOT optionlink (FIXME)
+NOT optionlink 2 (FIXME)
 ================================================================================
 
 '{a-z}  `{a-z}		Jump to the mark.
+x
 
 --------------------------------------------------------------------------------
 
 (help_file
   (ERROR))
+
+================================================================================
+NOT optionlink 3 (FIXME)
+================================================================================
+foo '"\ '.  Notice
+|foo|
+
+--------------------------------------------------------------------------------
+
+(help_file
+  (block
+    (line
+      (word)
+      (word
+        (ERROR)
+        (MISSING "'")))))

grammar.js

@@ -18,9 +18,6 @@ module.exports = grammar({
         repeat(/[\t ]*\n/),  // Eat whitespace at top of file.
         repeat(choice(
           $.block,
-          $.h1,
-          $.h2,
-          $.h3,
         )),
         // Last block may not be followed by a blank line or EOL.
         optional(alias($.block_end, $.block)),
@@ -123,15 +120,13 @@ module.exports = grammar({
         token.immediate(field('delimiter', /============+[\t ]*\n/)),
         repeat1($._atom),
         '\n',
-        repeat($._blank),
       ),
 
     h2: ($) =>
       seq(
         token.immediate(field('delimiter', /------------+[\t ]*\n/)),
         repeat1($._atom),
         '\n',
-        repeat($._blank),
       ),
 
     // Heading 3: UPPERCASE NAME, followed by optional *tags*.
@@ -140,7 +135,6 @@ module.exports = grammar({
         field('name', $.uppercase_name),
         repeat($.tag),
         '\n',
-        repeat($._blank),
       ),
 
     tag: ($) => _word($,
@@ -187,6 +181,9 @@ function _line($, require_eol) {
   const eol = require_eol ? '\n' : optional('\n');
   return choice(
     $.column_heading,
+    $.h1,
+    $.h2,
+    $.h3,
     seq(optional($.uppercase_words), repeat($._atom), $.codeblock),
     seq(optional($.uppercase_words), repeat1($._atom), choice($.codeblock, eol)),
   );