[markdown mode] improve fencedCodeBlocks and code behavior

Author: dwelle

mode/markdown/markdown.js

@@ -89,7 +89,7 @@ CodeMirror.defineMode("markdown", function(cmCfg, modeCfg) {
   ,   atxHeaderRE = modeCfg.allowAtxHeaderWithoutSpace ? /^(#+)/ : /^(#+)(?: |$)/
   ,   setextHeaderRE = /^ *(?:\={1,}|-{1,})\s*$/
   ,   textRE = /^[^#!\[\]*_\\<>` "'(~:]+/
-  ,   fencedCodeRE = /^(~~~+|```+)[ \t]*([\w+#-]*)/
+  ,   fencedCodeRE = /^(~~~+|```+)[ \t]*([\w+#-]*)[^\n`]*$/
   ,   linkDefRE = /^\s*\[[^\]]+?\]:\s*\S+(\s*\S*\s*)?$/ // naive link-definition
   ,   punctuation = /[!\"#$%&\'()*+,\-\.\/:;<=>?@\[\\\]^_`{|}~—]/
   ,   expandedTab = "    " // CommonMark specifies tab as 4 spaces
@@ -133,13 +133,13 @@ CodeMirror.defineMode("markdown", function(cmCfg, modeCfg) {
     state.trailingSpaceNewLine = false;
     // Mark this line as blank
     state.prevLine = state.thisLine
-    state.thisLine = null
+    state.thisLine = {stream: null}
     return null;
   }
 
   function blockNormal(stream, state) {
     var firstTokenOnLine = stream.column() === state.indentation;
-    var prevLineLineIsEmpty = lineIsEmpty(state.prevLine);
+    var prevLineLineIsEmpty = lineIsEmpty(state.prevLine.stream);
     var prevLineIsIndentedCode = state.indentedCode;
     var prevLineIsHr = state.hr;
     var prevLineIsList = state.list !== false;
@@ -176,7 +176,8 @@ CodeMirror.defineMode("markdown", function(cmCfg, modeCfg) {
       state.indentation <= maxNonCodeIndentation && stream.match(hrRE);
 
     var match = null;
-    if (state.indentationDiff >= 4 && (prevLineIsIndentedCode || prevLineLineIsEmpty)) {
+    if (state.indentationDiff >= 4 && (prevLineIsIndentedCode || state.prevLine.fencedCodeEnd ||
+         state.prevLine.header || prevLineLineIsEmpty)) {
       stream.skipToEnd();
       state.indentedCode = true;
       return tokenTypes.code;
@@ -185,6 +186,7 @@ CodeMirror.defineMode("markdown", function(cmCfg, modeCfg) {
     } else if (firstTokenOnLine && state.indentation <= maxNonCodeIndentation && (match = stream.match(atxHeaderRE)) && match[1].length <= 6) {
       state.quote = 0;
       state.header = match[1].length;
+      state.thisLine.header = true;
       if (modeCfg.highlightFormatting) state.formatting = "header";
       state.f = state.inline;
       return getType(state);
@@ -211,7 +213,7 @@ CodeMirror.defineMode("markdown", function(cmCfg, modeCfg) {
       return getType(state);
     } else if (firstTokenOnLine && state.indentation <= maxNonCodeIndentation && (match = stream.match(fencedCodeRE, true))) {
       state.quote = 0;
-      state.fencedChars = match[1]
+      state.fencedEndRE = new RegExp(match[1] + "+ *$");
       // try switching mode
       state.localMode = modeCfg.fencedCodeBlockHighlighting && getMode(match[2]);
       if (state.localMode) state.localState = CodeMirror.startState(state.localMode);
@@ -240,6 +242,7 @@ CodeMirror.defineMode("markdown", function(cmCfg, modeCfg) {
         stream.skipToEnd();
         if (modeCfg.highlightFormatting) state.formatting = "header";
       }
+      state.thisLine.header = true;
       state.f = state.inline;
       return getType(state);
     } else if (isHr) {
@@ -269,20 +272,21 @@ CodeMirror.defineMode("markdown", function(cmCfg, modeCfg) {
   }
 
   function local(stream, state) {
-    var hasExitedList = state.indentation < state.listStack[state.listStack.length - 1];
-    if (state.fencedChars && (hasExitedList || stream.match(state.fencedChars))) {
+    var currListInd = state.listStack[state.listStack.length - 1] || 0;
+    var hasExitedList = state.indentation < currListInd;
+    var maxFencedEndInd = currListInd + 3;
+    if (state.fencedEndRE && state.indentation <= maxFencedEndInd && (hasExitedList || stream.match(state.fencedEndRE))) {
       if (modeCfg.highlightFormatting) state.formatting = "code-block";
       var returnType;
       if (!hasExitedList) returnType = getType(state)
       state.localMode = state.localState = null;
       state.block = blockNormal;
       state.f = inlineNormal;
-      state.fencedChars = null;
+      state.fencedEndRE = null;
       state.code = 0
+      state.thisLine.fencedCodeEnd = true;
       if (hasExitedList) return switchBlock(stream, state, state.block);
       return returnType;
-    } else if (state.fencedChars && stream.skipTo(state.fencedChars)) {
-      return "comment"
     } else if (state.localMode) {
       return state.localMode.token(stream, state.localState);
     } else {
@@ -716,8 +720,8 @@ CodeMirror.defineMode("markdown", function(cmCfg, modeCfg) {
       return {
         f: blockNormal,
 
-        prevLine: null,
-        thisLine: null,
+        prevLine: {stream: null},
+        thisLine: {stream: null},
 
         block: blockNormal,
         htmlState: null,
@@ -744,7 +748,7 @@ CodeMirror.defineMode("markdown", function(cmCfg, modeCfg) {
         trailingSpaceNewLine: false,
         strikethrough: false,
         emoji: false,
-        fencedChars: null
+        fencedEndRE: null
       };
     },
 
@@ -783,7 +787,7 @@ CodeMirror.defineMode("markdown", function(cmCfg, modeCfg) {
         trailingSpace: s.trailingSpace,
         trailingSpaceNewLine: s.trailingSpaceNewLine,
         md_inside: s.md_inside,
-        fencedChars: s.fencedChars
+        fencedEndRE: s.fencedEndRE
       };
     },
 
@@ -792,8 +796,8 @@ CodeMirror.defineMode("markdown", function(cmCfg, modeCfg) {
       // Reset state.formatting
       state.formatting = false;
 
-      if (stream != state.thisLine) {
         // Reset state.header
+      if (stream != state.thisLine.stream) {
         state.header = 0;
 
         if (stream.match(/^\s*$/, true)) {
@@ -802,7 +806,7 @@ CodeMirror.defineMode("markdown", function(cmCfg, modeCfg) {
         }
 
         state.prevLine = state.thisLine
-        state.thisLine = stream
+        state.thisLine = {stream: stream}
 
         // Reset state.taskList
         state.taskList = false;

mode/markdown/test.js

@@ -151,6 +151,21 @@
      "Foo",
      "    Bar");
 
+  MT("codeBlocksAfterATX",
+     "[header&header-1 # foo]",
+     "    [comment code]");
+
+  MT("codeBlocksAfterSetext",
+     "[header&header-2 foo]",
+     "[header&header-2 ---]",
+     "    [comment code]");
+
+  MT("codeBlocksAfterFencedCode",
+     "[comment ```]",
+     "[comment foo]",
+     "[comment ```]",
+     "    [comment code]");
+
   // Inline code using backticks
   MT("inlineCodeUsingBackticks",
      "foo [comment `bar`]");
@@ -192,6 +207,10 @@
   MT("closedBackticks",
      "[comment ``foo ``` bar` hello``] world");
 
+  // info string cannot contain backtick, thus should result in inline code
+  MT("closingFencedMarksOnSameLine",
+     "[comment ``` code ```] foo");
+
   // atx headers
   // http://daringfireball.net/projects/markdown/syntax#header
 
@@ -237,7 +256,7 @@
 
   MT("atxIndentedTooMuch",
      "[header&header-1 # foo]",
-     "    # bar");
+     "    [comment # bar]");
 
   // disable atx inside blockquote until we implement proper blockquote inner mode
   // TODO: fix to be CommonMark-compliant
@@ -1100,6 +1119,39 @@
      "[comment ```]",
      "baz");
 
+  MT("fencedCodeBlocks_invalidClosingFence_trailingText",
+     "[comment ```]",
+     "[comment foo]",
+     "[comment ``` must not have trailing text]",
+     "[comment baz]");
+
+  MT("fencedCodeBlocks_invalidClosingFence_trailingTabs",
+     "[comment ```]",
+     "[comment foo]",
+     "[comment ```\t]",
+     "[comment baz]");
+
+  MT("fencedCodeBlocks_validClosingFence",
+     "[comment ```]",
+     "[comment foo]",
+     // may have trailing spaces
+     "[comment ```     ]",
+     "baz");
+
+  MT("fencedCodeBlocksInList_closingFenceIndented",
+     "[variable-2 - list]",
+     "    [variable-2&comment ```]",
+     "    [comment foo]",
+     "     [variable-2&comment ```]",
+     "    [variable-2 baz]");
+
+  MT("fencedCodeBlocksInList_closingFenceIndentedTooMuch",
+     "[variable-2 - list]",
+     "    [variable-2&comment ```]",
+     "    [comment foo]",
+     "      [comment ```]",
+     "    [comment baz]");
+
   MT("fencedCodeBlockModeSwitching",
      "[comment ```javascript]",
      "[variable foo]",