Add a baseToken method to string streams

That overlay modes can use to access the underlying token info.

Author: marijnh

doc/manual.html

@@ -3261,6 +3261,12 @@ <h2>Writing CodeMirror Modes</h2>
       one, in order to scan ahead across line boundaries. Note that
       you want to do this carefully, since looking far ahead will make
       mode state caching much less effective.</dd>
+
+      <dt><code><strong>baseToken</strong>() → ?{type: ?string, size: number}</code></dt>
+      <dd>Modes added
+      through <a href="#addOverlay"><code>addOverlay</code></a>
+      (and <em>only</em> such modes) can use this method to inspect
+      the current token produced by the underlying mode.</dd>
     </dl>
 
     <p id="blankLine">By default, blank lines are simply skipped when

src/line/highlight.js

@@ -18,6 +18,8 @@ class Context {
     this.doc = doc
     this.line = line
     this.maxLookAhead = lookAhead || 0
+    this.baseTokens = null
+    this.baseTokenPos = 1
   }
 
   lookAhead(n) {
@@ -26,6 +28,15 @@ class Context {
     return line
   }
 
+  baseToken(n) {
+    if (!this.baseTokens) return null
+    while (this.baseTokens[this.baseTokenPos] >= n)
+      this.baseTokenPos += 2
+    let type = this.baseTokens[this.baseTokenPos + 1]
+    return {type: type && type.replace(/( |^)overlay .*/, ""),
+            size: this.baseTokens[this.baseTokenPos] - n}
+  }
+
   nextLine() {
     this.line++
     if (this.maxLookAhead > 0) this.maxLookAhead--
@@ -60,6 +71,7 @@ export function highlightLine(cm, line, context, forceToEnd) {
 
   // Run overlays, adjust style array.
   for (let o = 0; o < cm.state.overlays.length; ++o) {
+    context.baseTokens = st
     let overlay = cm.state.overlays[o], i = 1, at = 0
     context.state = true
     runMode(cm, line.text, overlay.mode, context, (end, style) => {
@@ -83,8 +95,10 @@ export function highlightLine(cm, line, context, forceToEnd) {
         }
       }
     }, lineClasses)
+    context.state = state
+    context.baseTokens = null
+    context.baseTokenPos = 1
   }
-  context.state = state
 
   return {styles: st, classes: lineClasses.bgClass || lineClasses.textClass ? lineClasses : null}
 }

src/util/StringStream.js

@@ -81,6 +81,10 @@ class StringStream {
     let oracle = this.lineOracle
     return oracle && oracle.lookAhead(n)
   }
+  baseToken() {
+    let oracle = this.lineOracle
+    return oracle && oracle.baseToken(this.pos)
+  }
 }
 
 export default StringStream

test/test.js

@@ -2218,6 +2218,21 @@ testCM("getTokenTypeAt", function(cm) {
   eq(cm.getTokenTypeAt(Pos(0, 6)), "string");
 }, {value: "1 + 'foo'", mode: "javascript"});
 
+testCM("addOverlay", function(cm) {
+  cm.addOverlay({
+    token: function(stream) {
+      var base = stream.baseToken()
+      if (!/comment/.test(base.type) && stream.match(/\d+/)) return "x"
+      stream.next()
+    }
+  })
+  var x = byClassName(cm.getWrapperElement(), "cm-x")
+  is(x.length, 1)
+  is(x[0].textContent, "233")
+  cm.replaceRange("", Pos(0, 4), Pos(0, 6))
+  is(byClassName(cm.getWrapperElement(), "cm-x").length, 2)
+}, {value: "foo /* 100 */\nbar + 233;\nbaz", mode: "javascript"})
+
 testCM("resizeLineWidget", function(cm) {
   addDoc(cm, 200, 3);
   var widget = document.createElement("pre");