Fix tag highlighting in editor (#1773)

* Fix tag highlighting in editor

* Add test case for tag highlighting

app/src/main/java/com/keylesspalace/tusky/util/SpanUtils.kt

@@ -12,7 +12,7 @@ import kotlin.math.max
  * @see <a href="https://github.com/tootsuite/mastodon/blob/master/app/models/tag.rb">
  *     Tag#HASHTAG_RE</a>.
  */
-private const val TAG_REGEX = "(?:^|[^/)\\w])#([\\w_]*[\\p{Alpha}_][\\w_]*)"
+private const val TAG_REGEX = "(?:^|[^/)A-Za-z0-9_])#([\\w_]*[\\p{Alpha}_][\\w_]*)"
 
 /**
  * @see <a href="https://github.com/tootsuite/mastodon/blob/master/app/models/account.rb">
@@ -30,10 +30,10 @@ private val STRICT_WEB_URL_PATTERN = Pattern.compile("(((?:(?i:http|https|rtsp):
 
 private val spanClasses = listOf(ForegroundColorSpan::class.java, URLSpan::class.java)
 private val finders = mapOf(
-    FoundMatchType.HTTP_URL to PatternFinder(':', HTTP_URL_REGEX, 5),
+    FoundMatchType.HTTP_URL to PatternFinder(':', HTTP_URL_REGEX, 5, Character::isWhitespace),
-    FoundMatchType.HTTPS_URL to PatternFinder(':', HTTPS_URL_REGEX, 6),
+    FoundMatchType.HTTPS_URL to PatternFinder(':', HTTPS_URL_REGEX, 6, Character::isWhitespace),
-    FoundMatchType.TAG to PatternFinder('#', TAG_REGEX, 1),
+    FoundMatchType.TAG to PatternFinder('#', TAG_REGEX, 1, ::isValidForTagPrefix),
-    FoundMatchType.MENTION to PatternFinder('@', MENTION_REGEX, 1)
+    FoundMatchType.MENTION to PatternFinder('@', MENTION_REGEX, 1, Character::isWhitespace) // TODO: We also need a proper validator for mentions
 )
 
 private enum class FoundMatchType {
@@ -49,7 +49,8 @@ private class FindCharsResult {
     var end: Int = -1
 }
 
-private class PatternFinder(val searchCharacter: Char, regex: String, val searchPrefixWidth: Int) {
+private class PatternFinder(val searchCharacter: Char, regex: String, val searchPrefixWidth: Int,
+                            val prefixValidator: (Int) -> Boolean) {
     val pattern: Pattern = Pattern.compile(regex, Pattern.CASE_INSENSITIVE)
 }
 
@@ -67,7 +68,7 @@ private fun findPattern(string: String, fromIndex: Int): FindCharsResult {
             val finder = finders[matchType]
             if (finder!!.searchCharacter == c
                     && ((i - fromIndex) < finder.searchPrefixWidth ||
-                            Character.isWhitespace(string.codePointAt(i - finder.searchPrefixWidth)))) {
+                            finder.prefixValidator(string.codePointAt(i - finder.searchPrefixWidth)))) {
                 result.matchType = matchType
                 result.start = max(0, i - finder.searchPrefixWidth)
                 findEndOfPattern(string, result, finder.pattern)
@@ -87,10 +88,22 @@ private fun findEndOfPattern(string: String, result: FindCharsResult, pattern: P
         // Once we have API level 26+, we can use named captures...
         val end = matcher.end()
         result.start = matcher.start()
-        if (Character.isWhitespace(string.codePointAt(result.start))) {
+        when (result.matchType) {
-            ++result.start
+            FoundMatchType.TAG -> {
+                if (isValidForTagPrefix(string.codePointAt(result.start))) {
+                    if (string[result.start] != '#' ||
+                            (string[result.start] == '#' && string[result.start + 1] == '#')) {
+                        ++result.start
+                    }
+                }
+            }
+            else -> {
+                if (Character.isWhitespace(string.codePointAt(result.start))) {
+                    ++result.start
+                }
+            }
         }
-        when(result.matchType) {
+        when (result.matchType) {
             FoundMatchType.HTTP_URL, FoundMatchType.HTTPS_URL -> {
                 // Preliminary url patterns are fast/permissive, now we'll do full validation
                 if (STRICT_WEB_URL_PATTERN.matcher(string.substring(result.start, end)).matches()) {
@@ -133,3 +146,16 @@ fun highlightSpans(text: Spannable, colour: Int) {
         }
     }
 }
+
+private fun isWordCharacters(codePoint: Int): Boolean {
+    return (codePoint in 0x30..0x39) || // [0-9]
+            (codePoint in 0x41..0x5a) || // [A-Z]
+            (codePoint == 0x5f) || // _
+            (codePoint in 0x61..0x7a) // [a-z]
+}
+
+private fun isValidForTagPrefix(codePoint: Int): Boolean {
+    return !(isWordCharacters(codePoint) || // \w
+            (codePoint == 0x2f) || // /
+            (codePoint == 0x29)) // )
+}

app/src/test/java/com/keylesspalace/tusky/SpanUtilsTest.kt

@@ -10,11 +10,11 @@ import org.junit.runners.Parameterized
 class SpanUtilsTest {
     @Test
     fun matchesMixedSpans() {
-        val input = "one #one two: @two three : https://thr.ee/meh?foo=bar&wat=@at#hmm four #four five @five"
+        val input = "one #one two: @two three : https://thr.ee/meh?foo=bar&wat=@at#hmm four #four five @five ろく#six"
         val inputSpannable = FakeSpannable(input)
         highlightSpans(inputSpannable, 0xffffff)
         val spans = inputSpannable.spans
-        Assert.assertEquals(5, spans.size)
+        Assert.assertEquals(6, spans.size)
     }
 
     @Test
@@ -93,6 +93,38 @@ class SpanUtilsTest {
         }
     }
 
+    @RunWith(Parameterized::class)
+    class HighlightingTestsForTag(private val text: String,
+                                private val expectedStartIndex: Int,
+                                private val expectedEndIndex: Int) {
+        companion object {
+            @Parameterized.Parameters(name = "{0}")
+            @JvmStatic
+            fun data(): Iterable<Any> {
+                return listOf(
+                        arrayOf("#test", 0, 5),
+                        arrayOf(" #AfterSpace", 1, 12),
+                        arrayOf("#BeforeSpace ", 0, 12),
+                        arrayOf("@#after_at", 1, 10),
+                        arrayOf("あいうえお#after_hiragana", 5, 20),
+                        arrayOf("##DoubleHash", 1, 12),
+                        arrayOf("###TripleHash", 2, 13)
+                )
+            }
+        }
+
+        @Test
+        fun matchExpectations() {
+            val inputSpannable = FakeSpannable(text)
+            highlightSpans(inputSpannable, 0xffffff)
+            val spans = inputSpannable.spans
+            Assert.assertEquals(1, spans.size)
+            val span = spans.first()
+            Assert.assertEquals(expectedStartIndex, span.start)
+            Assert.assertEquals(expectedEndIndex, span.end)
+        }
+    }
+
     class FakeSpannable(private val text: String) : Spannable {
         val spans = mutableListOf<BoundedSpan>()