Account for underscores when tokenizing mentions for autocompletion (#888)

* Account for underscores when tokenizing mentions for autocompletion
Fixes #743

* Migrate MentionTokenizer to kotlin

* Add tests for mention tokenizer

app/src/main/java/com/keylesspalace/tusky/util/MentionTokenizer.java

@@ -1,67 +0,0 @@
-/* Copyright 2017 Andrew Dawson
- *
- * This file is a part of Tusky.
- *
- * This program is free software; you can redistribute it and/or modify it under the terms of the
- * GNU General Public License as published by the Free Software Foundation; either version 3 of the
- * License, or (at your option) any later version.
- *
- * Tusky is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even
- * the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
- * Public License for more details.
- *
- * You should have received a copy of the GNU General Public License along with Tusky; if not,
- * see <http://www.gnu.org/licenses>. */
-
-package com.keylesspalace.tusky.util;
-
-import android.text.SpannableString;
-import android.text.Spanned;
-import android.text.TextUtils;
-import android.widget.MultiAutoCompleteTextView;
-
-public class MentionTokenizer implements MultiAutoCompleteTextView.Tokenizer {
-    @Override
-    public int findTokenStart(CharSequence text, int cursor) {
-        int i = cursor;
-        while (i > 0 && text.charAt(i - 1) != '@') {
-            if (!Character.isLetterOrDigit(text.charAt(i - 1))) return cursor;
-            i--;
-        }
-        if (i < 1 || text.charAt(i - 1) != '@') {
-            return cursor;
-        }
-        return i;
-    }
-
-    @Override
-    public int findTokenEnd(CharSequence text, int cursor) {
-        int i = cursor;
-        int length = text.length();
-        while (i < length) {
-            if (text.charAt(i) == ' ') {
-                return i;
-            } else {
-                i++;
-            }
-        }
-        return length;
-    }
-
-    @Override
-    public CharSequence terminateToken(CharSequence text) {
-        int i = text.length();
-        while (i > 0 && text.charAt(i - 1) == ' ') {
-            i--;
-        }
-        if (i > 0 && text.charAt(i - 1) == ' ') {
-            return text;
-        } else if (text instanceof Spanned) {
-            SpannableString s = new SpannableString(text + " ");
-            TextUtils.copySpansFrom((Spanned) text, 0, text.length(), Object.class, s, 0);
-            return s;
-        } else {
-            return text + " ";
-        }
-    }
-}

app/src/main/java/com/keylesspalace/tusky/util/MentionTokenizer.kt

@@ -0,0 +1,72 @@
+/* Copyright 2017 Andrew Dawson
+ *
+ * This file is a part of Tusky.
+ *
+ * This program is free software; you can redistribute it and/or modify it under the terms of the
+ * GNU General Public License as published by the Free Software Foundation; either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * Tusky is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even
+ * the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
+ * Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along with Tusky; if not,
+ * see <http://www.gnu.org/licenses>. */
+
+package com.keylesspalace.tusky.util
+
+import android.text.SpannableString
+import android.text.Spanned
+import android.text.TextUtils
+import android.widget.MultiAutoCompleteTextView
+
+class MentionTokenizer : MultiAutoCompleteTextView.Tokenizer {
+    override fun findTokenStart(text: CharSequence, cursor: Int): Int {
+        if (cursor == 0) {
+            return cursor
+        }
+        var i = cursor
+        var character = text[i - 1]
+        while (i > 0 && character != '@') {
+            // See SpanUtils.MENTION_REGEX
+            if (!Character.isLetterOrDigit(character) && character != '_') {
+                return cursor
+            }
+            i--
+            character = if (i == 0) ' ' else text[i - 1]
+        }
+        if (i < 1 || character != '@') {
+            return cursor
+        }
+        return i
+    }
+
+    override fun findTokenEnd(text: CharSequence, cursor: Int): Int {
+        var i = cursor
+        val length = text.length
+        while (i < length) {
+            if (text[i] == ' ') {
+                return i
+            } else {
+                i++
+            }
+        }
+        return length
+    }
+
+    override fun terminateToken(text: CharSequence): CharSequence {
+        var i = text.length
+        while (i > 0 && text[i - 1] == ' ') {
+            i--
+        }
+        return if (i > 0 && text[i - 1] == ' ') {
+            text
+        } else if (text is Spanned) {
+            val s = SpannableString(text.toString() + " ")
+            TextUtils.copySpansFrom(text, 0, text.length, Object::class.java, s, 0)
+            s
+        } else {
+            text.toString() + " "
+        }
+    }
+}

app/src/test/java/com/keylesspalace/tusky/MentionTokenizerTest.kt

@@ -0,0 +1,61 @@
+/* Copyright 2018 Levi Bard
+ *
+ * This file is a part of Tusky.
+ *
+ * This program is free software; you can redistribute it and/or modify it under the terms of the
+ * GNU General Public License as published by the Free Software Foundation; either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * Tusky is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even
+ * the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
+ * Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along with Tusky; if not,
+ * see <http://www.gnu.org/licenses>. */
+
+package com.keylesspalace.tusky
+
+import com.keylesspalace.tusky.util.MentionTokenizer
+import org.junit.Assert
+import org.junit.Test
+import org.junit.runner.RunWith
+import org.junit.runners.Parameterized
+
+@RunWith(Parameterized::class)
+class MentionTokenizerTest(private val text: CharSequence,
+    private val expectedStartIndex: Int,
+    private val expectedEndIndex: Int) {
+
+    companion object {
+        @Parameterized.Parameters(name = "{0}")
+        @JvmStatic
+        fun data(): Iterable<Any> {
+            return listOf(
+                arrayOf("@mention", 1, 8),
+                arrayOf("@ment10n", 1, 8),
+                arrayOf("@ment10n_", 1, 9),
+                arrayOf("@ment10n_n", 1, 10),
+                arrayOf("@ment10n_9", 1, 10),
+                arrayOf(" @mention", 2, 9),
+                arrayOf(" @ment10n", 2, 9),
+                arrayOf(" @ment10n_", 2, 10),
+                arrayOf(" @ment10n_ @", 12, 12),
+                arrayOf(" @ment10n_ @ment20n", 12, 19),
+                arrayOf(" @ment10n_ @ment20n_", 12, 20),
+                arrayOf(" @ment10n_ @ment20n_n", 12, 21),
+                arrayOf(" @ment10n_ @ment20n_9", 12, 21),
+                arrayOf("mention", 7, 7),
+                arrayOf("ment10n", 7, 7),
+                arrayOf("mentio_", 7, 7)
+            )
+        }
+    }
+
+    private val tokenizer = MentionTokenizer()
+
+    @Test
+    fun tokenIndices_matchExpectations() {
+        Assert.assertEquals(expectedStartIndex, tokenizer.findTokenStart(text, text.length))
+        Assert.assertEquals(expectedEndIndex, tokenizer.findTokenEnd(text, text.length))
+    }
+}