1 files changed, 44 insertions, 0 deletions
diff --git a/www/wiki/extensions/SemanticMediaWiki/tests/phpunit/Unit/Utils/TokenizerTest.php b/www/wiki/extensions/SemanticMediaWiki/tests/phpunit/Unit/Utils/TokenizerTest.php
new file mode 100644
index 00000000..d9f12a2f
--- /dev/null
+++ b/www/wiki/extensions/SemanticMediaWiki/tests/phpunit/Unit/Utils/TokenizerTest.php
@@ -0,0 +1,44 @@
+<?php
+
+namespace SMW\Tests\Utils;
+
+use SMW\Utils\Tokenizer;
+
+/**
+ * @covers \SMW\Utils\Tokenizer
+ * @group semantic-mediawiki
+ *
+ * @license GNU GPL v2+
+ * @since 2.5
+ *
+ * @author mwjames
+ */
+class TokenizerTest extends \PHPUnit_Framework_TestCase {
+
+	/**
+	 * @dataProvider textProvider
+	 */
+	public function testTokenize( $text, $expected ) {
+
+		$this->assertEquals(
+			$expected,
+			Tokenizer::tokenize( $text )
+		);
+	}
+
+	public function textProvider() {
+
+		$provider[] = [
+			'foo',
+			[ 'foo' ]
+		];
+
+		$provider[] = [
+			'foo bar',
+			[ 'foo', 'bar' ]
+		];
+
+		return $provider;
+	}
+
+}