Author: yooper
Inheritance: extends SpacePunctuationFilter, implements TextAnalysis\Interfaces\ITokenTransformation
 /**
  * 
  * @return array
  */
 protected function getParsedOutput()
 {
     $data = [];
     $filter = new PunctuationFilter();
     $phrases = (new WhitespaceTokenizer())->tokenize($this->output);
     foreach ($phrases as $phrase) {
         $tokens = explode("{$this->getSeparator()}", $phrase);
         $type = array_pop($tokens);
         foreach ($tokens as $token) {
             if (empty($token) || empty($filter->transform($token))) {
                 continue;
             }
             $data[] = [$token, $type];
         }
     }
     return $data;
 }
 public function testOnDate()
 {
     $transformer = new PunctuationFilter(['\\/', ':'], []);
     $this->assertEquals('8/8/2016 5:51 PM', $transformer->transform('8/8/2016 5:51 PM'));
 }
 public function testPunctuation()
 {
     $transformer = new PunctuationFilter();
     $this->assertEquals("Yoopers", $transformer->transform("Yooper's!?;,"));
 }