Preserve original Unicode text on request

README.md

@@ -40,6 +40,9 @@ Splits the text into sentences.
 
 -	**`Sentence::SPLIT_TRIM`**: Trim whitespace off the left and right sides of
 	each returned sentence.
+-	**`Sentence::SPLIT_PRESERVE`**: Preserve original Unicode quote and
+	apostrophe characters in returned sentences instead of returning their normalized
+	ASCII equivalents. It can be combined with `SPLIT_TRIM` using `|`.
 
 ## Documentation
 You can find documentation generated from the source code by ApiGen here: [ApiGen documentation](doc/)

src/Sentence.php

@@ -22,6 +22,11 @@ class Sentence
      */
     const SPLIT_TRIM = 0x1;
 
+    /**
+     * Preserve the original Unicode characters in returned sentences.
+     */
+    const SPLIT_PRESERVE = 0x2;
+
     /**
      * List of characters used to terminate sentences.
      *
@@ -396,6 +401,42 @@ private function sentenceMerge($shorts)
         return $sentences;
     }
 
+    /**
+     * Project normalized sentence boundaries back onto the original text.
+     *
+     * Quote normalization is character-for-character, so offsets remain stable.
+     * If another normalization changes the character count, retain the normalized
+     * output rather than risk returning incorrect slices.
+     *
+     * @param string[] $sentences
+     * @param string   $normalized
+     * @param string   $original
+     *
+     * @return string[]
+     */
+    private static function preserveOriginalText($sentences, $normalized, $original)
+    {
+        if (mb_strlen($normalized, 'UTF-8') !== mb_strlen($original, 'UTF-8')) {
+            return $sentences;
+        }
+
+        $result = [];
+        $offset = 0;
+
+        foreach ($sentences as $sentence) {
+            $position = mb_strpos($normalized, $sentence, $offset, 'UTF-8');
+            if ($position === false) {
+                return $sentences;
+            }
+
+            $length = mb_strlen($sentence, 'UTF-8');
+            $result[] = mb_substr($original, $position, $length, 'UTF-8');
+            $offset = $position + $length;
+        }
+
+        return $result;
+    }
+
     /**
      * Return the sentences detected in the provided text.
      * Set the Sentence::SPLIT_TRIM flag to trim whitespace.
@@ -420,7 +461,9 @@ public function split($text, $flags = 0, $pipeline = [])
             ];
         }
 
-        // clean funny quotes
+        $originalText = $text;
+
+        // clean funny quotes for boundary detection
         $text = Multibyte::cleanUnicode($text);
 
         // Split
@@ -435,6 +478,10 @@ public function split($text, $flags = 0, $pipeline = [])
         }
 
         // Post process
+        if ($flags & self::SPLIT_PRESERVE) {
+            $sentences = self::preserveOriginalText($sentences, $text, $originalText);
+        }
+
         if ($flags & self::SPLIT_TRIM) {
             return self::trimSentences($sentences);
         }

tests/SentenceTest.php

@@ -148,6 +148,23 @@ public function testCleanupUnicode()
         $this->assertSame(['Fix "these" quotes'], $this->object->split("Fix \xC2\xABthese\xC2\xAB quotes"));
     }
 
+    /**
+     * @covers ::split
+     */
+    public function testPreserveOriginalUnicode()
+    {
+        $text = 'Super Tortas is outstanding! I haven’t had as many tortas as I would like.';
+
+        $this->assertSame(
+            ['Super Tortas is outstanding!', ' I haven’t had as many tortas as I would like.'],
+            $this->object->split($text, Sentence::SPLIT_PRESERVE)
+        );
+        $this->assertSame(
+            ['Super Tortas is outstanding!', 'I haven’t had as many tortas as I would like.'],
+            $this->object->split($text, Sentence::SPLIT_PRESERVE | Sentence::SPLIT_TRIM)
+        );
+    }
+
     /**
      * @covers ::split
      */