diff --git a/README.md b/README.md index 8e563c4..cd4e057 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,9 @@ Splits the text into sentences. - **`Sentence::SPLIT_TRIM`**: Trim whitespace off the left and right sides of each returned sentence. +- **`Sentence::SPLIT_PRESERVE`**: Preserve original Unicode quote and + apostrophe characters in returned sentences instead of returning their normalized + ASCII equivalents. It can be combined with `SPLIT_TRIM` using `|`. ## Documentation You can find documentation generated from the source code by ApiGen here: [ApiGen documentation](doc/) diff --git a/src/Sentence.php b/src/Sentence.php index 0ac1f4f..0821fce 100644 --- a/src/Sentence.php +++ b/src/Sentence.php @@ -22,6 +22,11 @@ class Sentence */ const SPLIT_TRIM = 0x1; + /** + * Preserve the original Unicode characters in returned sentences. + */ + const SPLIT_PRESERVE = 0x2; + /** * List of characters used to terminate sentences. * @@ -396,6 +401,42 @@ private function sentenceMerge($shorts) return $sentences; } + /** + * Project normalized sentence boundaries back onto the original text. + * + * Quote normalization is character-for-character, so offsets remain stable. + * If another normalization changes the character count, retain the normalized + * output rather than risk returning incorrect slices. + * + * @param string[] $sentences + * @param string $normalized + * @param string $original + * + * @return string[] + */ + private static function preserveOriginalText($sentences, $normalized, $original) + { + if (mb_strlen($normalized, 'UTF-8') !== mb_strlen($original, 'UTF-8')) { + return $sentences; + } + + $result = []; + $offset = 0; + + foreach ($sentences as $sentence) { + $position = mb_strpos($normalized, $sentence, $offset, 'UTF-8'); + if ($position === false) { + return $sentences; + } + + $length = mb_strlen($sentence, 'UTF-8'); + $result[] = mb_substr($original, $position, $length, 'UTF-8'); + $offset = $position + $length; + } + + return $result; + } + /** * Return the sentences detected in the provided text. * Set the Sentence::SPLIT_TRIM flag to trim whitespace. @@ -420,7 +461,9 @@ public function split($text, $flags = 0, $pipeline = []) ]; } - // clean funny quotes + $originalText = $text; + + // clean funny quotes for boundary detection $text = Multibyte::cleanUnicode($text); // Split @@ -435,6 +478,10 @@ public function split($text, $flags = 0, $pipeline = []) } // Post process + if ($flags & self::SPLIT_PRESERVE) { + $sentences = self::preserveOriginalText($sentences, $text, $originalText); + } + if ($flags & self::SPLIT_TRIM) { return self::trimSentences($sentences); } diff --git a/tests/SentenceTest.php b/tests/SentenceTest.php index 0bbc3b7..d5c9196 100644 --- a/tests/SentenceTest.php +++ b/tests/SentenceTest.php @@ -148,6 +148,23 @@ public function testCleanupUnicode() $this->assertSame(['Fix "these" quotes'], $this->object->split("Fix \xC2\xABthese\xC2\xAB quotes")); } + /** + * @covers ::split + */ + public function testPreserveOriginalUnicode() + { + $text = 'Super Tortas is outstanding! I haven’t had as many tortas as I would like.'; + + $this->assertSame( + ['Super Tortas is outstanding!', ' I haven’t had as many tortas as I would like.'], + $this->object->split($text, Sentence::SPLIT_PRESERVE) + ); + $this->assertSame( + ['Super Tortas is outstanding!', 'I haven’t had as many tortas as I would like.'], + $this->object->split($text, Sentence::SPLIT_PRESERVE | Sentence::SPLIT_TRIM) + ); + } + /** * @covers ::split */