diff --git a/CHANGELOG.md b/CHANGELOG.md index f18287c78..769c01312 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ - 3.3.0 - Added OPTICS and Ward clusterers + - Added Trustworthiness metric for scoring embedding quality - Added Squared Euclidean distance kernel - 3.2.1 diff --git a/benchmarks/CrossValidation/Metrics/TrustworthinessBench.php b/benchmarks/CrossValidation/Metrics/TrustworthinessBench.php new file mode 100644 index 000000000..311d6770c --- /dev/null +++ b/benchmarks/CrossValidation/Metrics/TrustworthinessBench.php @@ -0,0 +1,67 @@ +> + */ + protected array $samples; + + /** + * Low-dimensional embedding of $samples. + * + * @var list> + */ + protected array $embedding; + + public function setUp() : void + { + $center = array_fill(0, self::DIMS, 0.0); + + $dataset = (new Blob($center, 1.5))->generate(self::SIZE); + + $pca = new PrincipalComponentAnalysis(self::TARGET_DIMS); + + $this->samples = $dataset->samples(); + + $dataset->apply($pca); + + $this->embedding = $dataset->samples(); + + $this->metric = new Trustworthiness(5); + } + + /** + * @Subject + * @Iterations(5) + * @OutputTimeUnit("milliseconds", precision=3) + * @return float + */ + public function score() : float + { + return $this->metric->score($this->samples, $this->embedding); + } +} diff --git a/docs/cross-validation/metrics/api.md b/docs/cross-validation/metrics/api.md index 88c271ddc..c504c4981 100644 --- a/docs/cross-validation/metrics/api.md +++ b/docs/cross-validation/metrics/api.md @@ -50,6 +50,35 @@ $metric = new ProbabilisticAccuracy; $score = $metric->score($probabilities, $dataset->labels()); ``` +## Scoring Embeddings + +Metrics like Trustworthiness score the output of a dimensionality-reducing transformer (an embedding) rather than an estimator's predictions. They take the original samples and their transformed embedding and return a single score. + +```php +public score(array $samples, array $embedded) : float +``` + +```php +use Rubix\ML\Transformers\TSNE; +use Rubix\ML\CrossValidation\Metrics\Trustworthiness; + +$transformer = new TSNE(2); + +$samples = $dataset->samples(); +$embedding = $samples; +$transformer->transform($embedding); + +$metric = new Trustworthiness(5); + +$score = $metric->score($samples, $embedding); + +echo $score; +``` + +```text +0.88 +``` + ## Score Range Output the minimum and maximum value the validation score can take in a [2-tuple](../../faq.md#what-is-a-tuple). diff --git a/docs/cross-validation/metrics/trustworthiness.md b/docs/cross-validation/metrics/trustworthiness.md new file mode 100644 index 000000000..a29915a47 --- /dev/null +++ b/docs/cross-validation/metrics/trustworthiness.md @@ -0,0 +1,64 @@ +[source] + +# Trustworthiness + +Trustworthiness measures how well a low-dimensional embedding (e.g. produced by t-SNE or PCA) preserves the local neighborhood structure of the original feature space. For every sample, its nearest neighbors in the embedded space are inspected; any that were distant relatives in the input space are penalized in proportion to their rank. A perfect preservation of local structure yields a score of 1.0; the more the local ordering degrades, the lower the score. + +$$ +{\displaystyle T = 1 - \frac{2}{n\,k\,(2n - 3k - 1)} \sum_{i=1}^{n} \sum_{j \in \mathcal{N}^{k}_{i}} \max(0, r(i,j) - k)} +$$ + +where *n* is the number of samples, *k* is the number of neighbors, $\mathcal{N}^{k}_{i}$ is the set of *k* neighbors of sample *i* in the embedded space, and *r*(i, j) is the rank of sample *j* among *i*'s neighbors in the original space. + +A score of 1.0 means the embedding preserves local structure perfectly; lower scores indicate that points which were close together in the original space have been separated (or far-apart points have been brought together) by the projection. + +!!! note + The number of neighbors *k* must be less than *n* / 2, otherwise the score is not guaranteed to lie within [0, 1]. + +**Estimator Compatibility:** None + +**Score Range:** 0 to 1 + +## Parameters + +| # | Name | Default | Type | Description | +| --- | --- | --- | --- | --- | +| 1 | k | 5 | int | The number of nearest neighbors to consider when scoring local structure. | +| 2 | inputKernel | Euclidean | Symmetric | The symmetric kernel used to measure dissimilarity in the original feature space. The embedded space is always scored with the fixed `SquaredEuclidean` kernel. | + +## Example + +```php +use Rubix\ML\Transformers\TSNE; +use Rubix\ML\Kernels\Distance\Euclidean; +use Rubix\ML\CrossValidation\Metrics\Trustworthiness; + +$transformer = new TSNE( + dimensions: 2, + rate: 10.0, + perplexity: 30, + exaggeration: 12.0, + epochs: 1000, + minGradient: 1e-7, + evalInterval: 50, + window: 6, +); + +$original = clone $dataset; + +$dataset->apply($transformer); + +$metric = new Trustworthiness(5, new Euclidean()); + +$score = $metric->score($original->samples(), $dataset->samples()); + +echo $score; +``` + +```text +0.88 +``` + +## References + +[^1]: J. Venna and S. Kaski. (2001). Neighborhood Preservation in Nonlinear Projection Methods: An Experimental Study. diff --git a/mkdocs.yml b/mkdocs.yml index dbbd95c8c..c94eefe80 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -260,6 +260,7 @@ nav: - Homogeneity: cross-validation/metrics/homogeneity.md - Rand Index: cross-validation/metrics/rand-index.md - Top K Accuracy: cross-validation/metrics/top-k-accuracy.md + - Trustworthiness: cross-validation/metrics/trustworthiness.md - V Measure: cross-validation/metrics/v-measure.md - Reports: - API Reference: cross-validation/reports/api.md diff --git a/src/CrossValidation/Metrics/Trustworthiness.php b/src/CrossValidation/Metrics/Trustworthiness.php new file mode 100644 index 000000000..78012b09d --- /dev/null +++ b/src/CrossValidation/Metrics/Trustworthiness.php @@ -0,0 +1,201 @@ +k = $k; + $this->inputKernel = $inputKernel ?? new Euclidean(); + $this->embeddingKernel = new SquaredEuclidean(); + } + + /** + * Return a tuple of the min and max output value for this metric. + * + * @return Tuple + */ + public function range() : Tuple + { + return new Tuple(0.0, 1.0); + } + + /** + * Trustworthiness scores point clouds (embeddings) directly rather than + * estimator predictions, so it is compatible with no estimator type. + * + * @internal + * + * @return list + */ + public function compatibility() : array + { + return []; + } + + /** + * Score the extent to which the embedded space retains the local + * neighborhood structure of the original space. + * + * @param list> $samples + * @param list> $embedded + * @throws InvalidArgumentException + * @return float + */ + public function score(array $samples, array $embedded) : float + { + $n = count($samples); + + if ($n !== count($embedded)) { + throw new InvalidArgumentException('Number of samples and embedded' + . " points must match, $n samples vs " . count($embedded) . ' given.'); + } + + if ($n < 2) { + return 1.0; + } + + if (2 * $this->k >= $n) { + throw new InvalidArgumentException("Number of neighbors ($this->k) must be" + . ' less than n / 2 (' . ($n / 2) . ') or the' + . ' Trustworthiness could fall below 0.'); + } + + $total = 0.0; + + for ($i = 0; $i < $n; ++$i) { + $embeddedRow = $this->distancesFrom($embedded, $i, $this->embeddingKernel); + + asort($embeddedRow); + + $neighbors = array_slice(array_keys($embeddedRow), 0, $this->k); + + $inputRow = $this->distancesFrom($samples, $i, $this->inputKernel); + + foreach ($neighbors as $j) { + $rank = 1; + + foreach ($inputRow as $l => $distance) { + if ($distance < $inputRow[$j] or ($distance == $inputRow[$j] and $l < $j)) { + ++$rank; + } + } + + $total += max(0, $rank - $this->k); + } + } + + $denominator = $n * $this->k * (2 * $n - 3 * $this->k - 1); + + return 1.0 - (2.0 * $total) / $denominator; + } + + /** + * Compute the distance from sample $i to every other sample, keyed by index + * with $i itself omitted. Because $i is absent, no INF diagonal sentinel is + * needed — a point can never be its own neighbor — and the keys are emitted + * in ascending index order so that a stable sort reproduces the index-order + * tie-break. + * + * @internal + * + * @param list> $samples + * @param int $index + * @param Symmetric $kernel + * @return array + */ + protected function distancesFrom(array $samples, int $index, Symmetric $kernel) : array + { + $row = []; + + $sample = $samples[$index]; + + for ($i = 0; $i < count($samples); ++$i) { + if ($i === $index) { + continue; + } + + $row[$i] = $kernel->compute($sample, $samples[$i]); + } + + return $row; + } + + /** + * Return the string representation of the object. + * + * @internal + * + * @return string + */ + public function __toString() : string + { + return 'Trustworthiness'; + } +} diff --git a/tests/CrossValidation/Metrics/TrustworthinessTest.php b/tests/CrossValidation/Metrics/TrustworthinessTest.php new file mode 100644 index 000000000..1e013f8a0 --- /dev/null +++ b/tests/CrossValidation/Metrics/TrustworthinessTest.php @@ -0,0 +1,148 @@ +metric = new Trustworthiness(); + } + + #[Test] + public function range() : void + { + $tuple = $this->metric->range(); + + $this->assertInstanceOf(Tuple::class, $tuple); + $this->assertCount(2, $tuple); + $this->assertGreaterThan($tuple[0], $tuple[1]); + $this->assertEquals([0.0, 1.0], $tuple->list()); + } + + #[Test] + public function compatibility() : void + { + $this->assertSame([], $this->metric->compatibility()); + } + + #[Test] + public function zeroNeighborsThrows() : void + { + $this->expectException(InvalidArgumentException::class); + new Trustworthiness(0); + } + + #[Test] + public function identitySampleMatchesIdentityEmbedding() : void + { + $samples = [[0.0], [1.0], [2.0], [5.0], [10.0], [11.0], [12.0], [20.0]]; + $metric = (new Trustworthiness(2)); + + $score = $metric->score($samples, $samples); + + $this->assertEqualsWithDelta(1.0, $score, 1e-8); + } + + #[Test] + public function rigidRotationInvariance() : void + { + // 8 points on a unit circle, rotated rigidly. + $n = 8; + $samples = []; + + for ($i = 0; $i < $n; ++$i) { + $theta = 2 * M_PI * $i / $n; + $samples[] = [cos($theta), sin($theta)]; + } + + $angle = 0.7; + $c = cos($angle); + $s = sin($angle); + $embedded = []; + + foreach ($samples as [$x, $y]) { + $embedded[] = [$x * $c - $y * $s, $x * $s + $y * $c]; + } + + $score = (new Trustworthiness(2))->score($samples, $embedded); + + $this->assertEqualsWithDelta(1.0, $score, 1e-8); + } + + #[Test] + public function dimensionReductionLowersScore() : void + { + // Points on a 2D circle, projected to 1D (loses information). + $n = 8; + $samples = []; + $embedded = []; + + for ($i = 0; $i < $n; ++$i) { + $theta = 2 * M_PI * $i / $n; + $x = cos($theta); + $y = sin($theta); + $samples[] = [$x, $y]; + $embedded[] = [$x]; + } + + $good = (new Trustworthiness(2))->score($samples, $samples); + $bad = (new Trustworthiness(2))->score($samples, $embedded); + + $this->assertGreaterThan($bad, $good); + } + + #[Test] + public function mismatchedCountsThrows() : void + { + $this->expectException(InvalidArgumentException::class); + $this->metric->score([[0.0], [1.0], [2.0]], [[0.0], [1.0]]); + } + + #[Test] + public function tooManyNeighborsThrows() : void + { + $samples = [[0.0], [1.0], [2.0], [3.0]]; + $this->expectException(InvalidArgumentException::class); + (new Trustworthiness(2))->score($samples, $samples); + } + + #[Test] + public function emptySet() : void + { + $this->assertEqualsWithDelta(1.0, $this->metric->score([], []), 1e-8); + } + + #[Test] + public function singleSample() : void + { + $this->assertEqualsWithDelta(1.0, $this->metric->score([[0.0]], [[0.0]]), 1e-8); + } + + #[Test] + public function rangeBoundsEnforced() : void + { + $samples = [[0.0], [1.0], [3.0], [4.0], [10.0], [11.0]]; + $embedded = [[0.0], [1.0], [10.0], [4.0], [3.0], [11.0]]; + + $score = (new Trustworthiness(1))->score($samples, $embedded); + + [$min, $max] = $this->metric->range()->list(); + $this->assertGreaterThanOrEqual($min, $score); + $this->assertLessThanOrEqual($max, $score); + } +}