Trustworthiness metric

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 - 3.3.0
     - Added OPTICS and Ward clusterers
+    - Added Trustworthiness metric for scoring embedding quality
     - Added Squared Euclidean distance kernel
 
 - 3.2.1

benchmarks/CrossValidation/Metrics/TrustworthinessBench.php

@@ -0,0 +1,67 @@
+<?php
+
+namespace Rubix\ML\Benchmarks\CrossValidation\Metrics;
+
+use Rubix\ML\Datasets\Generators\Blob;
+use Rubix\ML\Transformers\PrincipalComponentAnalysis;
+use Rubix\ML\CrossValidation\Metrics\Trustworthiness;
+
+/**
+ * @Groups({"CrossValidation", "Metrics"})
+ * @BeforeMethods({"setUp"})
+ */
+class TrustworthinessBench
+{
+    protected const SIZE = 2000;
+
+    protected const DIMS = 10;
+
+    protected const TARGET_DIMS = 2;
+
+    /**
+     * @var Trustworthiness
+     */
+    protected Trustworthiness $metric;
+
+    /**
+     * High-dimensional original samples.
+     *
+     * @var list<list<float>>
+     */
+    protected array $samples;
+
+    /**
+     * Low-dimensional embedding of $samples.
+     *
+     * @var list<list<float>>
+     */
+    protected array $embedding;
+
+    public function setUp() : void
+    {
+        $center = array_fill(0, self::DIMS, 0.0);
+
+        $dataset = (new Blob($center, 1.5))->generate(self::SIZE);
+
+        $pca = new PrincipalComponentAnalysis(self::TARGET_DIMS);
+
+        $this->samples = $dataset->samples();
+
+        $dataset->apply($pca);
+
+        $this->embedding = $dataset->samples();
+
+        $this->metric = new Trustworthiness(5);
+    }
+
+    /**
+     * @Subject
+     * @Iterations(5)
+     * @OutputTimeUnit("milliseconds", precision=3)
+     * @return float
+     */
+    public function score() : float
+    {
+        return $this->metric->score($this->samples, $this->embedding);
+    }
+}

docs/cross-validation/metrics/api.md

@@ -50,6 +50,35 @@ $metric = new ProbabilisticAccuracy;
 $score = $metric->score($probabilities, $dataset->labels());
 ```
 
+## Scoring Embeddings
+
+Metrics like Trustworthiness score the output of a dimensionality-reducing transformer (an embedding) rather than an estimator's predictions. They take the original samples and their transformed embedding and return a single score.
+
+```php
+public score(array $samples, array $embedded) : float
+```
+
+```php
+use Rubix\ML\Transformers\TSNE;
+use Rubix\ML\CrossValidation\Metrics\Trustworthiness;
+
+$transformer = new TSNE(2);
+
+$samples = $dataset->samples();
+$embedding = $samples;
+$transformer->transform($embedding);
+
+$metric = new Trustworthiness(5);
+
+$score = $metric->score($samples, $embedding);
+
+echo $score;
+```
+
+```text
+0.88
+```
+
 ## Score Range
 
 Output the minimum and maximum value the validation score can take in a [2-tuple](../../faq.md#what-is-a-tuple).

docs/cross-validation/metrics/trustworthiness.md

@@ -0,0 +1,64 @@
+<span style="float:right;"><a href="https://github.com/RubixML/ML/blob/master/src/CrossValidation/Metrics/Trustworthiness.php">[source]</a></span>
+
+# Trustworthiness
+
+Trustworthiness measures how well a low-dimensional embedding (e.g. produced by t-SNE or PCA) preserves the local neighborhood structure of the original feature space. For every sample, its nearest neighbors in the embedded space are inspected; any that were distant relatives in the input space are penalized in proportion to their rank. A perfect preservation of local structure yields a score of 1.0; the more the local ordering degrades, the lower the score.
+
+$$
+{\displaystyle T = 1 - \frac{2}{n\,k\,(2n - 3k - 1)} \sum_{i=1}^{n} \sum_{j \in \mathcal{N}^{k}_{i}} \max(0, r(i,j) - k)}
+$$
+
+where *n* is the number of samples, *k* is the number of neighbors, $\mathcal{N}^{k}_{i}$ is the set of *k* neighbors of sample *i* in the embedded space, and *r*(i, j) is the rank of sample *j* among *i*'s neighbors in the original space.
+
+A score of 1.0 means the embedding preserves local structure perfectly; lower scores indicate that points which were close together in the original space have been separated (or far-apart points have been brought together) by the projection.
+
+!!! note
+    The number of neighbors *k* must be less than *n* / 2, otherwise the score is not guaranteed to lie within [0, 1].
+
+**Estimator Compatibility:** None
+
+**Score Range:** 0 to 1
+
+## Parameters
+
+| # | Name | Default | Type | Description |
+| --- | --- | --- | --- | --- |
+| 1 | k | 5 | int | The number of nearest neighbors to consider when scoring local structure. |
+| 2 | inputKernel | Euclidean | Symmetric | The symmetric kernel used to measure dissimilarity in the original feature space. The embedded space is always scored with the fixed `SquaredEuclidean` kernel. |
+
+## Example
+
+```php
+use Rubix\ML\Transformers\TSNE;
+use Rubix\ML\Kernels\Distance\Euclidean;
+use Rubix\ML\CrossValidation\Metrics\Trustworthiness;
+
+$transformer = new TSNE(
+    dimensions: 2,
+    rate: 10.0,
+    perplexity: 30,
+    exaggeration: 12.0,
+    epochs: 1000,
+    minGradient: 1e-7,
+    evalInterval: 50,
+    window: 6,
+);
+
+$original = clone $dataset;
+
+$dataset->apply($transformer);
+
+$metric = new Trustworthiness(5, new Euclidean());
+
+$score = $metric->score($original->samples(), $dataset->samples());
+
+echo $score;
+```
+
+```text
+0.88
+```
+
+## References
+
+[^1]: J. Venna and S. Kaski. (2001). Neighborhood Preservation in Nonlinear Projection Methods: An Experimental Study.

mkdocs.yml

@@ -260,6 +260,7 @@ nav:
         - Homogeneity: cross-validation/metrics/homogeneity.md
         - Rand Index: cross-validation/metrics/rand-index.md
         - Top K Accuracy: cross-validation/metrics/top-k-accuracy.md
+        - Trustworthiness: cross-validation/metrics/trustworthiness.md
         - V Measure: cross-validation/metrics/v-measure.md
       - Reports:
         - API Reference: cross-validation/reports/api.md

src/CrossValidation/Metrics/Trustworthiness.php

@@ -0,0 +1,201 @@
+<?php
+
+namespace Rubix\ML\CrossValidation\Metrics;
+
+use Rubix\ML\Tuple;
+use Rubix\ML\EstimatorType;
+use Rubix\ML\Kernels\Distance\Euclidean;
+use Rubix\ML\Kernels\Distance\SquaredEuclidean;
+use Rubix\ML\Kernels\Distance\Symmetric;
+use Rubix\ML\Exceptions\InvalidArgumentException;
+use Stringable;
+
+use function max;
+use function count;
+use function asort;
+use function array_keys;
+use function array_slice;
+
+/**
+ * Trustworthiness
+ *
+ * Trustworthiness is a manifold-learning quality metric that measures how well
+ * a low-dimensional embedding preserves the local neighborhood structure of
+ * the original feature space. For every sample, its nearest neighbors in the
+ * embedded space are inspected; any that were far relatives in the input space
+ * are penalized in proportion to how far they actually are. Perfect preservation
+ * of local structure yields a score of 1.0; a degraded embedding lowers the
+ * score.
+ *
+ * References:
+ * [1] J. Venna and S. Kaski. (2001). Neighborhood Preservation in Nonlinear
+ *     Projection Methods: An Experimental Study.
+ *
+ * @category    Machine Learning
+ * @package     Rubix/ML
+ * @author      Andrew DalPino
+ */
+class Trustworthiness implements Stringable
+{
+    /**
+     * The number of nearest neighbors considered when scoring local structure preservation.
+     *
+     * @var int
+     */
+    protected int $k;
+
+    /**
+     * The symmetric distance kernel used to measure dissimilarity in the input space.
+     *
+     * @var Symmetric
+     */
+    protected Symmetric $inputKernel;
+
+    /**
+     * The symmetric distance kernel used to measure dissimilarity in the embedded
+     * space.
+     *
+     * @var Symmetric
+     */
+    protected Symmetric $embeddingKernel;
+
+    /**
+     * @param int $k
+     * @param ?Symmetric $inputKernel
+     * @throws InvalidArgumentException
+     */
+    public function __construct(int $k = 5, ?Symmetric $inputKernel = null)
+    {
+        if ($k < 1) {
+            throw new InvalidArgumentException('Number of neighbors must be'
+                . " greater than 0, $k given.");
+        }
+
+        $this->k = $k;
+        $this->inputKernel = $inputKernel ?? new Euclidean();
+        $this->embeddingKernel = new SquaredEuclidean();
+    }
+
+    /**
+     * Return a tuple of the min and max output value for this metric.
+     *
+     * @return Tuple<float,float>
+     */
+    public function range() : Tuple
+    {
+        return new Tuple(0.0, 1.0);
+    }
+
+    /**
+     * Trustworthiness scores point clouds (embeddings) directly rather than
+     * estimator predictions, so it is compatible with no estimator type.
+     *
+     * @internal
+     *
+     * @return list<EstimatorType>
+     */
+    public function compatibility() : array
+    {
+        return [];
+    }
+
+    /**
+     * Score the extent to which the embedded space retains the local
+     * neighborhood structure of the original space.
+     *
+     * @param list<list<string|int|float>> $samples
+     * @param list<list<float>> $embedded
+     * @throws InvalidArgumentException
+     * @return float
+     */
+    public function score(array $samples, array $embedded) : float
+    {
+        $n = count($samples);
+
+        if ($n !== count($embedded)) {
+            throw new InvalidArgumentException('Number of samples and embedded'
+                . " points must match, $n samples vs " . count($embedded) . ' given.');
+        }
+
+        if ($n < 2) {
+            return 1.0;
+        }
+
+        if (2 * $this->k >= $n) {
+            throw new InvalidArgumentException("Number of neighbors ($this->k) must be"
+                . ' less than n / 2 (' . ($n / 2) . ') or the'
+                . ' Trustworthiness could fall below 0.');
+        }
+
+        $total = 0.0;
+
+        for ($i = 0; $i < $n; ++$i) {
+            $embeddedRow = $this->distancesFrom($embedded, $i, $this->embeddingKernel);
+
+            asort($embeddedRow);
+
+            $neighbors = array_slice(array_keys($embeddedRow), 0, $this->k);
+
+            $inputRow = $this->distancesFrom($samples, $i, $this->inputKernel);
+
+            foreach ($neighbors as $j) {
+                $rank = 1;
+
+                foreach ($inputRow as $l => $distance) {
+                    if ($distance < $inputRow[$j] or ($distance == $inputRow[$j] and $l < $j)) {
+                        ++$rank;
+                    }
+                }
+
+                $total += max(0, $rank - $this->k);
+            }
+        }
+
+        $denominator = $n * $this->k * (2 * $n - 3 * $this->k - 1);
+
+        return 1.0 - (2.0 * $total) / $denominator;
+    }
+
+    /**
+     * Compute the distance from sample $i to every other sample, keyed by index
+     * with $i itself omitted. Because $i is absent, no INF diagonal sentinel is
+     * needed — a point can never be its own neighbor — and the keys are emitted
+     * in ascending index order so that a stable sort reproduces the index-order
+     * tie-break.
+     *
+     * @internal
+     *
+     * @param list<list<string|int|float>> $samples
+     * @param int $index
+     * @param Symmetric $kernel
+     * @return array<int, float>
+     */
+    protected function distancesFrom(array $samples, int $index, Symmetric $kernel) : array
+    {
+        $row = [];
+
+        $sample = $samples[$index];
+
+        for ($i = 0; $i < count($samples); ++$i) {
+            if ($i === $index) {
+                continue;
+            }
+
+            $row[$i] = $kernel->compute($sample, $samples[$i]);
+        }
+
+        return $row;
+    }
+
+    /**
+     * Return the string representation of the object.
+     *
+     * @internal
+     *
+     * @return string
+     */
+    public function __toString() : string
+    {
+        return 'Trustworthiness';
+    }
+}