Finish repairing the deferred KxQ rebuild, and pin the estimator constants

.gitattributes

@@ -11,7 +11,6 @@ NOTICE text eol=lf
 *.html text eol=lf
 *.hpp text eol=lf
 *.cpp text eol=lf
-*.sk text eol=lf
 *.md text eol=lf
 *.properties text eol=lf
 *.sh text eol=lf
@@ -27,6 +26,10 @@ NOTICE text eol=lf
 *.cmd text eol=crlf
 
 # Explicitly denote all files that are truly binary and should not be modified.
+# .sk files are serialized sketch images: arbitrary bytes that must never be
+# line-ending converted. A stray CRLF pair would be rewritten to LF on check-in,
+# silently truncating the fixture by one byte.
+*.sk binary
 *.jpg binary
 *.png binary
 *.svg binary

.rat-excludes

@@ -7,6 +7,7 @@
 # the matching entry to LICENSE in the same commit.
 MurmurHash3\.h
 xxhash64\.h
+fdlibm_log\.hpp
 #
 # Dotfiles and repository metadata with no comment convention.
 \..*

LICENSE

@@ -296,3 +296,17 @@ APPENDIX B: Additional licenses relevant to this product.
     Code Locations:
       * common/include/ceiling_power_of_2.hpp
     that is adapted from the above.
+
+    =============================================================
+    FDLIBM
+    =============================================================
+    Original source code:
+      Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved.
+      Developed at SunSoft, a Sun Microsystems, Inc. business.
+
+      Permission to use, copy, modify, and distribute this software is freely
+      granted, provided that this notice is preserved.
+
+    Code Locations:
+      common/include/fdlibm_log.hpp
+    that is adapted from __ieee754_log in FDLIBM 5.3.

common/CMakeLists.txt

@@ -40,6 +40,7 @@ install(FILES
       include/conditional_back_inserter.hpp
       include/conditional_forward.hpp
       include/count_zeros.hpp
+      include/fdlibm_log.hpp
       include/inv_pow2_table.hpp
       include/kolmogorov_smirnov_impl.hpp
       include/kolmogorov_smirnov.hpp

common/include/fdlibm_log.hpp

@@ -0,0 +1,101 @@
+// fdlibm __ieee754_log, used by Java's StrictMath.log (and by Math.log on most JVMs).
+// Derived from FDLIBM 5.3, Copyright (C) 1993 by Sun Microsystems, Inc.
+// "Permission to use, copy, modify, and distribute this software is freely granted,
+//  provided that this notice is preserved."
+#ifndef FDLIBM_LOG_HPP
+#define FDLIBM_LOG_HPP
+#include <cstdint>
+#include <cstring>
+#include <limits>
+
+namespace fdlibm {
+
+inline int32_t hi_word(double x){ uint64_t u; std::memcpy(&u,&x,8); return (int32_t)(uint32_t)(u>>32); }
+inline uint32_t lo_word(double x){ uint64_t u; std::memcpy(&u,&x,8); return (uint32_t)u; }
+inline void set_hi_word(double& x, uint32_t hi){ uint64_t u; std::memcpy(&u,&x,8);
+  u = (u & 0x00000000ffffffffULL) | ((uint64_t)hi<<32); std::memcpy(&x,&u,8); }
+
+// Forces a value to be rounded to a double before it is used again. fdlibm needs strict
+// IEEE-754 evaluation: a fused multiply-add anywhere in the polynomial below changes the
+// result. Pragmas are overridden by an explicit -ffp-contract=fast, so use a volatile
+// round-trip, which the standard requires the compiler to honour.
+inline double rnd(double v) { volatile double t = v; return t; }
+
+inline double log(double x) {
+// fdlibm depends on strict IEEE-754 evaluation: a fused multiply-add would change the result
+// of the polynomial evaluation below, so contraction must be off for this function.
+#if defined(__clang__)
+#pragma clang fp contract(off)
+#endif
+  static const double
+    ln2_hi = 6.93147180369123816490e-01,  /* 3fe62e42 fee00000 */
+    ln2_lo = 1.90821492927058770002e-10,  /* 3dea39ef 35793c76 */
+    two54  = 1.80143985094819840000e+16,  /* 43500000 00000000 */
+    Lg1 = 6.666666666666735130e-01,       /* 3FE55555 55555593 */
+    Lg2 = 3.999999999940941908e-01,       /* 3FD99999 9997FA04 */
+    Lg3 = 2.857142874366239149e-01,       /* 3FD24924 94229359 */
+    Lg4 = 2.222219843214978396e-01,       /* 3FCC71C5 1D8E78AF */
+    Lg5 = 1.818357216161805012e-01,       /* 3FC74664 96CB03DE */
+    Lg6 = 1.531383769920937332e-01,       /* 3FC39A09 D078C69F */
+    Lg7 = 1.479819860511658591e-01,       /* 3FC2F112 DF3E5244 */
+    zero = 0.0;
+
+  double hfsq,f,s,z,R,w,t1,t2,dk;
+  int32_t k,hx,i,j;
+  uint32_t lx;
+
+  hx = hi_word(x);
+  lx = lo_word(x);
+
+  k = 0;
+  if (hx < 0x00100000) {                       /* x < 2**-1022 */
+    // fdlibm writes these as -two54/zero and (x-x)/zero, which also raise the divide-by-zero
+    // and invalid flags. MSVC rejects a compile-time division by a zero constant (C2124), so
+    // return the same values directly. The estimators never call log() with these inputs.
+    if (((hx & 0x7fffffff) | lx) == 0) {                          /* log(+-0) = -inf */
+      return -std::numeric_limits<double>::infinity();
+    }
+    if (hx < 0) {                                                 /* log(-#) = NaN */
+      return std::numeric_limits<double>::quiet_NaN();
+    }
+    k -= 54; x *= two54;                       /* subnormal: scale up */
+    hx = hi_word(x);
+  }
+  if (hx >= 0x7ff00000) { return x+x; }
+  k += (hx>>20) - 1023;
+  hx &= 0x000fffff;
+  i = (hx + 0x95f64) & 0x100000;
+  set_hi_word(x, (uint32_t)(hx | (i ^ 0x3ff00000)));   /* normalize x or x/2 */
+  k += (i>>20);
+  f = x - 1.0;
+  if ((0x000fffff & (2+hx)) < 3) {             /* |f| < 2**-20 */
+    if (f == zero) {
+      if (k == 0) { return zero; }
+      dk = (double)k; return rnd(dk*ln2_hi) + rnd(dk*ln2_lo);
+    }
+    R = rnd(rnd(f*f)*rnd(0.5 - rnd(0.33333333333333333*f)));
+    if (k == 0) { return f-R; }
+    dk = (double)k; return rnd(dk*ln2_hi) - (rnd(R - rnd(dk*ln2_lo)) - f);
+  }
+  s = f/(2.0+f);
+  dk = (double)k;
+  z = s*s;
+  i = hx - 0x6147a;
+  w = z*z;
+  j = 0x6b851 - hx;
+  t1 = rnd(w*rnd(Lg2 + rnd(w*rnd(Lg4 + rnd(w*Lg6)))));
+  t2 = rnd(z*rnd(Lg1 + rnd(w*rnd(Lg3 + rnd(w*rnd(Lg5 + rnd(w*Lg7)))))));
+  i |= j;
+  R = t2 + t1;
+  if (i > 0) {
+    hfsq = rnd(0.5*f)*f;
+    if (k == 0) { return f - rnd(hfsq - rnd(s*(hfsq+R))); }
+    return rnd(dk*ln2_hi) - (rnd(hfsq - rnd(rnd(s*(hfsq+R)) + rnd(dk*ln2_lo))) - f);
+  } else {
+    if (k == 0) { return f - rnd(s*(f-R)); }
+    return rnd(dk*ln2_hi) - (rnd(rnd(s*(f-R)) - rnd(dk*ln2_lo)) - f);
+  }
+}
+
+} // namespace fdlibm
+#endif

hll/include/HarmonicNumbers-internal.hpp

@@ -21,6 +21,7 @@
 #define _HARMONICNUMBERS_INTERNAL_HPP_
 
 #include "HarmonicNumbers.hpp"
+#include "fdlibm_log.hpp"
 
 #include <cmath>
 
@@ -70,7 +71,7 @@ double HarmonicNumbers<A>::harmonicNumber(const uint64_t x_i) {
   } else {
     double x = static_cast<double>(x_i);
     double invSq = 1.0 / (x * x);
-    double sum = log(x) + EULER_MASCHERONI_CONSTANT + (1.0 / (2.0 * x));
+    double sum = fdlibm::log(x) + EULER_MASCHERONI_CONSTANT + (1.0 / (2.0 * x));
     /* note: the number of terms included from this series expansion is appropriate
        for the size of the exact table (25) and the precision of doubles */
     double pow = invSq; // now n^-2

hll/include/HllArray-internal.hpp

@@ -21,6 +21,7 @@
 #define _HLLARRAY_INTERNAL_HPP_
 
 #include "HllArray.hpp"
+#include "fdlibm_log.hpp"
 #include "HllUtil.hpp"
 #include "HarmonicNumbers.hpp"
 #include "CubicInterpolation.hpp"
@@ -568,7 +569,7 @@ double HllArray<A>::getHllBitMapEstimate() const {
 
   //This will eventually go away.
   if (numUnhitBuckets == 0) {
-    return configK * log(configK / 0.5);
+    return configK * fdlibm::log(configK / 0.5);
   }
 
   const uint32_t numHitBuckets = configK - numUnhitBuckets;
@@ -601,34 +602,37 @@ bool HllArray<A>::isRebuildKxqCurminFlag() const {
 template<typename A>
 void HllArray<A>::check_rebuild_kxq_cur_min() {
   if (!rebuild_kxq_curmin_) { return; }
+  // the deferred rebuild is only ever set on an HLL_8 union gadget. Guarding here also keeps
+  // this from ever rewriting curMin_ on an HLL_4 array, whose nibbles are stored relative to it.
+  if (this->getCurMode() != hll_mode::HLL || this->getTgtHllType() != target_hll_type::HLL_8) {
+    rebuild_kxq_curmin_ = false;
+    return;
+  }
 
-  uint8_t cur_min = 64;
-  uint32_t num_at_cur_min = 0;
+  uint32_t num_zeros = 0;
   double kxq0 = 1 << this->lgConfigK_;
   double kxq1 = 0;
 
-  auto it = this->begin(true); // want all points to adjust cur_min
+  auto it = this->begin(true); // want all slots, including the empty ones
   const auto end = this->end();
   while (it != end) {
     uint8_t v = HllUtil<A>::getValue(*it);
     if (v > 0) {
       if (v < 32) { kxq0 += INVERSE_POWERS_OF_2[v] - 1.0; }
       else        { kxq1 += INVERSE_POWERS_OF_2[v] - 1.0; }
-    }
-    if (v > cur_min) { ++it; continue; }
-    if (v < cur_min) {
-      cur_min = v;
-      num_at_cur_min = 1;
     } else {
-      ++num_at_cur_min;
-    }    
+      ++num_zeros;
+    }
     ++it;
   }
 
   kxq0_ = kxq0;
   kxq1_ = kxq1;
-  curMin_ = cur_min;
-  numAtCurMin_ = num_at_cur_min;
+  // HLL_8 convention: curMin is always 0 and numAtCurMin is the number of zero registers.
+  // That is the representation the incremental update path maintains, so the rebuilt state is
+  // indistinguishable from it and the timing of this rebuild is not observable in the image.
+  curMin_ = 0;
+  numAtCurMin_ = num_zeros;
   rebuild_kxq_curmin_ = false;
   // HipAccum is not affected
 

hll/include/HllUtil.hpp

@@ -50,7 +50,8 @@ static const uint8_t OUT_OF_ORDER_FLAG_MASK   = 16;
 // are free.
 static const uint8_t RESERVED_FLAG_MASK_32    = 32;
 // Deprecated alias for the bit above, retained for source compatibility. Do not use.
-// (No [[deprecated]] attribute: this library targets C++11, where it is unavailable.)
+// Added 2026-09-07 (#521), first shipping in the release after 5.2.0. No [[deprecated]]:
+// this library targets C++11, so the alias warns no one.
 static const uint8_t FULL_SIZE_FLAG_MASK      = RESERVED_FLAG_MASK_32;
 
 static const uint32_t PREAMBLE_INTS_BYTE = 0;
@@ -90,8 +91,8 @@ static const uint32_t EMPTY = 0;
 static const uint8_t MIN_LOG_K = 4;
 static const uint8_t MAX_LOG_K = 21;
 
-static const double HLL_HIP_RSE_FACTOR = 0.8325546; // sqrt(ln(2))
-static const double HLL_NON_HIP_RSE_FACTOR = 1.03896; // sqrt((3 * ln(2)) - 1)
+static const double HLL_HIP_RSE_FACTOR = 0.8325546111576977; // sqrt(ln(2))
+static const double HLL_NON_HIP_RSE_FACTOR = 1.0389617614136892; // sqrt((3 * ln(2)) - 1)
 static const double COUPON_RSE_FACTOR = 0.409; // at transition point not the asymptote
 static const double COUPON_RSE = COUPON_RSE_FACTOR / (1 << 13);
 

hll/test/CMakeLists.txt

@@ -43,6 +43,7 @@ target_sources(hll_test
     CrossCountingTest.cpp
     HllArrayTest.cpp
     HllFullSizeTest.cpp
+    HllKxqRebuildTest.cpp
     HllSketchTest.cpp
     HllUnionTest.cpp
     TablesTest.cpp