From 618d071bf457dfa18f90247a962541942d952c53 Mon Sep 17 00:00:00 2001 From: Andrew Priestley Date: Sat, 3 Oct 2026 18:38:55 -0400 Subject: [PATCH] vu1: cut interpreter overhead (~2.3x faster), bit-exact The VU1 interpreter was ~90% of the frame in KFIV's 3D areas. Same cycle and flag model, cheaper bookkeeping: - Hazards: each decoded pair carries the ready-time slots it reads and writes (VF lanes, VI, ACC in one array), instead of re-deriving them from the usages with nested loops on every issue. - VF/VI/ACC results are written back at issue. These registers are interlocked: every reader already stalls until the write's latency has elapsed, so the delayed write-back pipelines (and their copy-out / restore per instruction) could never be observed. A program still ends only once its last write would have landed (write-back horizon), so cycle counts are unchanged. Flags, Q, P and stores keep their pipelines. - Pipelines track occupied slots in bitmasks and the earliest pending commit, so cycles with nothing due skip the commit scan, idle cycles are skipped, and commits still run in slot order. - FMAC flags: a single-rounding (toward zero) result that is normal and below FLT_MAX can only report its sign, so the exact recomputation is skipped; ADD/SUB results that round to zero are exactly zero. Otherwise the exact result is computed in double, which classifies like the long double (a float product is exact in double); a sum that lands exactly on FLT_MAX still takes the long double. Operands are decoded once per instruction and normalized with SSE4.1 where available. - XGKICK copies a qword at a time; decoded pairs are used by reference. Verified by running the original interpreter in lockstep on every microprogram of a KFIV session (720k runs into gameplay): registers, flags, cycle counts, VU1 memory and XGKICK output identical. --- ps2xRuntime/include/runtime/ps2_vu1.h | 116 ++-- ps2xRuntime/src/lib/vu/ps2_vu1_core.cpp | 825 +++++++++++------------ ps2xRuntime/src/lib/vu/ps2_vu1_upper.cpp | 43 +- 3 files changed, 515 insertions(+), 469 deletions(-) diff --git a/ps2xRuntime/include/runtime/ps2_vu1.h b/ps2xRuntime/include/runtime/ps2_vu1.h index 67c8184fd..c19926aa9 100644 --- a/ps2xRuntime/include/runtime/ps2_vu1.h +++ b/ps2xRuntime/include/runtime/ps2_vu1.h @@ -3,6 +3,8 @@ #include #include +#include +#include class GS; class PS2Memory; @@ -115,8 +117,24 @@ class VU1Interpreter bool tBit = false; uint8_t upperVfShadowReg = 0; uint8_t suppressedLowerVf = 0; + // Precomputed from the usages: ready-time slots (see m_ready) the + // pair reads, and the slots it writes with their latencies, in the + // order markPairWrites applies them. + uint8_t readSlotCount = 0; + uint8_t writeSlotCount = 0; + uint8_t firstViWrite = 0; + uint8_t maxWriteLatency = 0; + std::array readSlots{}; + std::array writeSlots{}; + std::array writeLatencies{}; }; + // m_ready slots: VF register r lane c, then VI registers, then ACC lanes. + static constexpr uint32_t kReadyVfBase = 0u; + static constexpr uint32_t kReadyViBase = 128u; + static constexpr uint32_t kReadyAccBase = 144u; + static constexpr uint32_t kReadySlotCount = 148u; + struct FlagPipelineEntry { uint64_t readyCycle = 0; @@ -149,34 +167,6 @@ class VU1Interpreter bool valid = false; }; - struct PendingVfWrite - { - uint64_t readyCycle = 0; - uint64_t sequence = 0; - std::array value{}; - uint8_t reg = 0; - uint8_t laneMask = 0; - bool valid = false; - }; - - struct PendingViWrite - { - uint64_t readyCycle = 0; - uint64_t sequence = 0; - int32_t value = 0; - uint8_t reg = 0; - bool valid = false; - }; - - struct PendingAccWrite - { - uint64_t readyCycle = 0; - uint64_t sequence = 0; - std::array value{}; - uint8_t laneMask = 0; - bool valid = false; - }; - struct XgkickPipeline { static constexpr uint32_t kBufferSize = 0x10000u; @@ -195,9 +185,6 @@ class VU1Interpreter static constexpr uint32_t kAccForwardLatency = 1u; static constexpr uint32_t kMaxFlagEntries = 8u; static constexpr uint32_t kMaxPendingStores = 8u; - static constexpr uint32_t kMaxPendingVfWrites = 16u; - static constexpr uint32_t kMaxPendingViWrites = 8u; - static constexpr uint32_t kMaxPendingAccWrites = 8u; static constexpr uint32_t kMaxDecodedPairs = 0x4000u / 8u; Unit m_unit; @@ -213,22 +200,35 @@ class VU1Interpreter ScalarPipelineEntry m_fdiv{}; std::array m_efu{}; std::array m_storePipeline{}; - std::array m_vfWritePipeline{}; - std::array m_viWritePipeline{}; - std::array m_accWritePipeline{}; XgkickPipeline m_xgkick{}; - std::array, 32> m_vfReady{}; - std::array m_viReady{}; - std::array m_accReady{}; - std::array, 32> m_vfLatestWrite{}; - std::array m_viLatestWrite{}; - std::array m_accLatestWrite{}; + // Cycle at which each VF lane / VI register / ACC lane write becomes + // readable by a following instruction. + std::array m_ready{}; uint64_t m_cycle = 0; - uint64_t m_nextWriteSequence = 0; uint64_t m_efuResourceReady = 0; + // Earliest readyCycle of any valid pipeline entry (max when none), so + // the per-cycle commit and pending checks don't scan every pipeline. + uint64_t m_nextCommitCycle = std::numeric_limits::max(); + // Latest cycle at which an issued VF/VI/ACC write lands. The values are + // written back at issue (see run), but a program only ends once its + // writes would have committed. + uint64_t m_writebackHorizon = 0; + // Occupied slots per pipeline (bit i = entry i valid); commits walk set + // bits in ascending slot order, the same order as a full array scan. + uint32_t m_flagMask = 0; + uint32_t m_efuMask = 0; + uint32_t m_storeMask = 0; + DecodedInstructionPair m_uncachedDecoded{}; uint32_t m_workingClip = 0; uint32_t m_currentUpperInstruction = 0; + // Normalized operands of the executing upper instruction (set by + // execUpper, read by normalizeFmacResult). + const float *m_fmacVs = nullptr; + const float *m_fmacVt = nullptr; + const float *m_fmacAcc = nullptr; + float m_fmacQ = 0.0f; + float m_fmacI = 0.0f; int32_t m_viBranchBackupValue = 0; uint8_t m_viBranchBackupReg = 0; bool m_viBranchBackupValid = false; @@ -250,7 +250,7 @@ class VU1Interpreter static void addVfWrite(InstructionUsage &usage, uint8_t reg, uint8_t lanes); static uint8_t vfReadLanes(const InstructionUsage &usage, uint8_t reg); DecodedInstructionPair decodeInstructionPair(const uint8_t *vuCode, uint32_t pc) const; - DecodedInstructionPair getDecodedInstructionPairForPc(const uint8_t *vuCode, uint32_t codeSize, PS2Memory *memory, uint32_t pc); + const DecodedInstructionPair &getDecodedInstructionPairForPc(const uint8_t *vuCode, uint32_t codeSize, PS2Memory *memory, uint32_t pc); void rebuildDecodedCodeCache(const uint8_t *vuCode, uint32_t codeSize, const PS2Memory *memory, uint64_t generation); void execUpper(uint32_t instr); @@ -260,10 +260,11 @@ class VU1Interpreter void applyDestAcc(const float *result, uint8_t dest); void applyFmacDest(float *dst, float *result, uint8_t dest); void applyFmacDestAcc(float *result, uint8_t dest); - void normalizeFmacResult(float *result, uint8_t dest, uint8_t laneFlags[4]); - bool calculateFmacExactResult(uint32_t component, long double &result) const; - uint8_t normalizeFmacExactResult(float &value, long double exactResult) const; - uint32_t calculateFmacProductSticky(uint8_t dest) const; + void normalizeFmacResult(float *result, uint8_t dest, uint8_t laneFlags[4], uint32_t &productSticky); + template + bool calculateFmacExactResult(uint32_t component, Real &result) const; + template + uint8_t normalizeFmacExactResult(float &value, Real exactResult) const; void updateFmacFlags(const uint8_t laneFlags[4], uint8_t dest, uint32_t extraSticky); void queueFsset(uint16_t immediate); void queueClip(uint32_t clip); @@ -271,13 +272,16 @@ class VU1Interpreter void queueQ(float value, uint32_t latency, uint32_t statusDi); void queueP(float value, uint32_t latency); void queueStore(uint32_t address, const uint32_t words[4], uint8_t laneMask); - void queueVfWrite(uint8_t reg, uint8_t laneMask, const float value[4], uint32_t latency); - void queueViWrite(uint8_t reg, int32_t value, uint32_t latency); - void queueAccWrite(uint8_t laneMask, const float value[4], uint32_t latency); void startXgkick(uint32_t qwordAddress); void resetScheduler(); void commitReadyPipelines(); + void commitDuePipelines(); + void noteReadyCycle(uint64_t readyCycle) + { + if (readyCycle < m_nextCommitCycle) + m_nextCommitCycle = readyCycle; + } void advanceOneCycle(); void advanceTo(uint64_t targetCycle); void flushPipelines(); @@ -287,7 +291,19 @@ class VU1Interpreter void markPairWrites(const DecodedInstructionPair &decoded); bool pipelinesPending() const; - float normalizeOperand(float value) const; + // Denormals flush to signed zero; Inf/NaN clamp to the signed max. + float normalizeOperand(float value) const + { + uint32_t bits = 0; + std::memcpy(&bits, &value, sizeof(bits)); + const uint32_t exponent = (bits >> 23) & 0xFFu; + if (exponent == 0u) + bits &= 0x80000000u; + else if (exponent == 0xFFu) + bits = (bits & 0x80000000u) | 0x7F7FFFFFu; + std::memcpy(&value, &bits, sizeof(value)); + return value; + } float normalizeResult(float value, uint32_t &laneFlags) const; uint32_t microAddressMask() const; int32_t readBranchVi(uint8_t reg) const; diff --git a/ps2xRuntime/src/lib/vu/ps2_vu1_core.cpp b/ps2xRuntime/src/lib/vu/ps2_vu1_core.cpp index bd75bba31..80d6a0cc6 100644 --- a/ps2xRuntime/src/lib/vu/ps2_vu1_core.cpp +++ b/ps2xRuntime/src/lib/vu/ps2_vu1_core.cpp @@ -5,6 +5,7 @@ #include "ps2_vu1_detail.h" #include +#include #include #include #include @@ -20,6 +21,99 @@ namespace { return static_cast(1u << (3u - component)); } + + // Claims the lowest free slot of an N-entry pipeline; -1 when full. + template + int claimSlot(uint32_t &mask) + { + const uint32_t freeSlots = ~mask & ((1u << N) - 1u); + if (freeSlots == 0u) + return -1; + const int slot = std::countr_zero(freeSlots); + mask |= 1u << slot; + return slot; + } + + // What an FMAC upper instruction computes per lane, from its normalized + // operands: A is VS (VS crossed for OPMULA/OPMSUB), B the broadcast lane, + // Q, I, VT, or VT crossed. Mirrors calculateFmacExactResult. + enum class FmacOp : uint8_t + { + None, + Add, // A + B + Sub, // A - B + Mul, // A * B + MAdd, // ACC + A * B + MSub, // ACC - A * B + OuterMul, // A * B, w = 0 (OPMULA) + OuterMSub // ACC - A * B, w = 0 (OPMSUB) + }; + + enum class FmacSource : uint8_t + { + Broadcast, + Q, + I, + Full, + Cross, + }; + + struct FmacDecode + { + FmacOp op = FmacOp::None; + FmacSource source = FmacSource::Full; + uint8_t broadcast = 0; + }; + + FmacDecode decodeFmac(uint32_t upper) + { + const uint32_t op = upper & 0x3Fu; + const bool special = op >= 0x3Cu; + const uint32_t code = special ? ((upper & 3u) | ((upper >> 4) & 0x7Cu)) : op; + const uint8_t bc = static_cast(code & 3u); + if (code <= 0x03u) + return {FmacOp::Add, FmacSource::Broadcast, bc}; + if (code <= 0x07u) + return {FmacOp::Sub, FmacSource::Broadcast, bc}; + if (code <= 0x0Bu) + return {FmacOp::MAdd, FmacSource::Broadcast, bc}; + if (code <= 0x0Fu) + return {FmacOp::MSub, FmacSource::Broadcast, bc}; + if (code >= 0x18u && code <= 0x1Bu) + return {FmacOp::Mul, FmacSource::Broadcast, bc}; + switch (code) + { + case 0x1Cu: return {FmacOp::Mul, FmacSource::Q}; + case 0x1Eu: return {FmacOp::Mul, FmacSource::I}; + case 0x20u: return {FmacOp::Add, FmacSource::Q}; + case 0x21u: return {FmacOp::MAdd, FmacSource::Q}; + case 0x22u: return {FmacOp::Add, FmacSource::I}; + case 0x23u: return {FmacOp::MAdd, FmacSource::I}; + case 0x24u: return {FmacOp::Sub, FmacSource::Q}; + case 0x25u: return {FmacOp::MSub, FmacSource::Q}; + case 0x26u: return {FmacOp::Sub, FmacSource::I}; + case 0x27u: return {FmacOp::MSub, FmacSource::I}; + case 0x28u: return {FmacOp::Add, FmacSource::Full}; + case 0x29u: return {FmacOp::MAdd, FmacSource::Full}; + case 0x2Au: return {FmacOp::Mul, FmacSource::Full}; + case 0x2Cu: return {FmacOp::Sub, FmacSource::Full}; + case 0x2Du: return {FmacOp::MSub, FmacSource::Full}; + case 0x2Eu: return {special ? FmacOp::OuterMul : FmacOp::OuterMSub, FmacSource::Cross}; + default: return {}; + } + } + + bool isProductSum(FmacOp op) + { + return op == FmacOp::MAdd || op == FmacOp::MSub || op == FmacOp::OuterMSub; + } + + inline int nextSlot(uint32_t &bits) + { + const int slot = std::countr_zero(bits); + bits &= bits - 1u; + return slot; + } } void VU1Interpreter::addVfRead(InstructionUsage &usage, uint8_t reg, uint8_t lanes) @@ -70,18 +164,14 @@ void VU1Interpreter::resetScheduler() m_fdiv = {}; m_efu = {}; m_storePipeline = {}; - m_vfWritePipeline = {}; - m_viWritePipeline = {}; - m_accWritePipeline = {}; m_xgkick = {}; - m_vfReady = {}; - m_viReady = {}; - m_accReady = {}; - m_vfLatestWrite = {}; - m_viLatestWrite = {}; - m_accLatestWrite = {}; - m_nextWriteSequence = 0; + m_ready = {}; m_efuResourceReady = 0; + m_nextCommitCycle = std::numeric_limits::max(); + m_writebackHorizon = 0; + m_flagMask = 0; + m_efuMask = 0; + m_storeMask = 0; m_workingClip = m_state.clip; m_viBranchBackupValue = 0; m_viBranchBackupReg = 0; @@ -106,23 +196,6 @@ float VU1Interpreter::broadcast(const float *vf, uint8_t bc) return normalizeOperand(vf[bc & 3u]); } -float VU1Interpreter::normalizeOperand(float value) const -{ - uint32_t bits = 0; - std::memcpy(&bits, &value, sizeof(bits)); - const uint32_t exponent = (bits >> 23) & 0xFFu; - if (exponent == 0u) - { - bits &= 0x80000000u; - } - else if (exponent == 0xFFu) - { - bits = (bits & 0x80000000u) | 0x7F7FFFFFu; - } - std::memcpy(&value, &bits, sizeof(value)); - return value; -} - float VU1Interpreter::normalizeResult(float value, uint32_t &laneFlags) const { uint32_t bits = 0; @@ -195,29 +268,137 @@ void VU1Interpreter::applyDestAcc(const float *result, uint8_t dest) } void VU1Interpreter::normalizeFmacResult(float *result, uint8_t dest, - uint8_t laneFlags[4]) + uint8_t laneFlags[4], uint32_t &productSticky) { + productSticky = 0u; + const FmacDecode fmac = decodeFmac(m_currentUpperInstruction); + const bool single = fmac.op == FmacOp::Add || fmac.op == FmacOp::Sub || + fmac.op == FmacOp::Mul || fmac.op == FmacOp::OuterMul; + const bool productSum = isProductSum(fmac.op); + + // Operands as calculateFmacExactResult derives them, from the values + // execUpper already normalized. + bool haveOperands = false; + double a[4]{}, b[4]{}, acc[4]{}; + auto gatherOperands = [&]() + { + if (haveOperands) + return; + haveOperands = true; + static constexpr uint8_t crossLeft[4] = {1u, 2u, 0u, 3u}; + static constexpr uint8_t crossRight[4] = {2u, 0u, 1u, 3u}; + const bool cross = fmac.source == FmacSource::Cross; + for (uint32_t lane = 0; lane < 4u; ++lane) + { + a[lane] = m_fmacVs[cross ? crossLeft[lane] : lane]; + acc[lane] = m_fmacAcc[lane]; + } + switch (fmac.source) + { + case FmacSource::Broadcast: + for (uint32_t lane = 0; lane < 4u; ++lane) + b[lane] = m_fmacVt[fmac.broadcast]; + break; + case FmacSource::Q: + for (uint32_t lane = 0; lane < 4u; ++lane) + b[lane] = m_fmacQ; + break; + case FmacSource::I: + for (uint32_t lane = 0; lane < 4u; ++lane) + b[lane] = m_fmacI; + break; + case FmacSource::Full: + for (uint32_t lane = 0; lane < 4u; ++lane) + b[lane] = m_fmacVt[lane]; + break; + case FmacSource::Cross: + for (uint32_t lane = 0; lane < 4u; ++lane) + b[lane] = m_fmacVt[crossRight[lane]]; + break; + } + }; + for (uint32_t component = 0; component < 4u; ++component) { laneFlags[component] = 0u; if ((dest & laneForComponent(component)) == 0u) continue; - long double exactResult = 0.0L; - if (calculateFmacExactResult(component, exactResult)) + if (productSum) { - laneFlags[component] = normalizeFmacExactResult(result[component], exactResult); + // Every product condition accumulates into the sticky flags (the + // current flags come from the sum). A float product is exact in + // double. + gatherOperands(); + float ignored = 0.0f; + productSticky |= normalizeFmacExactResult(ignored, a[component] * b[component]) & 0xFu; + } + + // With a single rounding toward zero, a normal result below FLT_MAX + // means the exact result is nonzero, same-signed and in range: the + // exact check below would only report the sign and keep the value. + if (single) + { + uint32_t bits = 0; + std::memcpy(&bits, &result[component], sizeof(bits)); + const uint32_t exponent = (bits >> 23) & 0xFFu; + if (exponent != 0u && exponent != 0xFFu && (bits & 0x7FFFFFFFu) != 0x7F7FFFFFu) + { + laneFlags[component] = (bits & 0x80000000u) != 0u ? 0x2u : 0u; + continue; + } + // Operands are zero or normal, so a sum or difference is a + // multiple of the smallest denormal: one that rounds to zero is + // exactly zero, signed by the same IEEE rule as the exact sum. + if ((bits & 0x7FFFFFFFu) == 0u && (fmac.op == FmacOp::Add || fmac.op == FmacOp::Sub)) + { + laneFlags[component] = (bits & 0x80000000u) != 0u ? 0x3u : 0x1u; + continue; + } + } + + if (fmac.op == FmacOp::None) + { + uint32_t flags = 0u; + result[component] = normalizeResult(result[component], flags); + laneFlags[component] = static_cast(flags); continue; } - uint32_t flags = 0u; - result[component] = normalizeResult(result[component], flags); - laneFlags[component] = static_cast(flags); + // The exact result in double classifies like the long double one: a + // float product is exact in double, and zero, sign and FLT_MIN are + // preserved by rounding toward zero. Only a sum that rounds to exactly + // FLT_MAX may hide an overflow, so that case takes the long double. + gatherOperands(); + const double product = a[component] * b[component]; + double exactResult = 0.0; + switch (fmac.op) + { + case FmacOp::Add: exactResult = a[component] + b[component]; break; + case FmacOp::Sub: exactResult = a[component] - b[component]; break; + case FmacOp::Mul: exactResult = product; break; + case FmacOp::MAdd: exactResult = acc[component] + product; break; + case FmacOp::MSub: exactResult = acc[component] - product; break; + case FmacOp::OuterMul: exactResult = component == 3u ? 0.0 : product; break; + case FmacOp::OuterMSub: exactResult = component == 3u ? 0.0 : acc[component] - product; break; + case FmacOp::None: break; + } + if (std::fabs(exactResult) == static_cast(std::numeric_limits::max())) + { + long double preciseResult = 0.0L; + calculateFmacExactResult(component, preciseResult); + laneFlags[component] = normalizeFmacExactResult(result[component], preciseResult); + } + else + { + laneFlags[component] = normalizeFmacExactResult(result[component], exactResult); + } } } +template bool VU1Interpreter::calculateFmacExactResult(uint32_t component, - long double &result) const + Real &result) const { const uint32_t upper = m_currentUpperInstruction; const uint8_t op = static_cast(upper & 0x3Fu); @@ -229,7 +410,7 @@ bool VU1Interpreter::calculateFmacExactResult(uint32_t component, const auto operand = [this](float value) { - return static_cast(normalizeOperand(value)); + return static_cast(normalizeOperand(value)); }; const auto vs = [&](uint32_t lane) { @@ -244,8 +425,8 @@ bool VU1Interpreter::calculateFmacExactResult(uint32_t component, return operand(m_state.acc[lane]); }; - const long double q = operand(m_state.q); - const long double i = operand(m_state.i); + const Real q = operand(m_state.q); + const Real i = operand(m_state.i); if (op < 0x3Cu) { @@ -313,7 +494,7 @@ bool VU1Interpreter::calculateFmacExactResult(uint32_t component, static constexpr uint8_t left[4] = {1u, 2u, 0u, 3u}; static constexpr uint8_t right[4] = {2u, 0u, 1u, 3u}; result = component == 3u - ? 0.0L + ? Real(0) : acc(component) - vs(left[component]) * vt(right[component]); break; } @@ -388,7 +569,7 @@ bool VU1Interpreter::calculateFmacExactResult(uint32_t component, static constexpr uint8_t left[4] = {1u, 2u, 0u, 3u}; static constexpr uint8_t right[4] = {2u, 0u, 1u, 3u}; result = component == 3u - ? 0.0L + ? Real(0) : vs(left[component]) * vt(right[component]); break; } @@ -399,17 +580,18 @@ bool VU1Interpreter::calculateFmacExactResult(uint32_t component, return true; } +template uint8_t VU1Interpreter::normalizeFmacExactResult(float &value, - long double exactResult) const + Real exactResult) const { const bool negative = std::signbit(exactResult); - const long double magnitude = std::fabs(exactResult); - const long double maximum = static_cast(std::numeric_limits::max()); - const long double minimum = static_cast(std::numeric_limits::min()); + const Real magnitude = std::fabs(exactResult); + const Real maximum = static_cast(std::numeric_limits::max()); + const Real minimum = static_cast(std::numeric_limits::min()); uint8_t flags = negative ? 0x2u : 0u; uint32_t bits = negative ? 0x80000000u : 0u; - if (magnitude == 0.0L) + if (magnitude == Real(0)) { flags |= 0x1u; std::memcpy(&value, &bits, sizeof(value)); @@ -429,65 +611,6 @@ uint8_t VU1Interpreter::normalizeFmacExactResult(float &value, return flags; } -uint32_t VU1Interpreter::calculateFmacProductSticky(uint8_t dest) const -{ - uint32_t extraSticky = 0u; - const uint32_t upper = m_currentUpperInstruction; - const uint8_t op = static_cast(upper & 0x3Fu); - const uint8_t special = op >= 0x3Cu ? static_cast((upper & 3u) | ((upper >> 4) & 0x7Cu)) : 0xFFu; - const bool productSum = - (op >= 0x08u && op <= 0x0Fu) || - op == 0x21u || op == 0x23u || op == 0x25u || op == 0x27u || - op == 0x29u || op == 0x2Du || op == 0x2Eu || - (special >= 0x08u && special <= 0x0Fu) || - special == 0x21u || special == 0x23u || special == 0x25u || - special == 0x27u || special == 0x29u || special == 0x2Du; - if (!productSum) - return 0u; - - const uint8_t fs = FS(upper); - const uint8_t ft = FT(upper); - for (uint32_t component = 0; component < 4u; ++component) - { - if ((dest & laneForComponent(component)) == 0u) - continue; - static constexpr uint8_t crossLeft[4] = {1u, 2u, 0u, 3u}; - static constexpr uint8_t crossRight[4] = {2u, 0u, 1u, 3u}; - const uint8_t leftComponent = op == 0x2Eu ? crossLeft[component] : static_cast(component); - const float left = normalizeOperand(m_state.vf[fs][leftComponent]); - float right = 0.0f; - if ((op >= 0x08u && op <= 0x0Fu) || (special >= 0x08u && special <= 0x0Fu)) - { - right = normalizeOperand(m_state.vf[ft][(op >= 0x08u && op <= 0x0Fu ? op : special) & 3u]); - } - else if (op == 0x21u || op == 0x25u || special == 0x21u || special == 0x25u) - { - right = normalizeOperand(m_state.q); - } - else if (op == 0x23u || op == 0x27u || special == 0x23u || special == 0x27u) - { - right = normalizeOperand(m_state.i); - } - else if (op == 0x2Eu) - { - right = normalizeOperand(m_state.vf[ft][crossRight[component]]); - } - else - { - right = normalizeOperand(m_state.vf[ft][component]); - } - - float product = left * right; - const long double exactProduct = static_cast(left) * static_cast(right); - const uint8_t productFlags = normalizeFmacExactResult(product, exactProduct); - // Product-sum instructions report Z/S/U/O from the add/subtract result - // as current flags, while every product condition accumulates into the - // corresponding sticky flag. - extraSticky |= productFlags & 0xFu; - } - return extraSticky; -} - void VU1Interpreter::updateFmacFlags(const uint8_t laneFlags[4], uint8_t dest, uint32_t extraSticky) { @@ -514,25 +637,19 @@ void VU1Interpreter::updateFmacFlags(const uint8_t laneFlags[4], uint8_t dest, status |= flags; } - FlagPipelineEntry *entry = nullptr; - for (FlagPipelineEntry &candidate : m_flagPipeline) - { - if (!candidate.valid) - { - entry = &candidate; - break; - } - } - if (!entry) + const int slot = claimSlot(m_flagMask); + if (slot < 0) { reportReservedInstruction(true, 0xFFFFFFFFu); return; } + FlagPipelineEntry *entry = &m_flagPipeline[static_cast(slot)]; *entry = {}; entry->valid = true; entry->issueCycle = m_cycle; entry->readyCycle = m_cycle + kFmacLatency; + noteReadyCycle(entry->readyCycle); entry->mac = mac; entry->status = status; entry->extraSticky = extraSticky; @@ -543,35 +660,39 @@ void VU1Interpreter::updateFmacFlags(const uint8_t laneFlags[4], uint8_t dest, void VU1Interpreter::applyFmacDest(float *dst, float *result, uint8_t dest) { uint8_t laneFlags[4]{}; - normalizeFmacResult(result, dest, laneFlags); - updateFmacFlags(laneFlags, dest, calculateFmacProductSticky(dest)); + uint32_t productSticky = 0u; + normalizeFmacResult(result, dest, laneFlags, productSticky); + updateFmacFlags(laneFlags, dest, productSticky); applyDest(dst, result, dest); } void VU1Interpreter::applyFmacDestAcc(float *result, uint8_t dest) { uint8_t laneFlags[4]{}; - normalizeFmacResult(result, dest, laneFlags); - updateFmacFlags(laneFlags, dest, calculateFmacProductSticky(dest)); + uint32_t productSticky = 0u; + normalizeFmacResult(result, dest, laneFlags, productSticky); + updateFmacFlags(laneFlags, dest, productSticky); applyDestAcc(result, dest); } void VU1Interpreter::queueFsset(uint16_t immediate) { - for (FlagPipelineEntry &entry : m_flagPipeline) + for (uint32_t bits = m_flagMask; bits != 0u;) { - if (entry.valid && entry.issueCycle == m_cycle) + FlagPipelineEntry &entry = m_flagPipeline[static_cast(nextSlot(bits))]; + if (entry.issueCycle == m_cycle) entry.writesStatus = false; } - for (FlagPipelineEntry &entry : m_flagPipeline) + if (const int slot = claimSlot(m_flagMask); slot >= 0) { - if (!entry.valid) + FlagPipelineEntry &entry = m_flagPipeline[static_cast(slot)]; { entry = {}; entry.valid = true; entry.issueCycle = m_cycle; entry.readyCycle = m_cycle + kFmacLatency; + noteReadyCycle(entry.readyCycle); entry.status = static_cast(immediate) & 0xFC0u; entry.writesSticky = true; return; @@ -583,14 +704,15 @@ void VU1Interpreter::queueFsset(uint16_t immediate) void VU1Interpreter::queueClip(uint32_t clip) { m_workingClip = ((m_workingClip << 6) | (clip & 0x3Fu)) & 0xFFFFFFu; - for (FlagPipelineEntry &entry : m_flagPipeline) + if (const int slot = claimSlot(m_flagMask); slot >= 0) { - if (!entry.valid) + FlagPipelineEntry &entry = m_flagPipeline[static_cast(slot)]; { entry = {}; entry.valid = true; entry.issueCycle = m_cycle; entry.readyCycle = m_cycle + kFmacLatency; + noteReadyCycle(entry.readyCycle); entry.clip = m_workingClip; entry.writesClip = true; return; @@ -602,19 +724,21 @@ void VU1Interpreter::queueClip(uint32_t clip) void VU1Interpreter::queueFcset(uint32_t clip) { m_workingClip = clip & 0xFFFFFFu; - for (FlagPipelineEntry &entry : m_flagPipeline) + for (uint32_t bits = m_flagMask; bits != 0u;) { - if (entry.valid && entry.issueCycle == m_cycle) + FlagPipelineEntry &entry = m_flagPipeline[static_cast(nextSlot(bits))]; + if (entry.issueCycle == m_cycle) entry.writesClip = false; } - for (FlagPipelineEntry &entry : m_flagPipeline) + if (const int slot = claimSlot(m_flagMask); slot >= 0) { - if (!entry.valid) + FlagPipelineEntry &entry = m_flagPipeline[static_cast(slot)]; { entry = {}; entry.valid = true; entry.issueCycle = m_cycle; entry.readyCycle = m_cycle + kFmacLatency; + noteReadyCycle(entry.readyCycle); entry.clip = m_workingClip; entry.writesClip = true; return; @@ -629,6 +753,7 @@ void VU1Interpreter::queueQ(float value, uint32_t latency, uint32_t statusDi) value = normalizeResult(value, ignoredFlags); m_fdiv.valid = true; m_fdiv.readyCycle = m_cycle + latency; + noteReadyCycle(m_fdiv.readyCycle); m_fdiv.value = value; m_fdiv.statusDi = statusDi & 0x30u; } @@ -637,12 +762,13 @@ void VU1Interpreter::queueP(float value, uint32_t latency) { uint32_t ignoredFlags = 0u; value = normalizeResult(value, ignoredFlags); - for (ScalarPipelineEntry &entry : m_efu) + if (const int slot = claimSlot<2u>(m_efuMask); slot >= 0) { - if (!entry.valid) + ScalarPipelineEntry &entry = m_efu[static_cast(slot)]; { entry.valid = true; entry.readyCycle = m_cycle + latency; + noteReadyCycle(entry.readyCycle); entry.value = value; // EFU throughput is one cycle shorter than result visibility. m_efuResourceReady = m_cycle + (latency > 0u ? latency - 1u : 0u); @@ -654,12 +780,13 @@ void VU1Interpreter::queueP(float value, uint32_t latency) void VU1Interpreter::queueStore(uint32_t address, const uint32_t words[4], uint8_t laneMask) { - for (PendingStore &store : m_storePipeline) + if (const int slot = claimSlot(m_storeMask); slot >= 0) { - if (!store.valid) + PendingStore &store = m_storePipeline[static_cast(slot)]; { store.valid = true; store.readyCycle = m_cycle + 1u; + noteReadyCycle(store.readyCycle); store.address = address; store.laneMask = laneMask; std::copy(words, words + 4, store.words.begin()); @@ -669,85 +796,25 @@ void VU1Interpreter::queueStore(uint32_t address, const uint32_t words[4], uint8 reportReservedInstruction(false, 0xFFFFFFFCu); } -void VU1Interpreter::queueVfWrite(uint8_t reg, uint8_t laneMask, - const float value[4], uint32_t latency) +void VU1Interpreter::commitReadyPipelines() { - if (reg == 0u || laneMask == 0u) - return; - for (PendingVfWrite &write : m_vfWritePipeline) - { - if (!write.valid) - { - write = {}; - write.valid = true; - write.readyCycle = m_cycle + latency; - write.sequence = ++m_nextWriteSequence; - write.reg = reg; - write.laneMask = laneMask; - std::copy(value, value + 4, write.value.begin()); - for (uint32_t component = 0; component < 4u; ++component) - { - if ((laneMask & laneForComponent(component)) != 0u) - m_vfLatestWrite[reg][component] = write.sequence; - } - return; - } - } - reportReservedInstruction(false, 0xFFFFFFF7u); + if (m_cycle >= m_nextCommitCycle) + commitDuePipelines(); } -void VU1Interpreter::queueViWrite(uint8_t reg, int32_t value, uint32_t latency) +void VU1Interpreter::commitDuePipelines() { - if (reg == 0u) - return; - for (PendingViWrite &write : m_viWritePipeline) - { - if (!write.valid) - { - write = {}; - write.valid = true; - write.readyCycle = m_cycle + latency; - write.sequence = ++m_nextWriteSequence; - write.reg = reg; - write.value = value; - m_viLatestWrite[reg] = write.sequence; - return; - } - } - reportReservedInstruction(false, 0xFFFFFFF6u); -} + uint64_t next = std::numeric_limits::max(); -void VU1Interpreter::queueAccWrite(uint8_t laneMask, const float value[4], uint32_t latency) -{ - if (laneMask == 0u) - return; - for (PendingAccWrite &write : m_accWritePipeline) + for (uint32_t bits = m_flagMask; bits != 0u;) { - if (!write.valid) + const int slot = nextSlot(bits); + FlagPipelineEntry &entry = m_flagPipeline[static_cast(slot)]; + if (entry.readyCycle > m_cycle) { - write = {}; - write.valid = true; - write.readyCycle = m_cycle + latency; - write.sequence = ++m_nextWriteSequence; - write.laneMask = laneMask; - std::copy(value, value + 4, write.value.begin()); - for (uint32_t component = 0; component < 4u; ++component) - { - if ((laneMask & laneForComponent(component)) != 0u) - m_accLatestWrite[component] = write.sequence; - } - return; - } - } - reportReservedInstruction(true, 0xFFFFFFF5u); -} - -void VU1Interpreter::commitReadyPipelines() -{ - for (FlagPipelineEntry &entry : m_flagPipeline) - { - if (!entry.valid || entry.readyCycle > m_cycle) + next = std::min(next, entry.readyCycle); continue; + } if (entry.writesMac) m_state.mac = entry.mac; @@ -763,29 +830,47 @@ void VU1Interpreter::commitReadyPipelines() if (entry.writesClip) m_state.clip = entry.clip; entry = {}; + m_flagMask &= ~(1u << slot); } - if (m_fdiv.valid && m_fdiv.readyCycle <= m_cycle) + if (m_fdiv.valid) { - m_state.q = m_fdiv.value; - const uint32_t currentDi = m_fdiv.statusDi & 0x30u; - m_state.status = (m_state.status & 0xFCFu) | currentDi | (currentDi << 6); - m_fdiv = {}; + if (m_fdiv.readyCycle <= m_cycle) + { + m_state.q = m_fdiv.value; + const uint32_t currentDi = m_fdiv.statusDi & 0x30u; + m_state.status = (m_state.status & 0xFCFu) | currentDi | (currentDi << 6); + m_fdiv = {}; + } + else + { + next = std::min(next, m_fdiv.readyCycle); + } } - for (ScalarPipelineEntry &entry : m_efu) + for (uint32_t bits = m_efuMask; bits != 0u;) { - if (entry.valid && entry.readyCycle <= m_cycle) + const int slot = nextSlot(bits); + ScalarPipelineEntry &entry = m_efu[static_cast(slot)]; + if (entry.readyCycle > m_cycle) { - m_state.p = entry.value; - entry = {}; + next = std::min(next, entry.readyCycle); + continue; } + m_state.p = entry.value; + entry = {}; + m_efuMask &= ~(1u << slot); } - for (PendingStore &store : m_storePipeline) + for (uint32_t bits = m_storeMask; bits != 0u;) { - if (!store.valid || store.readyCycle > m_cycle) + const int slot = nextSlot(bits); + PendingStore &store = m_storePipeline[static_cast(slot)]; + if (store.readyCycle > m_cycle) + { + next = std::min(next, store.readyCycle); continue; + } if (m_activeVuData && store.address + 16u <= m_activeVuDataSize) { uint32_t oldWords[4]{}; @@ -798,46 +883,10 @@ void VU1Interpreter::commitReadyPipelines() std::memcpy(m_activeVuData + store.address, oldWords, sizeof(oldWords)); } store = {}; + m_storeMask &= ~(1u << slot); } - for (PendingVfWrite &write : m_vfWritePipeline) - { - if (!write.valid || write.readyCycle > m_cycle) - continue; - for (uint32_t component = 0; component < 4u; ++component) - { - if ((write.laneMask & laneForComponent(component)) != 0u && - m_vfLatestWrite[write.reg][component] == write.sequence) - { - m_state.vf[write.reg][component] = write.value[component]; - } - } - write = {}; - } - - for (PendingViWrite &write : m_viWritePipeline) - { - if (!write.valid || write.readyCycle > m_cycle) - continue; - if (m_viLatestWrite[write.reg] == write.sequence) - m_state.vi[write.reg] = static_cast(write.value); - write = {}; - } - - for (PendingAccWrite &write : m_accWritePipeline) - { - if (!write.valid || write.readyCycle > m_cycle) - continue; - for (uint32_t component = 0; component < 4u; ++component) - { - if ((write.laneMask & laneForComponent(component)) != 0u && - m_accLatestWrite[component] == write.sequence) - { - m_state.acc[component] = write.value[component]; - } - } - write = {}; - } + m_nextCommitCycle = next; } void VU1Interpreter::progressXgkick() @@ -857,10 +906,18 @@ void VU1Interpreter::progressXgkick() } const uint32_t qwordOffset = m_xgkick.copiedBytes; - for (uint32_t i = 0; i < 16u; ++i) + const uint32_t qwordSource = (m_xgkick.sourceAddress + m_xgkick.copiedBytes) % m_activeVuDataSize; + if (qwordSource + 16u <= m_activeVuDataSize) { - const uint32_t source = (m_xgkick.sourceAddress + m_xgkick.copiedBytes + i) % m_activeVuDataSize; - m_xgkick.packet[m_xgkick.copiedBytes + i] = m_activeVuData[source]; + std::memcpy(m_xgkick.packet.data() + qwordOffset, m_activeVuData + qwordSource, 16u); + } + else + { + for (uint32_t i = 0; i < 16u; ++i) + { + const uint32_t source = (m_xgkick.sourceAddress + m_xgkick.copiedBytes + i) % m_activeVuDataSize; + m_xgkick.packet[m_xgkick.copiedBytes + i] = m_activeVuData[source]; + } } m_xgkick.copiedBytes += 16u; @@ -952,70 +1009,46 @@ void VU1Interpreter::advanceOneCycle() void VU1Interpreter::advanceTo(uint64_t targetCycle) { while (m_cycle < targetCycle) + { + // With no XGKICK in flight a cycle does nothing until the next + // pipeline commit, so skip straight to the cycle before it. + if (!m_xgkick.active) + { + const uint64_t idleEnd = std::min(targetCycle, m_nextCommitCycle) - 1u; + if (idleEnd > m_cycle) + m_cycle = idleEnd; + } advanceOneCycle(); + } } bool VU1Interpreter::pipelinesPending() const { - if (m_fdiv.valid || m_xgkick.active) - return true; - for (const ScalarPipelineEntry &entry : m_efu) - if (entry.valid) - return true; - for (const FlagPipelineEntry &entry : m_flagPipeline) - if (entry.valid) - return true; - for (const PendingStore &store : m_storePipeline) - if (store.valid) - return true; - for (const PendingVfWrite &write : m_vfWritePipeline) - if (write.valid) - return true; - for (const PendingViWrite &write : m_viWritePipeline) - if (write.valid) - return true; - for (const PendingAccWrite &write : m_accWritePipeline) - if (write.valid) - return true; - return false; + return m_xgkick.active || m_nextCommitCycle != std::numeric_limits::max() || + m_writebackHorizon > m_cycle; } void VU1Interpreter::flushPipelines() { while (pipelinesPending()) + { + if (!m_xgkick.active) + { + uint64_t next = m_nextCommitCycle; + if (m_writebackHorizon > m_cycle) + next = std::min(next, m_writebackHorizon); + if (next > m_cycle + 1u) + m_cycle = next - 1u; + } advanceOneCycle(); + } } uint64_t VU1Interpreter::calculatePairReadyCycle(const DecodedInstructionPair &decoded) const { uint64_t ready = m_cycle; - const InstructionUsage *usages[2] = { - &decoded.upperUsage, - &decoded.lowerUsage}; - for (const InstructionUsage *usage : usages) - { - if (!usage) - continue; - for (uint32_t index = 0; index < usage->vfReadCount; ++index) - { - const VfAccess &access = usage->vfRead[index]; - for (uint32_t component = 0; component < 4u; ++component) - { - if ((access.lanes & laneForComponent(component)) != 0u) - ready = std::max(ready, m_vfReady[access.reg][component]); - } - } - for (uint32_t reg = 1; reg < m_viReady.size(); ++reg) - { - if ((usage->viRead & (1u << reg)) != 0u) - ready = std::max(ready, m_viReady[reg]); - } - for (uint32_t component = 0; component < 4u; ++component) - { - if ((usage->accRead & laneForComponent(component)) != 0u) - ready = std::max(ready, m_accReady[component]); - } - } + for (uint32_t index = 0; index < decoded.readSlotCount; ++index) + ready = std::max(ready, m_ready[decoded.readSlots[index]]); if (decoded.lowerUsage.pipeline == PipelineFdiv && m_fdiv.valid) ready = std::max(ready, m_fdiv.readyCycle); @@ -1036,43 +1069,10 @@ uint64_t VU1Interpreter::calculatePairReadyCycle(const DecodedInstructionPair &d void VU1Interpreter::markPairWrites(const DecodedInstructionPair &decoded) { - const VfAccess lowerWrite = decoded.lowerUsage.vfWrite; - if (lowerWrite.reg != 0u && - decoded.suppressedLowerVf != lowerWrite.reg) - { - const uint32_t latency = decoded.lowerUsage.vfLatency != 0u - ? decoded.lowerUsage.vfLatency - : decoded.lowerUsage.latency; - for (uint32_t component = 0; component < 4u; ++component) - { - if ((lowerWrite.lanes & laneForComponent(component)) != 0u) - m_vfReady[lowerWrite.reg][component] = m_cycle + latency; - } - } - - const VfAccess upperWrite = decoded.upperUsage.vfWrite; - if (upperWrite.reg != 0u) - { - const uint32_t latency = decoded.upperUsage.vfLatency != 0u - ? decoded.upperUsage.vfLatency - : decoded.upperUsage.latency; - for (uint32_t component = 0; component < 4u; ++component) - { - if ((upperWrite.lanes & laneForComponent(component)) != 0u) - m_vfReady[upperWrite.reg][component] = m_cycle + latency; - } - } - - for (uint32_t reg = 1; reg < m_viReady.size(); ++reg) - { - if ((decoded.lowerUsage.viWrite & (1u << reg)) != 0u) - m_viReady[reg] = m_cycle + (decoded.lowerUsage.viLatency != 0u ? decoded.lowerUsage.viLatency : decoded.lowerUsage.latency); - } - for (uint32_t component = 0; component < 4u; ++component) - { - if ((decoded.upperUsage.accWrite & laneForComponent(component)) != 0u) - m_accReady[component] = m_cycle + kAccForwardLatency; - } + for (uint32_t index = 0; index < decoded.writeSlotCount; ++index) + m_ready[decoded.writeSlots[index]] = m_cycle + decoded.writeLatencies[index]; + if (decoded.writeSlotCount != 0u) + m_writebackHorizon = std::max(m_writebackHorizon, m_cycle + decoded.maxWriteLatency); } VU1Interpreter::InstructionUsage VU1Interpreter::decodeUpperUsage(uint32_t upper) const @@ -1528,6 +1528,66 @@ VU1Interpreter::DecodedInstructionPair VU1Interpreter::decodeInstructionPair(con if (decoded.lowerUsage.vfWrite.reg == upperWriteReg) decoded.suppressedLowerVf = upperWriteReg; } + + // Hazard slots, matching the checks calculatePairReadyCycle and + // markPairWrites used to derive from the usages on every issue. + auto addRead = [&decoded](uint32_t slot) + { + decoded.readSlots[decoded.readSlotCount++] = static_cast(slot); + }; + for (const InstructionUsage *usage : {&decoded.upperUsage, &decoded.lowerUsage}) + { + for (uint32_t index = 0; index < usage->vfReadCount; ++index) + { + const VfAccess &access = usage->vfRead[index]; + if (access.reg == 0u) + continue; // VF0 is never written, so never pending. + for (uint32_t component = 0; component < 4u; ++component) + if ((access.lanes & laneForComponent(component)) != 0u) + addRead(kReadyVfBase + access.reg * 4u + component); + } + for (uint32_t bits = usage->viRead & 0xFFFEu; bits != 0u; bits &= bits - 1u) + addRead(kReadyViBase + static_cast(std::countr_zero(bits))); + for (uint32_t component = 0; component < 4u; ++component) + if ((usage->accRead & laneForComponent(component)) != 0u) + addRead(kReadyAccBase + component); + } + + auto addWrite = [&decoded](uint32_t slot, uint32_t latency) + { + decoded.writeSlots[decoded.writeSlotCount] = static_cast(slot); + decoded.writeLatencies[decoded.writeSlotCount] = static_cast(latency); + decoded.maxWriteLatency = std::max(decoded.maxWriteLatency, static_cast(latency)); + ++decoded.writeSlotCount; + }; + const VfAccess lowerWrite = decoded.lowerUsage.vfWrite; + if (lowerWrite.reg != 0u && decoded.suppressedLowerVf != lowerWrite.reg) + { + const uint32_t latency = decoded.lowerUsage.vfLatency != 0u ? decoded.lowerUsage.vfLatency + : decoded.lowerUsage.latency; + for (uint32_t component = 0; component < 4u; ++component) + if ((lowerWrite.lanes & laneForComponent(component)) != 0u) + addWrite(kReadyVfBase + lowerWrite.reg * 4u + component, latency); + } + const VfAccess upperWrite = decoded.upperUsage.vfWrite; + if (upperWrite.reg != 0u) + { + const uint32_t latency = decoded.upperUsage.vfLatency != 0u ? decoded.upperUsage.vfLatency + : decoded.upperUsage.latency; + for (uint32_t component = 0; component < 4u; ++component) + if ((upperWrite.lanes & laneForComponent(component)) != 0u) + addWrite(kReadyVfBase + upperWrite.reg * 4u + component, latency); + } + const uint32_t viLatency = decoded.lowerUsage.viLatency != 0u ? decoded.lowerUsage.viLatency + : decoded.lowerUsage.latency; + for (uint32_t bits = decoded.lowerUsage.viWrite & 0xFFFEu; bits != 0u; bits &= bits - 1u) + addWrite(kReadyViBase + static_cast(std::countr_zero(bits)), viLatency); + for (uint32_t component = 0; component < 4u; ++component) + if ((decoded.upperUsage.accWrite & laneForComponent(component)) != 0u) + addWrite(kReadyAccBase + component, kAccForwardLatency); + + if ((decoded.lowerUsage.viWrite & 0xFFFEu) != 0u) + decoded.firstViWrite = static_cast(std::countr_zero(decoded.lowerUsage.viWrite & 0xFFFEu)); return decoded; } @@ -1545,17 +1605,17 @@ void VU1Interpreter::rebuildDecodedCodeCache(const uint8_t *vuCode, uint32_t cod m_decodedCodeCacheValid = true; } -VU1Interpreter::DecodedInstructionPair VU1Interpreter::getDecodedInstructionPairForPc( +const VU1Interpreter::DecodedInstructionPair &VU1Interpreter::getDecodedInstructionPairForPc( const uint8_t *vuCode, uint32_t codeSize, PS2Memory *memory, uint32_t pc) { if ((pc & 7u) != 0u) - return decodeInstructionPair(vuCode, pc); + return m_uncachedDecoded = decodeInstructionPair(vuCode, pc); const bool trackedVu1Code = memory != nullptr && ((m_unit == Unit::VU1 && vuCode == memory->getVU1Code()) || (m_unit == Unit::VU0 && vuCode == memory->getVU0Code())); if (!trackedVu1Code) - return decodeInstructionPair(vuCode, pc); + return m_uncachedDecoded = decodeInstructionPair(vuCode, pc); const uint64_t generation = m_unit == Unit::VU1 ? memory->getVU1CodeGeneration() : memory->getVU0CodeGeneration(); if (!m_decodedCodeCacheValid || @@ -1568,7 +1628,7 @@ VU1Interpreter::DecodedInstructionPair VU1Interpreter::getDecodedInstructionPair } const uint32_t pairIndex = pc / 8u; if (pairIndex >= kMaxDecodedPairs) - return decodeInstructionPair(vuCode, pc); + return m_uncachedDecoded = decodeInstructionPair(vuCode, pc); return m_decodedCodeCache[pairIndex]; } @@ -1639,7 +1699,7 @@ void VU1Interpreter::run(uint8_t *vuCode, uint32_t codeSize, if (m_state.pc + 8u > codeSize) break; - const DecodedInstructionPair decoded = getDecodedInstructionPairForPc(vuCode, codeSize, memory, m_state.pc); + const DecodedInstructionPair &decoded = getDecodedInstructionPairForPc(vuCode, codeSize, memory, m_state.pc); if (decoded.upperUsage.reserved || decoded.lowerUsage.reserved) { reportReservedInstruction(decoded.upperUsage.reserved, decoded.upperUsage.reserved ? decoded.upper : decoded.lower); @@ -1660,35 +1720,8 @@ void VU1Interpreter::run(uint8_t *vuCode, uint32_t codeSize, if (m_cycle >= budgetEnd) break; - uint8_t writtenVi = 0u; - int32_t oldVi = 0; - for (uint32_t reg = 1; reg < 16u; ++reg) - { - if ((decoded.lowerUsage.viWrite & (1u << reg)) != 0u) - { - writtenVi = static_cast(reg); - oldVi = m_state.vi[reg]; - break; - } - } - - const VfAccess upperWrite = decoded.upperUsage.vfWrite; - const VfAccess lowerWrite = decoded.lowerUsage.vfWrite; - const bool hasUpperWrite = upperWrite.reg != 0u; - const bool hasLowerWrite = lowerWrite.reg != 0u && decoded.suppressedLowerVf != lowerWrite.reg; - const bool hasDistinctLowerWrite = hasLowerWrite && (!hasUpperWrite || lowerWrite.reg != upperWrite.reg); - float oldUpperVf[4]{}; - float newUpperVf[4]{}; - float oldLowerVf[4]{}; - float newLowerVf[4]{}; - float oldAcc[4]{}; - float newAcc[4]{}; - if (hasUpperWrite) - std::memcpy(oldUpperVf, m_state.vf[upperWrite.reg], sizeof(oldUpperVf)); - if (hasDistinctLowerWrite) - std::memcpy(oldLowerVf, m_state.vf[lowerWrite.reg], sizeof(oldLowerVf)); - if (decoded.upperUsage.accWrite != 0u) - std::memcpy(oldAcc, m_state.acc, sizeof(oldAcc)); + const uint8_t writtenVi = decoded.firstViWrite; + const int32_t oldVi = writtenVi != 0u ? m_state.vi[writtenVi] : 0; if (decoded.iBit) { @@ -1724,45 +1757,11 @@ void VU1Interpreter::run(uint8_t *vuCode, uint32_t codeSize, m_viBranchBackupValid = false; - if (hasUpperWrite) - { - std::memcpy(newUpperVf, m_state.vf[upperWrite.reg], sizeof(newUpperVf)); - std::memcpy(m_state.vf[upperWrite.reg], oldUpperVf, sizeof(oldUpperVf)); - const uint32_t latency = - decoded.upperUsage.vfLatency != 0u - ? decoded.upperUsage.vfLatency - : decoded.upperUsage.latency; - queueVfWrite(upperWrite.reg, upperWrite.lanes, newUpperVf, latency); - } - if (hasDistinctLowerWrite) - { - std::memcpy(newLowerVf, m_state.vf[lowerWrite.reg], sizeof(newLowerVf)); - std::memcpy(m_state.vf[lowerWrite.reg], oldLowerVf, sizeof(oldLowerVf)); - const uint32_t latency = decoded.lowerUsage.vfLatency != 0u - ? decoded.lowerUsage.vfLatency - : decoded.lowerUsage.latency; - queueVfWrite(lowerWrite.reg, lowerWrite.lanes, newLowerVf, latency); - } - if (decoded.upperUsage.accWrite != 0u) - { - std::memcpy(newAcc, m_state.acc, sizeof(newAcc)); - std::memcpy(m_state.acc, oldAcc, sizeof(oldAcc)); - // ACC is forwarded to the next upper instruction. Its arithmetic - // flags still use the normal four-cycle FMAC timeline. - queueAccWrite(decoded.upperUsage.accWrite, newAcc, - kAccForwardLatency); - } - if (writtenVi != 0u) - { - const int32_t newVi = m_state.vi[writtenVi]; - m_state.vi[writtenVi] = oldVi; - const uint32_t latency = - decoded.lowerUsage.viLatency != 0u - ? decoded.lowerUsage.viLatency - : decoded.lowerUsage.latency; - queueViWrite(writtenVi, newVi, latency); - } - + // VF, VI and ACC results are written back immediately: these + // registers are interlocked, so calculatePairReadyCycle stalls every + // reader until the result's latency has elapsed and an early value + // can never be observed. (Flags, Q, P and stores are not interlocked + // and keep their pipelines.) markPairWrites(decoded); if (writtenVi != 0u && decoded.lowerUsage.delaysNextBranchRead) recordViWriteForBranch(writtenVi, oldVi); diff --git a/ps2xRuntime/src/lib/vu/ps2_vu1_upper.cpp b/ps2xRuntime/src/lib/vu/ps2_vu1_upper.cpp index 69742546f..79d200085 100644 --- a/ps2xRuntime/src/lib/vu/ps2_vu1_upper.cpp +++ b/ps2xRuntime/src/lib/vu/ps2_vu1_upper.cpp @@ -4,9 +4,38 @@ #include #include #include +#if defined(__SSE4_1__) +#include +#endif namespace { + // normalizeOperand on four lanes. + void normalizeOperands(const float *in, float *out) + { +#if defined(__SSE4_1__) + const __m128i bits = _mm_loadu_si128(reinterpret_cast(in)); + const __m128i exponent = _mm_and_si128(bits, _mm_set1_epi32(0x7F800000)); + const __m128i sign = _mm_and_si128(bits, _mm_set1_epi32(static_cast(0x80000000u))); + __m128i normalized = _mm_blendv_epi8(bits, sign, _mm_cmpeq_epi32(exponent, _mm_setzero_si128())); + normalized = _mm_blendv_epi8(normalized, _mm_or_si128(sign, _mm_set1_epi32(0x7F7FFFFF)), + _mm_cmpeq_epi32(exponent, _mm_set1_epi32(0x7F800000))); + _mm_storeu_si128(reinterpret_cast<__m128i *>(out), normalized); +#else + for (uint32_t lane = 0; lane < 4u; ++lane) + { + uint32_t bits = 0; + std::memcpy(&bits, &in[lane], sizeof(bits)); + const uint32_t exponent = (bits >> 23) & 0xFFu; + if (exponent == 0u) + bits &= 0x80000000u; + else if (exponent == 0xFFu) + bits = (bits & 0x80000000u) | 0x7F7FFFFFu; + std::memcpy(&out[lane], &bits, sizeof(bits)); + } +#endif + } + int32_t vuFloatToInt(float value, float scale) { const double scaled = static_cast(value) * static_cast(scale); @@ -34,17 +63,19 @@ void VU1Interpreter::execUpper(uint32_t instr) float normalizedVs[4]; float normalizedVt[4]; float normalizedAcc[4]; - for (uint32_t component = 0; component < 4u; ++component) - { - normalizedVs[component] = normalizeOperand(m_state.vf[fs][component]); - normalizedVt[component] = normalizeOperand(m_state.vf[ft][component]); - normalizedAcc[component] = normalizeOperand(m_state.acc[component]); - } + normalizeOperands(m_state.vf[fs], normalizedVs); + normalizeOperands(m_state.vf[ft], normalizedVt); + normalizeOperands(m_state.acc, normalizedAcc); const float *vs = normalizedVs; const float *vt = normalizedVt; const float *acc = normalizedAcc; const float q = normalizeOperand(m_state.q); const float i = normalizeOperand(m_state.i); + m_fmacVs = vs; + m_fmacVt = vt; + m_fmacAcc = acc; + m_fmacQ = q; + m_fmacI = i; float result[4]; // Upper opcode decoding (bits 5:0 of upper word)