No public description

centipede/BUILD

@@ -1940,6 +1940,7 @@ cc_test(
         ":crash_deduplication_test_util",
         ":crash_summary",
         ":environment",
+        ":runner_result",
         ":stop",
         ":util",
         ":workdir",
@@ -1953,6 +1954,8 @@ cc_test(
         "@abseil-cpp//absl/time",
         "@abseil-cpp//absl/time:clock_interface",
         "@abseil-cpp//absl/time:simulated_clock",
+        "@abseil-cpp//absl/types:span",
+        "@com_google_fuzztest//common:defs",
         "@com_google_fuzztest//common:temp_dir",
         "@googletest//:gtest_main",
     ],

centipede/centipede.cc

@@ -1076,44 +1076,49 @@ void Centipede::ReportCrash(std::string_view binary,
   FUZZTEST_LOG(INFO)
       << log_prefix
       << "Executing inputs one-by-one, trying to find the reproducer";
+  const size_t max_attempts = std::max<size_t>(1, env_.replay_crash_attempts);
   for (auto input_idx : input_idxs_to_try) {
     if (stop_condition_.ShouldStop()) break;
     const auto one_input = input_vec[input_idx];
-    BatchResult one_input_batch_result;
-    if (!user_callbacks_.Execute(binary, {one_input}, one_input_batch_result) &&
-        one_input_batch_result.IsInputFailure() &&
-        one_input_batch_result.failure_signature() ==
-            batch_result.failure_signature() &&
-        !stop_condition_.ShouldStop()) {
-      auto hash = Hash(one_input);
-      auto crash_dir = wd_.CrashReproducerDirPaths().MyShard();
-      FUZZTEST_CHECK_OK(RemoteMkdir(crash_dir));
-      std::string input_file_path = std::filesystem::path(crash_dir) / hash;
-      auto crash_metadata_dir = wd_.CrashMetadataDirPaths().MyShard();
-      FUZZTEST_CHECK_OK(RemoteMkdir(crash_metadata_dir));
-      std::string crash_metadata_path_prefix =
-          std::filesystem::path(crash_metadata_dir) / hash;
-      FUZZTEST_LOG(INFO)
-          << log_prefix << "Detected crash-reproducing input:"
-          << "\nInput index    : " << input_idx << "\nInput bytes    : "
-          << AsPrintableString(one_input, /*max_len=*/32)
-          << "\nExit code      : " << one_input_batch_result.exit_code()
-          << "\nFailure        : "
-          << one_input_batch_result.failure_description()
-          << "\nSignature      : "
-          << AsPrintableString(
-                 AsByteSpan(one_input_batch_result.failure_signature()),
-                 /*max_len=*/32)
-          << "\nSaving input to: " << input_file_path << "\nSaving crash"  //
-          << "\nmetadata to    : " << crash_metadata_path_prefix << ".*";
-      FUZZTEST_CHECK_OK(RemoteFileSetContents(input_file_path, one_input));
-      FUZZTEST_CHECK_OK(RemoteFileSetContents(
-          absl::StrCat(crash_metadata_path_prefix, ".desc"),
-          one_input_batch_result.failure_description()));
-      FUZZTEST_CHECK_OK(RemoteFileSetContents(
-          absl::StrCat(crash_metadata_path_prefix, ".sig"),
-          one_input_batch_result.failure_signature()));
-      return;
+    for (size_t attempt = 0; attempt < max_attempts; ++attempt) {
+      if (stop_condition_.ShouldStop()) break;
+      BatchResult one_input_batch_result;
+      if (!user_callbacks_.Execute(binary, {one_input},
+                                   one_input_batch_result) &&
+          one_input_batch_result.IsInputFailure() &&
+          one_input_batch_result.failure_signature() ==
+              batch_result.failure_signature() &&
+          !stop_condition_.ShouldStop()) {
+        auto hash = Hash(one_input);
+        auto crash_dir = wd_.CrashReproducerDirPaths().MyShard();
+        FUZZTEST_CHECK_OK(RemoteMkdir(crash_dir));
+        std::string input_file_path = std::filesystem::path(crash_dir) / hash;
+        auto crash_metadata_dir = wd_.CrashMetadataDirPaths().MyShard();
+        FUZZTEST_CHECK_OK(RemoteMkdir(crash_metadata_dir));
+        std::string crash_metadata_path_prefix =
+            std::filesystem::path(crash_metadata_dir) / hash;
+        FUZZTEST_LOG(INFO)
+            << log_prefix << "Detected crash-reproducing input:"
+            << "\nInput index    : " << input_idx << "\nInput bytes    : "
+            << AsPrintableString(one_input, /*max_len=*/32)
+            << "\nExit code      : " << one_input_batch_result.exit_code()
+            << "\nFailure        : "
+            << one_input_batch_result.failure_description()
+            << "\nSignature      : "
+            << AsPrintableString(
+                   AsByteSpan(one_input_batch_result.failure_signature()),
+                   /*max_len=*/32)
+            << "\nSaving input to: " << input_file_path << "\nSaving crash"  //
+            << "\nmetadata to    : " << crash_metadata_path_prefix << ".*";
+        FUZZTEST_CHECK_OK(RemoteFileSetContents(input_file_path, one_input));
+        FUZZTEST_CHECK_OK(RemoteFileSetContents(
+            absl::StrCat(crash_metadata_path_prefix, ".desc"),
+            one_input_batch_result.failure_description()));
+        FUZZTEST_CHECK_OK(RemoteFileSetContents(
+            absl::StrCat(crash_metadata_path_prefix, ".sig"),
+            one_input_batch_result.failure_signature()));
+        return;
+      }
     }
   }
 

centipede/centipede_flags.inc

@@ -526,3 +526,7 @@ CENTIPEDE_FLAG(bool, fuzztest_replay_coverage_inputs, false,
 CENTIPEDE_FLAG(
     absl::Duration, fuzztest_time_limit_per_test, absl::InfiniteDuration(),
     "The time limit per fuzz test for working on the corpus database.")
+CENTIPEDE_FLAG(
+    size_t, replay_crash_attempts, 1,
+    "Number of attempts to replay a crash input during batch failure triage "
+    "or deduplication before considering it non-reproducible.")

centipede/centipede_test.cc

@@ -956,6 +956,96 @@ TEST(Centipede, UndetectedCrashingInput) {
   EXPECT_EQ(suspect_only_mock.num_inputs_triaged(), 1);
 }
 
+// Mock callback that fails on a specific attempt of an input during triage.
+class FlakyCrashingInputMock : public CentipedeCallbacks {
+ public:
+  FlakyCrashingInputMock(const Environment& env, size_t crashing_input_idx,
+                         size_t fail_on_triage_attempt)
+      : CentipedeCallbacks{env, internal_stop_condition_},
+        crashing_input_idx_(crashing_input_idx),
+        fail_on_triage_attempt_(fail_on_triage_attempt) {}
+
+  bool Execute(std::string_view binary, absl::Span<const ByteSpan> inputs,
+               BatchResult& batch_result) override {
+    batch_result.ClearAndResize(inputs.size());
+    if (first_pass_) {
+      for (const auto& input : inputs) {
+        if (input[0] == crashing_input_idx_) {
+          first_pass_ = false;
+          crashing_input_ = {input.begin(), input.end()};
+          batch_result.num_outputs_read() =
+              crashing_input_idx_ % env_.batch_size;
+          batch_result.exit_code() = 1;
+          return false;
+        }
+      }
+      return true;
+    }
+    // In triage
+    for (const auto& input : inputs) {
+      if (input == AsByteSpan(crashing_input_)) {
+        ++triage_attempts_;
+        if (triage_attempts_ == fail_on_triage_attempt_) {
+          batch_result.exit_code() = 1;
+          return false;
+        }
+      }
+    }
+    return true;
+  }
+
+  std::vector<Mutant> Mutate(absl::Span<const MutationInputRef> inputs,
+                             size_t num_mutants) override {
+    std::vector<Mutant> mutants;
+    mutants.reserve(num_mutants);
+    for (size_t i = 0; i < num_mutants; ++i) {
+      mutants.push_back({/*data=*/{static_cast<uint8_t>(curr_input_idx_++)},
+                         Mutant::kOriginNone});
+    }
+    return mutants;
+  }
+
+  ByteArray crashing_input() const { return crashing_input_; }
+  size_t triage_attempts() const { return triage_attempts_; }
+
+ private:
+  const size_t crashing_input_idx_;
+  const size_t fail_on_triage_attempt_;
+  size_t curr_input_idx_ = 0;
+  size_t triage_attempts_ = 0;
+  ByteArray crashing_input_ = {};
+  bool first_pass_ = true;
+  StopCondition internal_stop_condition_;
+};
+
+TEST(Centipede, ReportCrashRetriesWithReplayCrashAttempts) {
+  constexpr size_t kNumBatches = 5;
+  constexpr size_t kBatchSize = 10;
+  constexpr size_t kCrashingInputIdx = 15;
+
+  TempDir temp_dir{test_info_->name()};
+  Environment env;
+  env.workdir = temp_dir.path();
+  env.num_runs = kBatchSize * kNumBatches;
+  env.batch_size = kBatchSize;
+  env.require_pc_table = false;
+  env.exit_on_crash = true;
+  env.batch_triage_suspect_only = true;
+  env.replay_crash_attempts = 3;
+
+  FlakyCrashingInputMock mock(env, kCrashingInputIdx,
+                              /*fail_on_triage_attempt=*/2);
+  NonOwningCallbacksFactory factory(mock);
+  CentipedeMain(env, factory);
+
+  EXPECT_EQ(mock.triage_attempts(), 2);
+  const auto crashing_input_hash = Hash(mock.crashing_input());
+  const auto crasher_path =
+      std::filesystem::path{WorkDir{env}.CrashReproducerDirPaths().MyShard()} /
+      crashing_input_hash;
+  EXPECT_TRUE(std::filesystem::exists(crasher_path)) << crasher_path;
+}
+
 TEST_F(CentipedeWithTemporaryLocalDir, GetsSeedInputs) {
   Environment env;
   env.binary =

centipede/crash_deduplication.cc

@@ -14,6 +14,7 @@
 
 #include "./centipede/crash_deduplication.h"
 
+#include <algorithm>
 #include <cstddef>
 #include <cstdlib>
 #include <filesystem>  // NOLINT
@@ -158,18 +159,27 @@ absl::Status ReplayCrash(CentipedeCallbacks& callbacks, const Environment& env,
   ByteArray input_bytes;
   RETURN_IF_NOT_OK(RemoteFileGetContents(input_path, input_bytes));
 
-  BatchResult batch_result;
-  const bool is_reproducible =
-      !callbacks.Execute(env.binary, {input_bytes}, batch_result) &&
-      batch_result.IsInputFailure();
-
-  if (is_reproducible) {
-    out_signature = batch_result.failure_signature();
-    out_description = batch_result.failure_description();
-  } else {
-    out_signature = "";
-    out_description = "";
+  const size_t max_attempts = std::max<size_t>(1, env.replay_crash_attempts);
+  for (size_t attempt = 0; attempt < max_attempts; ++attempt) {
+    BatchResult batch_result;
+    if (!callbacks.Execute(env.binary, {input_bytes}, batch_result) &&
+        batch_result.IsInputFailure()) {
+      out_signature = batch_result.failure_signature();
+      out_description = batch_result.failure_description();
+      if (attempt > 0) {
+        FUZZTEST_LOG(INFO) << "Crash reproduced on attempt " << (attempt + 1)
+                           << " of " << max_attempts << " for " << input_path;
+      }
+      return absl::OkStatus();
+    }
+  }
+
+  if (max_attempts > 1) {
+    FUZZTEST_LOG(INFO) << "Crash failed to reproduce after " << max_attempts
+                       << " attempts for " << input_path;
   }
+  out_signature = "";
+  out_description = "";
   return absl::OkStatus();
 }
 

centipede/crash_deduplication_test.cc

@@ -14,6 +14,7 @@
 
 #include "./centipede/crash_deduplication.h"
 
+#include <cstdlib>
 #include <filesystem>  // NOLINT
 #include <string>
 #include <string_view>
@@ -33,13 +34,16 @@
 #include "absl/time/clock_interface.h"
 #include "absl/time/simulated_clock.h"
 #include "absl/time/time.h"
+#include "absl/types/span.h"
 #include "./centipede/centipede_callbacks.h"
 #include "./centipede/crash_deduplication_test_util.h"
 #include "./centipede/crash_summary.h"
 #include "./centipede/environment.h"
+#include "./centipede/runner_result.h"
 #include "./centipede/stop.h"
 #include "./centipede/util.h"
 #include "./centipede/workdir.h"
+#include "./common/defs.h"
 #include "./common/temp_dir.h"
 
 namespace fuzztest::internal {
@@ -1116,5 +1120,75 @@ TEST_F(OrganizeCrashingInputsTest, LogsActionMoveToRegression) {
                     HasSubstr("Reason: Crash expired (not reproduced for")));
 }
 
+class FlakyCrashCallbacks : public CentipedeCallbacks {
+ public:
+  FlakyCrashCallbacks(const Environment& env, int crash_on_attempt)
+      : CentipedeCallbacks(env, internal_stop_condition_),
+        crash_on_attempt_(crash_on_attempt) {}
+
+  bool Execute(std::string_view binary, absl::Span<const ByteSpan> inputs,
+               BatchResult& batch_result) override {
+    ++execution_count_;
+    batch_result.ClearAndResize(inputs.size());
+    if (execution_count_ == crash_on_attempt_) {
+      batch_result.exit_code() = EXIT_FAILURE;
+      batch_result.failure_signature() = "csig";
+      batch_result.failure_description() = "flaky crash";
+      return false;
+    }
+    return true;
+  }
+
+  int execution_count() const { return execution_count_; }
+
+ private:
+  int crash_on_attempt_;
+  int execution_count_ = 0;
+  StopCondition internal_stop_condition_;
+};
+
+TEST_F(OrganizeCrashingInputsTest, ReplaysCrashMultipleTimesUntilSuccess) {
+  LogCapture log_capture;
+  SetContentsAndGetPath(incubating_dir(), "isig1", "input1");
+
+  Environment test_env = env();
+  test_env.replay_crash_attempts = 3;
+
+  FlakyCrashCallbacks callbacks(test_env, /*crash_on_attempt=*/2);
+  NonOwningCallbacksFactory factory(callbacks);
+
+  ASSERT_TRUE(OrganizeCrashingInputs(regression_dir(), crashing_dir(), test_env,
+                                     factory, /*new_crashes_by_signature=*/{},
+                                     crash_summary())
+                  .ok());
+
+  EXPECT_EQ(callbacks.execution_count(), 2);
+  EXPECT_THAT(log_capture.FullLog(),
+              AllOf(HasSubstr("Crash reproduced on attempt 2 of 3 for "),
+                    HasSubstr("Action: CleanUpIncubating for"),
+                    HasSubstr("Reason: Input reproduced with signature 'csig' "
+                              "and graduated from incubation")));
+}
+
+TEST_F(OrganizeCrashingInputsTest, ReplaysCrashUpToMaxAttemptsOnFailure) {
+  LogCapture log_capture;
+  SetContentsAndGetPath(incubating_dir(), "isig1", "input1");
+
+  Environment test_env = env();
+  test_env.replay_crash_attempts = 3;
+
+  FlakyCrashCallbacks callbacks(test_env, /*crash_on_attempt=*/5);
+  NonOwningCallbacksFactory factory(callbacks);
+
+  ASSERT_TRUE(OrganizeCrashingInputs(regression_dir(), crashing_dir(), test_env,
+                                     factory, /*new_crashes_by_signature=*/{},
+                                     crash_summary())
+                  .ok());
+
+  EXPECT_EQ(callbacks.execution_count(), 3);
+  EXPECT_THAT(log_capture.FullLog(),
+              HasSubstr("Crash failed to reproduce after 3 attempts for "));
+}
+
 }  // namespace
 }  // namespace fuzztest::internal

centipede/environment.cc

@@ -250,6 +250,9 @@ void Environment::UpdateWithTargetConfig(
   fuzztest_execution_id = config.execution_id.value_or("");
   fuzztest_replay_coverage_inputs = config.replay_coverage_inputs;
   fuzztest_time_limit_per_test = config.GetTimeLimitPerTest();
+  if (replay_crash_attempts == Default().replay_crash_attempts) {
+    replay_crash_attempts = config.replay_crash_attempts;
+  }
 
   // Allow more crashes to be reported when running with FuzzTest. This allows
   // more unique crashes to collected after deduplication. But we don't want to

centipede/environment_test.cc

@@ -220,4 +220,21 @@ TEST(Environment, UpdatesReplayOnlyConfiguration) {
   EXPECT_FALSE(env.populate_binary_info);
 }
 
+TEST(Environment, UpdatesReplayAttemptsFromTargetConfigWhenDefault) {
+  Environment env;
+  fuzztest::internal::Configuration config;
+  config.replay_crash_attempts = 5;
+  env.UpdateWithTargetConfig(config);
+  EXPECT_EQ(env.replay_crash_attempts, 5);
+}
+
+TEST(Environment, PreservesReplayAttemptsWhenExplicitlySet) {
+  Environment env;
+  env.replay_crash_attempts = 10;
+  fuzztest::internal::Configuration config;
+  config.replay_crash_attempts = 5;
+  env.UpdateWithTargetConfig(config);
+  EXPECT_EQ(env.replay_crash_attempts, 10);
+}
+
 }  // namespace fuzztest::internal