diff --git a/plugins/src/main/java/common/org/tron/plugins/DbLite.java b/plugins/src/main/java/common/org/tron/plugins/DbLite.java index 1a7e4e270f7..7f59e3b1e1f 100644 --- a/plugins/src/main/java/common/org/tron/plugins/DbLite.java +++ b/plugins/src/main/java/common/org/tron/plugins/DbLite.java @@ -9,11 +9,15 @@ import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.FileVisitOption; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.Arrays; +import java.util.Comparator; import java.util.List; +import java.util.Map; import java.util.NoSuchElementException; import java.util.Objects; import java.util.Optional; @@ -257,9 +261,10 @@ public void completeHistoryData(String historyDir, String liteDir) { // and return the block numbers of snapshot and history BlockNumInfo blockNumInfo = checkAndGetBlockNumInfo(historyDir, liteDir); // 2. move archive dbs to bak - backupArchiveDbs(liteDir); + Map movedDbs = getMovedArchiveDbs(liteDir); + backupArchiveDbs(liteDir, movedDbs); // 3. copy history data to liteDir - copyHistory2Database(historyDir, liteDir); + copyHistory2Database(historyDir, liteDir, movedDbs); // 4. delete the extra block data in history data trimExtraHistory(liteDir, blockNumInfo); // 5. merge bak to database @@ -554,7 +559,24 @@ private BlockNumInfo checkAndGetBlockNumInfo(String historyDir, String liteDir) return new BlockNumInfo(snapshotMinNum, historyMaxNum, snapshotMaxNum); } - private void backupArchiveDbs(String databaseDir) throws IOException { + @VisibleForTesting + static Map getMovedArchiveDbs(String databaseDir) { + Map movedDbs = Maps.newHashMap(); + try { + for (String db : archiveDbs) { + Path path = Paths.get(databaseDir, db); + if (Files.isSymbolicLink(path)) { + movedDbs.put(db, path.toRealPath()); + } + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + return movedDbs; + } + + private void backupArchiveDbs(String databaseDir, Map movedDbs) + throws IOException { Path bakDir = Paths.get(databaseDir, BACKUP_DIR_PREFIX + START_TIME); logger.info("Backup the archive dbs to {}.", bakDir); spec.commandLine().getOut().format("Backup the archive dbs to %s.", bakDir).println(); @@ -562,13 +584,61 @@ private void backupArchiveDbs(String databaseDir) throws IOException { throw new RuntimeException(String.format("create bak dir %s failed", bakDir)); } FileUtils.copyDatabases(Paths.get(databaseDir), bakDir, archiveDbs); - archiveDbs.forEach(db -> FileUtils.deleteDir(new File(databaseDir, db))); + for (String db : archiveDbs) { + Path moved = movedDbs.get(db); + if (moved == null) { + FileUtils.deleteDir(new File(databaseDir, db)); + } else { + clearDir(moved); + } + } } - private void copyHistory2Database(String historyDir, String databaseDir) throws IOException { + private void copyHistory2Database(String historyDir, String databaseDir, + Map movedDbs) throws IOException { logger.info("Begin to copy history to database."); spec.commandLine().getOut().println("Begin to copy history to database."); - FileUtils.copyDatabases(Paths.get(historyDir), Paths.get(databaseDir), archiveDbs); + List localDbs = archiveDbs.stream() + .filter(db -> !movedDbs.containsKey(db)) + .collect(Collectors.toList()); + FileUtils.copyDatabases(Paths.get(historyDir), Paths.get(databaseDir), localDbs); + for (Map.Entry moved : movedDbs.entrySet()) { + copyInto(Paths.get(historyDir, moved.getKey()), moved.getValue()); + } + } + + @VisibleForTesting + static void clearDir(Path dir) { + try (Stream walk = Files.walk(dir)) { + List entries = walk.filter(entry -> !entry.equals(dir)) + .sorted(Comparator.reverseOrder()) + .collect(Collectors.toList()); + for (Path entry : entries) { + Files.delete(entry); + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + @VisibleForTesting + static void copyInto(Path source, Path target) { + if (!Files.exists(source)) { + return; + } + try (Stream walk = Files.walk(source, FileVisitOption.FOLLOW_LINKS)) { + List entries = walk.collect(Collectors.toList()); + for (Path entry : entries) { + Path dest = target.resolve(source.relativize(entry)); + if (Files.isDirectory(entry)) { + Files.createDirectories(dest); + } else { + FileUtils.copy(entry, dest); + } + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } } private void trimExtraHistory(String liteDir, BlockNumInfo blockNumInfo) diff --git a/plugins/src/test/java/org/tron/plugins/DbLiteMergeFailureTest.java b/plugins/src/test/java/org/tron/plugins/DbLiteMergeFailureTest.java new file mode 100644 index 00000000000..6cca5d3c868 --- /dev/null +++ b/plugins/src/test/java/org/tron/plugins/DbLiteMergeFailureTest.java @@ -0,0 +1,145 @@ +package org.tron.plugins; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.List; +import org.junit.After; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.mockito.ArgumentMatchers; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import org.rocksdb.RocksDBException; +import org.tron.plugins.utils.db.DBInterface; +import org.tron.plugins.utils.db.DbTool; + +public class DbLiteMergeFailureTest { + + @Rule + public final TemporaryFolder folder = new TemporaryFolder(); + + @After + public void closeDatabases() { + DbTool.close(); + } + + @Test + public void testBackupFailureLeavesOriginalDbs() throws IOException, RocksDBException { + DbLiteMergeFixture fixture = movedFixture(); + byte[] originalInfo = Files.readAllBytes(fixture.lite.resolve("info.properties")); + Path current = fixture.lite.resolve("block").resolve("CURRENT"); + IOException failure = new IOException("Injected archive backup copy failure"); + + try (MockedStatic files = Mockito.mockStatic(Files.class, Mockito.CALLS_REAL_METHODS)) { + files.when(() -> Files.copy(ArgumentMatchers.eq(current), ArgumentMatchers.any(Path.class), + ArgumentMatchers.eq(StandardCopyOption.REPLACE_EXISTING))).thenThrow(failure); + assertMergeFailed(fixture, originalInfo, failure); + files.verify(() -> Files.copy(ArgumentMatchers.eq(current), ArgumentMatchers.any(Path.class), + ArgumentMatchers.eq(StandardCopyOption.REPLACE_EXISTING))); + } + + assertSnapshotData(fixture.movedRoot); + for (String db : DbLiteMergeFixture.archiveDbs()) { + assertTrue(db, Files.exists(fixture.movedDirectory(db) + .resolve(DbLiteMergeFixture.STALE_FILE))); + } + } + + @Test + public void testClearFailureKeepsRecoverableBackup() + throws IOException, RocksDBException { + DbLiteMergeFixture fixture = movedFixture(); + byte[] originalInfo = Files.readAllBytes(fixture.lite.resolve("info.properties")); + Path current = fixture.movedDirectory("trans").resolve("CURRENT"); + IOException failure = new IOException("Injected moved archive deletion failure"); + + try (MockedStatic files = Mockito.mockStatic(Files.class, Mockito.CALLS_REAL_METHODS)) { + files.when(() -> Files.delete(ArgumentMatchers.eq(current))).thenThrow(failure); + assertMergeFailed(fixture, originalInfo, failure); + files.verify(() -> Files.delete(current)); + } + + assertTrue(Files.exists(current)); + assertFalse(Files.exists(fixture.movedDirectory("block") + .resolve(DbLiteMergeFixture.STALE_FILE))); + assertFalse(Files.exists(fixture.movedDirectory("block-index") + .resolve(DbLiteMergeFixture.STALE_FILE))); + assertRecoverableBackup(fixture); + } + + @Test + public void testHistoryCopyFailureKeepsRecoverableBackup() + throws IOException, RocksDBException { + DbLiteMergeFixture fixture = movedFixture(); + byte[] originalInfo = Files.readAllBytes(fixture.lite.resolve("info.properties")); + Path target = fixture.movedDirectory("block"); + IOException failure = new IOException("Injected history directory creation failure"); + + try (MockedStatic files = Mockito.mockStatic(Files.class, Mockito.CALLS_REAL_METHODS)) { + files.when(() -> Files.createDirectories(ArgumentMatchers.eq(target))).thenThrow(failure); + assertMergeFailed(fixture, originalInfo, failure); + files.verify(() -> Files.createDirectories(target)); + } + + for (String db : DbLiteMergeFixture.archiveDbs()) { + assertFalse(db, Files.exists(fixture.movedDirectory(db) + .resolve(DbLiteMergeFixture.STALE_FILE))); + } + assertRecoverableBackup(fixture); + } + + private DbLiteMergeFixture movedFixture() throws IOException, RocksDBException { + DbLiteMergeFixture fixture = new DbLiteMergeFixture(folder, + DbLiteMergeFixture.SNAPSHOT_MAX - 1); + fixture.migrate(DbLiteMergeFixture.archiveDbs()); + return fixture; + } + + private static void assertMergeFailed(DbLiteMergeFixture fixture, byte[] originalInfo, + IOException failure) throws IOException { + int exitCode = fixture.executeMerge(); + assertEquals(fixture.error(), 1, exitCode); + assertTrue(fixture.error(), fixture.error().contains(failure.getMessage())); + fixture.assertLinks(); + assertArrayEquals(originalInfo, Files.readAllBytes(fixture.lite.resolve("info.properties"))); + assertEquals(1, fixture.backupDirectories().size()); + } + + private static void assertRecoverableBackup(DbLiteMergeFixture fixture) + throws IOException, RocksDBException { + List backups = fixture.backupDirectories(); + assertEquals(1, backups.size()); + assertSnapshotData(backups.get(0)); + for (String db : DbLiteMergeFixture.archiveDbs()) { + assertTrue(db, Files.exists(backups.get(0).resolve(db) + .resolve(DbLiteMergeFixture.STALE_FILE))); + } + } + + private static void assertSnapshotData(Path database) throws IOException, RocksDBException { + for (String db : DbLiteMergeFixture.archiveDbs()) { + DBInterface store = DbTool.getDB(database.toString(), db); + boolean containsGenesis = "block".equals(db) || "block-index".equals(db); + if (containsGenesis) { + assertArrayEquals(db + " snapshot genesis", DbLiteMergeFixture.value(db, 0), + store.get(DbLiteMergeFixture.key(db, 0))); + } + for (long number = DbLiteMergeFixture.SNAPSHOT_MIN; + number <= DbLiteMergeFixture.SNAPSHOT_MAX; number++) { + assertArrayEquals(db + " snapshot block " + number, + DbLiteMergeFixture.value(db, number), store.get(DbLiteMergeFixture.key(db, number))); + } + long expectedEntries = DbLiteMergeFixture.SNAPSHOT_MAX - DbLiteMergeFixture.SNAPSHOT_MIN + 1; + assertEquals(db + " snapshot entry count", expectedEntries + (containsGenesis ? 1 : 0), + store.size()); + } + } +} diff --git a/plugins/src/test/java/org/tron/plugins/DbLiteMergeFixture.java b/plugins/src/test/java/org/tron/plugins/DbLiteMergeFixture.java new file mode 100644 index 00000000000..140ee5488b5 --- /dev/null +++ b/plugins/src/test/java/org/tron/plugins/DbLiteMergeFixture.java @@ -0,0 +1,244 @@ +package org.tron.plugins; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import com.google.protobuf.ByteString; +import java.io.IOException; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.junit.rules.TemporaryFolder; +import org.rocksdb.RocksDBException; +import org.tron.plugins.utils.ByteArray; +import org.tron.plugins.utils.DBUtils; +import org.tron.plugins.utils.Sha256Hash; +import org.tron.plugins.utils.db.DBInterface; +import org.tron.plugins.utils.db.DbTool; +import org.tron.protos.Protocol; +import picocli.CommandLine; + +final class DbLiteMergeFixture { + + static final long SNAPSHOT_MIN = 4; + static final long SNAPSHOT_MAX = 5; + static final String STALE_FILE = "STALE"; + private static final String INFO_FILE = "info.properties"; + private static final String LATEST_BLOCK = "latest_block_header_number"; + private static final List ARCHIVE_DBS = Collections.unmodifiableList(Arrays.asList( + "block", "block-index", "trans", "transactionRetStore", "transactionHistoryStore")); + + final Path lite; + final Path history; + final Path movedRoot; + final CommandLine cli; + private final Path config; + private final long historyMax; + private final StringWriter output = new StringWriter(); + private final StringWriter error = new StringWriter(); + private List movedDbs = Collections.emptyList(); + + DbLiteMergeFixture(TemporaryFolder folder, long historyMax) + throws IOException, RocksDBException { + this.historyMax = historyMax; + lite = Files.createDirectories(folder.newFolder("lite").toPath().resolve("database")) + .toRealPath(); + history = folder.newFolder("history").toPath().toRealPath(); + movedRoot = Files.createDirectories(folder.newFolder("moved").toPath().resolve("database")) + .toRealPath(); + config = folder.getRoot().toPath().resolve("move.conf"); + cli = new CommandLine(new Toolkit()); + cli.setOut(new PrintWriter(output, true)); + cli.setErr(new PrintWriter(error, true)); + initializeStores(lite); + initializeStores(history); + putBlock(lite, 0); + for (long number = SNAPSHOT_MIN; number <= SNAPSHOT_MAX; number++) { + putBlock(lite, number); + } + for (long number = 0; number <= historyMax; number++) { + putBlock(history, number); + } + DbTool.getDB(lite.toString(), "properties", DbTool.DbType.RocksDB) + .put(ByteArray.fromString(LATEST_BLOCK), ByteArray.fromLong(SNAPSHOT_MAX)); + DbTool.getDB(lite.toString(), DBUtils.TMP, DbTool.DbType.RocksDB); + writeInfo(lite, SNAPSHOT_MIN); + writeInfo(history, historyMax); + DbTool.close(); + } + + static List archiveDbs() { + return ARCHIVE_DBS; + } + + void migrate(List databases) throws IOException { + movedDbs = databases; + String destination = movedRoot.getParent().toRealPath().toString(); + String properties = databases.stream() + .map(db -> "{ name = \"" + db + "\", path = \"" + destination + "\" }") + .collect(Collectors.joining(",\n", "storage.properties = [\n", "\n]\n")); + Files.write(config, properties.getBytes(StandardCharsets.UTF_8)); + assertEquals(error.toString(), 0, cli.execute("db", "mv", "-d", + lite.getParent().toString(), "-c", config.toString())); + assertLinks(); + for (String db : databases) { + Files.write(movedDirectory(db).resolve(STALE_FILE), new byte[] {1}); + } + } + + int executeMerge() { + return cli.execute("db", "lite", "-o", "merge", "-fn", lite.toString(), + "-ds", history.toString()); + } + + String output() { + return output.toString(); + } + + String error() { + return error.toString(); + } + + Path movedDirectory(String db) { + return movedRoot.resolve(db); + } + + List backupDirectories() throws IOException { + try (Stream entries = Files.list(lite)) { + return entries.filter(entry -> entry.getFileName().toString().startsWith(".bak_")) + .collect(Collectors.toList()); + } + } + + void assertLinks() throws IOException { + for (String db : movedDbs) { + Path link = lite.resolve(db); + assertTrue(db, Files.isSymbolicLink(link)); + assertEquals(db, movedDirectory(db).toRealPath(), link.toRealPath()); + } + } + + void assertMergedData() throws IOException, RocksDBException { + assertLinks(); + assertFalse(Files.exists(lite.resolve(INFO_FILE))); + assertTrue(backupDirectories().isEmpty()); + assertTrue(output(), output().contains("Merge history finished")); + for (String db : ARCHIVE_DBS) { + Path database = movedDbs.contains(db) ? movedRoot : lite; + assertFalse(db, Files.exists(database.resolve(db).resolve(STALE_FILE))); + assertEquals(db, DbTool.DbType.RocksDB, DbTool.getDbType(database.toString(), db)); + assertStoreData(database, db); + } + } + + static byte[] value(String db, long number) { + switch (db) { + case "block-index": + return blockId(number); + case "block": + return block(number).toByteArray(); + case "trans": + return ByteArray.fromLong(number); + case "transactionRetStore": + return Protocol.TransactionRet.newBuilder().setBlockNumber(number) + .addTransactioninfo(transactionInfo(number)).build().toByteArray(); + case "transactionHistoryStore": + return transactionInfo(number).toByteArray(); + default: + throw new IllegalArgumentException("Unexpected archive database: " + db); + } + } + + static byte[] key(String db, long number) { + switch (db) { + case "block": + return blockId(number); + case "trans": + case "transactionHistoryStore": + return DBUtils.getTransactionId(transaction(number)).getBytes(); + default: + return ByteArray.fromLong(number); + } + } + + private void assertStoreData(Path database, String db) throws IOException, RocksDBException { + DBInterface store = DbTool.getDB(database.toString(), db); + boolean containsGenesis = "block".equals(db) || "block-index".equals(db); + if (containsGenesis) { + assertArrayEquals(db + " genesis", value(db, 0), store.get(key(db, 0))); + } + for (long number = 1; number <= SNAPSHOT_MAX; number++) { + assertArrayEquals(db + " block " + number, value(db, number), store.get(key(db, number))); + } + // Existing trimming excludes transactionHistoryStore; this test preserves that behavior. + boolean retainsExtra = "transactionHistoryStore".equals(db) && historyMax > SNAPSHOT_MAX; + long expectedEntries = SNAPSHOT_MAX + (containsGenesis ? 1 : 0); + if (retainsExtra) { + assertArrayEquals(db + " retained extra", value(db, historyMax), + store.get(key(db, historyMax))); + expectedEntries += historyMax - SNAPSHOT_MAX; + } else { + assertNull(db + " extra", store.get(key(db, SNAPSHOT_MAX + 1))); + } + assertEquals(db + " entry count", expectedEntries, store.size()); + } + + private static void initializeStores(Path database) throws IOException, RocksDBException { + for (String db : ARCHIVE_DBS) { + DbTool.getDB(database.toString(), db, DbTool.DbType.RocksDB); + } + } + + private static void putBlock(Path database, long number) throws IOException, RocksDBException { + for (String db : ARCHIVE_DBS) { + if (number != 0 || "block".equals(db) || "block-index".equals(db)) { + DbTool.getDB(database.toString(), db).put(key(db, number), value(db, number)); + } + } + } + + private static void writeInfo(Path database, long number) throws IOException { + String property = DBUtils.SPLIT_BLOCK_NUM + "=" + number + "\n"; + Files.write(database.resolve(INFO_FILE), property.getBytes(StandardCharsets.UTF_8)); + } + + private static Protocol.Transaction transaction(long number) { + Protocol.Transaction.raw raw = Protocol.Transaction.raw.newBuilder() + .setTimestamp(number).setExpiration(number + 1).build(); + return Protocol.Transaction.newBuilder().setRawData(raw).build(); + } + + private static Protocol.TransactionInfo transactionInfo(long number) { + ByteString id = ByteString.copyFrom(DBUtils.getTransactionId(transaction(number)).getBytes()); + return Protocol.TransactionInfo.newBuilder().setId(id).setBlockNumber(number).build(); + } + + private static Protocol.Block block(long number) { + Protocol.BlockHeader.raw raw = Protocol.BlockHeader.raw.newBuilder() + .setNumber(number).setTimestamp(number + 1).build(); + Protocol.BlockHeader header = Protocol.BlockHeader.newBuilder().setRawData(raw).build(); + Protocol.Block.Builder block = Protocol.Block.newBuilder().setBlockHeader(header); + if (number != 0) { + block.addTransactions(transaction(number)); + } + return block.build(); + } + + private static byte[] blockId(long number) { + byte[] header = block(number).getBlockHeader().getRawData().toByteArray(); + byte[] hash = Sha256Hash.of(header).getBytes(); + byte[] height = ByteArray.fromLong(number); + System.arraycopy(height, 0, hash, 0, height.length); + return hash; + } +} diff --git a/plugins/src/test/java/org/tron/plugins/DbLiteMergeTest.java b/plugins/src/test/java/org/tron/plugins/DbLiteMergeTest.java new file mode 100644 index 00000000000..badc19df4d3 --- /dev/null +++ b/plugins/src/test/java/org/tron/plugins/DbLiteMergeTest.java @@ -0,0 +1,44 @@ +package org.tron.plugins; + +import static org.junit.Assert.assertEquals; + +import java.io.IOException; +import org.junit.After; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.rocksdb.RocksDBException; +import org.tron.plugins.utils.db.DbTool; + +public class DbLiteMergeTest { + + @Rule + public final TemporaryFolder folder = new TemporaryFolder(); + + @After + public void closeDatabases() { + DbTool.close(); + } + + @Test + public void testMovedDbsMergeNewerSnapshot() throws IOException, RocksDBException { + assertMovedMerge(DbLiteMergeFixture.SNAPSHOT_MAX - 1); + } + + @Test + public void testMovedDbsMergeMatchingHistory() throws IOException, RocksDBException { + assertMovedMerge(DbLiteMergeFixture.SNAPSHOT_MAX); + } + + @Test + public void testMovedDbsTrimNewerHistory() throws IOException, RocksDBException { + assertMovedMerge(DbLiteMergeFixture.SNAPSHOT_MAX + 1); + } + + private void assertMovedMerge(long historyMax) throws IOException, RocksDBException { + DbLiteMergeFixture fixture = new DbLiteMergeFixture(folder, historyMax); + fixture.migrate(DbLiteMergeFixture.archiveDbs()); + assertEquals(fixture.error(), 0, fixture.executeMerge()); + fixture.assertMergedData(); + } +} diff --git a/plugins/src/test/java/org/tron/plugins/DbLiteMovedDbTest.java b/plugins/src/test/java/org/tron/plugins/DbLiteMovedDbTest.java new file mode 100644 index 00000000000..2605acc902e --- /dev/null +++ b/plugins/src/test/java/org/tron/plugins/DbLiteMovedDbTest.java @@ -0,0 +1,67 @@ +package org.tron.plugins; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.nio.file.Path; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.mockito.ArgumentMatchers; +import org.mockito.MockedStatic; +import org.mockito.Mockito; + +public class DbLiteMovedDbTest { + + @Rule + public final TemporaryFolder folder = new TemporaryFolder(); + + @Test + public void testDanglingMovedDbPropagates() throws IOException { + Path database = folder.newFolder("database").toPath(); + Path link = database.resolve("block"); + Files.createSymbolicLink(link, database.resolveSibling("missing")); + + UncheckedIOException error = assertThrows(UncheckedIOException.class, + () -> DbLite.getMovedArchiveDbs(database.toString())); + assertTrue(error.getCause() instanceof NoSuchFileException); + assertEquals(link.toString(), ((NoSuchFileException) error.getCause()).getFile()); + } + + @Test + public void testClearFailurePropagates() throws IOException { + Path moved = folder.newFolder("trans").toPath(); + Path current = Files.write(moved.resolve("CURRENT"), new byte[] {1}); + IOException failure = new IOException("Read-only file system"); + + try (MockedStatic files = Mockito.mockStatic(Files.class, Mockito.CALLS_REAL_METHODS)) { + files.when(() -> Files.delete(ArgumentMatchers.eq(current))).thenThrow(failure); + UncheckedIOException error = assertThrows(UncheckedIOException.class, + () -> DbLite.clearDir(moved)); + assertSame(failure, error.getCause()); + } + assertTrue(Files.exists(current)); + } + + @Test + public void testCopyFailurePropagates() throws IOException { + Path history = folder.newFolder("history-trans").toPath(); + Files.createDirectories(history.resolve("nested")); + Path moved = folder.newFolder("trans").toPath(); + IOException failure = new IOException("No space left on device"); + + try (MockedStatic files = Mockito.mockStatic(Files.class, Mockito.CALLS_REAL_METHODS)) { + files.when(() -> Files.createDirectories(ArgumentMatchers.eq(moved.resolve("nested")))) + .thenThrow(failure); + UncheckedIOException error = assertThrows(UncheckedIOException.class, + () -> DbLite.copyInto(history, moved)); + assertSame(failure, error.getCause()); + } + } +} diff --git a/plugins/src/test/java/org/tron/plugins/DbLiteTest.java b/plugins/src/test/java/org/tron/plugins/DbLiteTest.java index 4ee7567ec28..858f7180a51 100644 --- a/plugins/src/test/java/org/tron/plugins/DbLiteTest.java +++ b/plugins/src/test/java/org/tron/plugins/DbLiteTest.java @@ -1,6 +1,8 @@ package org.tron.plugins; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.tron.common.utils.PublicMethod.getRandomPrivateKey; @@ -8,12 +10,18 @@ import io.grpc.ManagedChannelBuilder; import java.io.File; import java.io.IOException; +import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.Collections; +import java.util.List; +import java.util.stream.Stream; import lombok.extern.slf4j.Slf4j; import org.junit.After; import org.junit.Rule; import org.junit.rules.TemporaryFolder; +import org.rocksdb.RocksDBException; +import org.tron.api.GrpcAPI.EmptyMessage; import org.tron.api.WalletGrpc; import org.tron.common.TestConstants; import org.tron.common.application.Application; @@ -26,6 +34,10 @@ import org.tron.common.utils.Utils; import org.tron.core.config.DefaultConfig; import org.tron.core.config.args.Args; +import org.tron.plugins.utils.ByteArray; +import org.tron.plugins.utils.DBUtils; +import org.tron.plugins.utils.FileUtils; +import org.tron.plugins.utils.db.DbTool; import picocli.CommandLine; @Slf4j @@ -40,6 +52,10 @@ public class DbLiteTest { @Rule public final TemporaryFolder folder = new TemporaryFolder(); + private static final String STALE_FILE = "STALE"; + private static final int RECENT_BLKS = 3; + private static final long HEAD_BLOCK_TIMEOUT_MS = 60_000; + private String dbPath; CommandLine cli = new CommandLine(new DbLite()); @@ -115,11 +131,12 @@ public void testTools(String dbType, int checkpointVersion, boolean excludeHisto new String[] {"-o", "merge", "--fn-data-path", dbPath + File.separator + databaseDir, "--dataset-path", dbPath + File.separator + "history"}; Args.getInstance().getStorage().setCheckpointVersion(checkpointVersion); - DbLite.setRecentBlks(3); + DbLite.setRecentBlks(RECENT_BLKS); // start fullNode startApp(); // produce transactions for 18 seconds generateSomeTransactions(18); + awaitHeadBlock(RECENT_BLKS + 2); // stop the node shutdown(); // delete tran-cache @@ -164,7 +181,13 @@ public void testTools(String dbType, int checkpointVersion, boolean excludeHisto // stop the node shutdown(); // merge history - cli.execute(argsForMerge); + Path liteDatabase = Paths.get(dbPath, databaseDir); + long snapshotMin = Long.parseLong(FileUtils.readProperty( + liteDatabase.resolve("info.properties").toString(), DBUtils.SPLIT_BLOCK_NUM)); + Path hdd = moveDbs(liteDatabase); + assertEquals(0, cli.execute(argsForMerge)); + assertMergeCompleted(liteDatabase); + assertMovedDbsMerged(liteDatabase, hdd, snapshotMin); // start and validate startApp(); generateSomeTransactions(6); @@ -172,6 +195,66 @@ public void testTools(String dbType, int checkpointVersion, boolean excludeHisto DbLite.reSetRecentBlks(); } + protected List movedDbs() { + return Collections.emptyList(); + } + + private Path moveDbs(Path database) throws IOException { + if (movedDbs().isEmpty()) { + return null; + } + Path hdd = folder.newFolder("hdd").toPath().toRealPath(); + for (String db : movedDbs()) { + Path target = hdd.resolve(db); + Files.move(database.resolve(db), target); + Files.write(target.resolve(STALE_FILE), new byte[] {1}); + Files.createSymbolicLink(database.resolve(db), target); + } + return hdd; + } + + private static void assertMergeCompleted(Path database) throws IOException { + assertFalse(Files.exists(database.resolve("info.properties"))); + try (Stream entries = Files.list(database)) { + assertTrue(entries.noneMatch(entry -> entry.getFileName().toString().startsWith(".bak_"))); + } + } + + private void assertMovedDbsMerged(Path database, Path hdd, long snapshotMin) + throws IOException { + if (movedDbs().isEmpty()) { + return; + } + for (String db : movedDbs()) { + assertEquals(hdd.resolve(db), Files.readSymbolicLink(database.resolve(db))); + assertFalse(Files.exists(hdd.resolve(db).resolve(STALE_FILE))); + } + assertTrue("the snapshot must start after block 1, got " + snapshotMin, snapshotMin > 1); + try { + byte[] historyOnly = DbTool.getDB(database.toString(), "block-index") + .get(ByteArray.fromLong(snapshotMin - 1)); + assertNotNull(historyOnly); + assertNotNull(DbTool.getDB(hdd.toString(), "block").get(historyOnly)); + } catch (RocksDBException e) { + throw new IOException(e); + } finally { + DbTool.close(); + } + } + + private void awaitHeadBlock(long number) throws InterruptedException { + long deadline = System.currentTimeMillis() + HEAD_BLOCK_TIMEOUT_MS; + while (headBlockNumber() < number) { + assertTrue("head block did not reach " + number, System.currentTimeMillis() < deadline); + Thread.sleep(500); + } + } + + private long headBlockNumber() { + return blockingStubFull.getNowBlock(EmptyMessage.getDefaultInstance()) + .getBlockHeader().getRawData().getNumber(); + } + private void generateSomeTransactions(int during) { during *= 1000; // ms int runTime = 0; diff --git a/plugins/src/test/java/org/tron/plugins/leveldb/DbLiteLevelDbTest.java b/plugins/src/test/java/org/tron/plugins/leveldb/DbLiteLevelDbTest.java index 7666806e2b5..a79de6ef800 100644 --- a/plugins/src/test/java/org/tron/plugins/leveldb/DbLiteLevelDbTest.java +++ b/plugins/src/test/java/org/tron/plugins/leveldb/DbLiteLevelDbTest.java @@ -1,6 +1,8 @@ package org.tron.plugins.leveldb; import java.io.IOException; +import java.util.Arrays; +import java.util.List; import org.junit.Test; import org.tron.plugins.DbLiteTest; @@ -11,4 +13,9 @@ public class DbLiteLevelDbTest extends DbLiteTest { public void testToolsWithLevelDB() throws InterruptedException, IOException { testTools("LEVELDB", 1); } + + @Override + protected List movedDbs() { + return Arrays.asList("block", "trans"); + } } diff --git a/plugins/src/test/java/org/tron/plugins/leveldb/DbLiteLevelDbV2Test.java b/plugins/src/test/java/org/tron/plugins/leveldb/DbLiteLevelDbV2Test.java index de32ae29c7c..532691b94f1 100644 --- a/plugins/src/test/java/org/tron/plugins/leveldb/DbLiteLevelDbV2Test.java +++ b/plugins/src/test/java/org/tron/plugins/leveldb/DbLiteLevelDbV2Test.java @@ -1,6 +1,8 @@ package org.tron.plugins.leveldb; import java.io.IOException; +import java.util.Arrays; +import java.util.List; import org.junit.Test; import org.tron.plugins.DbLiteTest; @@ -10,4 +12,9 @@ public class DbLiteLevelDbV2Test extends DbLiteTest { public void testToolsWithLevelDBV2() throws InterruptedException, IOException { testTools("LEVELDB", 2); } + + @Override + protected List movedDbs() { + return Arrays.asList("block", "trans"); + } } diff --git a/plugins/src/test/java/org/tron/plugins/rocksdb/DbLiteRocksDbTest.java b/plugins/src/test/java/org/tron/plugins/rocksdb/DbLiteRocksDbTest.java index 2f9c92f9679..498be8114c2 100644 --- a/plugins/src/test/java/org/tron/plugins/rocksdb/DbLiteRocksDbTest.java +++ b/plugins/src/test/java/org/tron/plugins/rocksdb/DbLiteRocksDbTest.java @@ -1,6 +1,8 @@ package org.tron.plugins.rocksdb; import java.io.IOException; +import java.util.Arrays; +import java.util.List; import org.junit.Test; import org.tron.plugins.DbLiteTest; @@ -10,4 +12,9 @@ public class DbLiteRocksDbTest extends DbLiteTest { public void testToolsWithRocksDB() throws InterruptedException, IOException { testTools("ROCKSDB", 1); } + + @Override + protected List movedDbs() { + return Arrays.asList("block", "trans"); + } } diff --git a/plugins/src/test/java/org/tron/plugins/rocksdb/DbLiteRocksDbV2Test.java b/plugins/src/test/java/org/tron/plugins/rocksdb/DbLiteRocksDbV2Test.java index ebc4074ccc0..0aab307541d 100644 --- a/plugins/src/test/java/org/tron/plugins/rocksdb/DbLiteRocksDbV2Test.java +++ b/plugins/src/test/java/org/tron/plugins/rocksdb/DbLiteRocksDbV2Test.java @@ -1,6 +1,8 @@ package org.tron.plugins.rocksdb; import java.io.IOException; +import java.util.Arrays; +import java.util.List; import org.junit.Test; import org.tron.plugins.DbLiteTest; @@ -10,4 +12,9 @@ public class DbLiteRocksDbV2Test extends DbLiteTest { public void testToolsWithRocksDbV2() throws InterruptedException, IOException { testTools("ROCKSDB", 2); } + + @Override + protected List movedDbs() { + return Arrays.asList("block", "trans"); + } }