HBASE-29641 Simpler incremental backup test for archived files

Integrates the test
TestIncrementalBackup#TestIncBackupRestoreHandlesArchivedFiles
into the existing test
TestIncrementalBackupWithBulkLoad#testIncBackupWithBulkLoads.

Author: DieterDP-ng

hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackup.java

@@ -59,7 +59,6 @@
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.CommonFSUtils;
 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
-import org.apache.hadoop.hbase.util.HFileArchiveUtil;
 import org.apache.hadoop.hbase.util.HFileTestUtil;
 import org.junit.After;
 import org.junit.Assert;
@@ -433,73 +432,6 @@ public void TestIncBackupRestoreWithOriginalSplitsSeperateFs() throws Exception
 
   }
 
-  @Test
-  public void TestIncBackupRestoreHandlesArchivedFiles() throws Exception {
-    byte[] fam2 = Bytes.toBytes("f2");
-    TableDescriptor newTable1Desc = TableDescriptorBuilder.newBuilder(table1Desc)
-      .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(fam2).build()).build();
-    TEST_UTIL.getAdmin().modifyTable(newTable1Desc);
-    try (Connection conn = ConnectionFactory.createConnection(conf1);
-      BackupAdminImpl admin = new BackupAdminImpl(conn)) {
-      String backupTargetDir = TEST_UTIL.getDataTestDir("backupTarget").toString();
-      BACKUP_ROOT_DIR = new File(backupTargetDir).toURI().toString();
-
-      List<TableName> tables = Lists.newArrayList(table1);
-
-      insertIntoTable(conn, table1, famName, 3, 100);
-      String fullBackupId = takeFullBackup(tables, admin, true);
-      assertTrue(checkSucceeded(fullBackupId));
-
-      insertIntoTable(conn, table1, famName, 4, 100);
-
-      HRegion regionToBulkload = TEST_UTIL.getHBaseCluster().getRegions(table1).get(0);
-      String regionName = regionToBulkload.getRegionInfo().getEncodedName();
-      // Requires a mult-fam bulkload to ensure we're appropriately handling
-      // multi-file bulkloads
-      Path regionDir = doBulkload(table1, regionName, famName, fam2);
-
-      // archive the files in the region directory
-      Path archiveDir =
-        HFileArchiveUtil.getStoreArchivePath(conf1, table1, regionName, Bytes.toString(famName));
-      TEST_UTIL.getTestFileSystem().mkdirs(archiveDir);
-      RemoteIterator<LocatedFileStatus> iter =
-        TEST_UTIL.getTestFileSystem().listFiles(regionDir, true);
-      List<Path> paths = new ArrayList<>();
-      while (iter.hasNext()) {
-        Path path = iter.next().getPath();
-        if (path.toString().contains("_SeqId_")) {
-          paths.add(path);
-        }
-      }
-      assertTrue(paths.size() > 1);
-      Path path = paths.get(0);
-      String name = path.toString();
-      int startIdx = name.lastIndexOf(Path.SEPARATOR);
-      String filename = name.substring(startIdx + 1);
-      Path archiveFile = new Path(archiveDir, filename);
-      // archive 1 of the files
-      boolean success = TEST_UTIL.getTestFileSystem().rename(path, archiveFile);
-      assertTrue(success);
-      assertTrue(TEST_UTIL.getTestFileSystem().exists(archiveFile));
-      assertFalse(TEST_UTIL.getTestFileSystem().exists(path));
-
-      BackupRequest request =
-        createBackupRequest(BackupType.INCREMENTAL, tables, BACKUP_ROOT_DIR, true);
-      String incrementalBackupId = admin.backupTables(request);
-      assertTrue(checkSucceeded(incrementalBackupId));
-
-      TableName[] fromTable = new TableName[] { table1 };
-      TableName[] toTable = new TableName[] { table1_restore };
-
-      admin.restore(BackupUtils.createRestoreRequest(BACKUP_ROOT_DIR, incrementalBackupId, false,
-        fromTable, toTable, true));
-
-      int actualRowCount = TEST_UTIL.countRows(table1_restore);
-      int expectedRowCount = TEST_UTIL.countRows(table1);
-      assertEquals(expectedRowCount, actualRowCount);
-    }
-  }
-
   private void checkThrowsCFMismatch(IOException ex, List<TableName> tables) {
     Throwable cause = Throwables.getRootCause(ex);
     assertEquals(cause.getClass(), ColumnFamilyMismatchException.class);

hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupWithBulkLoad.java

@@ -62,7 +62,7 @@ public class TestIncrementalBackupWithBulkLoad extends TestBackupBase {
 
   // implement all test cases in 1 test since incremental backup/restore has dependencies
   @Test
-  public void TestIncBackupDeleteTable() throws Exception {
+  public void testIncBackupWithBulkLoads() throws Exception {
     try (BackupSystemTable systemTable = new BackupSystemTable(TEST_UTIL.getConnection())) {
       // The test starts with some data, and no bulk loaded rows.
       int expectedRowCount = NB_ROWS_IN_BATCH;
@@ -99,13 +99,19 @@ public void TestIncBackupDeleteTable() throws Exception {
       assertEquals(expectedRowCount, TEST_UTIL.countRows(table1));
       assertEquals(0, systemTable.readBulkloadRows(List.of(table1)).size());
 
-      // Creating an incremental backup clears the bulk loads
+      // Create bulk loads that will end up in the incremental backup.
       performBulkLoad("bulk4");
       performBulkLoad("bulk5");
       performBulkLoad("bulk6");
       expectedRowCount += 3 * ROWS_IN_BULK_LOAD;
       assertEquals(expectedRowCount, TEST_UTIL.countRows(table1));
-      assertEquals(3, systemTable.readBulkloadRows(List.of(table1)).size());
+      List<BulkLoad> bulkLoadsForIncrBackup = systemTable.readBulkloadRows(List.of(table1));
+      assertEquals(3, bulkLoadsForIncrBackup.size());
+
+      // Manually move one bulk loaded file to simulate archival
+      simulateArchival(bulkLoadsForIncrBackup.get(0));
+
+      // Creating an incremental backup clears the bulk loads
       String backup3 = backupTables(BackupType.INCREMENTAL, List.of(table1), BACKUP_ROOT_DIR);
       assertTrue(checkSucceeded(backup3));
       assertEquals(expectedRowCount, TEST_UTIL.countRows(table1));
@@ -256,4 +262,17 @@ private void performBulkLoad(String keyPrefix) throws IOException {
       BulkLoadHFiles.create(TEST_UTIL.getConfiguration()).bulkLoad(table1, baseDirectory);
     assertFalse(result.isEmpty());
   }
+
+  private void simulateArchival(BulkLoad bulkLoad) throws IOException {
+    FileSystem fs = TEST_UTIL.getTestFileSystem();
+    Path rootDir = CommonFSUtils.getRootDir(TEST_UTIL.getConfiguration());
+    Path archiveRoot = HFileArchiveUtil.getArchivePath(TEST_UTIL.getConfiguration());
+
+    Path srcPath = new Path(bulkLoad.getHfilePath());
+    String relativePath = rootDir.toUri().relativize(srcPath.toUri()).getPath();
+    Path tgtPath = new Path(archiveRoot, relativePath);
+
+    fs.mkdirs(tgtPath.getParent());
+    assertTrue("Renaming HFile to simulate archival failed", fs.rename(srcPath, tgtPath));
+  }
 }