Improve backup consistency during snapshot

We now create the snapshot as "next" and only promote it to "current"
once it finishes. WAL files are linked to both current and next.

1 files changed, 24 insertions, 23 deletions

diff --git a/test.py b/test.py
index 163c589..ec0c5b8 100644
--- a/test.py
+++ b/test.py
@@ -25,8 +25,7 @@ class Test(unittest.TestCase):
         self._wal_req = threading.Condition(self._wal_lock)
         self._wal_done = threading.Condition(self._wal_lock)
 
-        self._bak_relax = False
-        self._bak_list = []
+        self._bak_list = [(0, 0)]
 
         self._prog = os.path.realpath('./pgbak')
         self._sock,self._sock_prog = socket.socketpair(socket.AF_UNIX)
@@ -42,12 +41,12 @@ class Test(unittest.TestCase):
         os.mkdir(os.path.join(self._db_path, 'pg_wal'))
 
         with open(os.path.join(self._bin_path, 'pg_basebackup'), 'w') as f:
-            f.write('#!/bin/sh\necho -n r >&0; read t; cat base; sleep "$t"; [ "$((RANDOM%200))" != 0 ]\n')
+            f.write('#!/bin/sh\necho -n r >&0; read t; [ "$t" != k ] || exec kill -9 "$PPID"; cat base; sleep "$t"; [ "$((RANDOM%200))" != 0 ]\n')
             os.fchmod(f.fileno(), 0o755)
 
         with open(os.path.join(self._bak_path, 'scripts', 'backup'), 'w') as f:
             shp = shlex.quote(self._tmpdir)
-            f.write(f'#!/bin/sh\nset -e; sleep 0.1; [ "$((RANDOM%200))" == 0 ] && exit 1; tar -c . > {shp}/bak-dir.tar; echo -n w >&0; read t\n')
+            f.write(f'#!/bin/sh\nset -e; sleep 0.1; [ "$((RANDOM%200))" == 0 ] && exit 1; tar -c . > {shp}/bak-dir.tar; echo -n w >&0; read t; [ "$t" != k ] || kill -9 "$PPID"\n')
             os.fchmod(f.fileno(), 0o755)
 
         self._wal_thr = threading.Thread(target=self._run_wal)
@@ -84,7 +83,11 @@ class Test(unittest.TestCase):
 
             if op == b'w':
                 self._bak_read()
-                self._sock.send(b'.\n')
+                if random.randrange(10) == 0:
+                    print('>>> killing backup')
+                    self._sock.send(b'k\n')
+                else:
+                    self._sock.send(b'.\n')
                 continue
 
             if op == b'r':
@@ -92,9 +95,13 @@ class Test(unittest.TestCase):
                 i = self._wal_make()
                 with open(os.path.join(self._db_path, 'base'), 'wb') as f:
                     f.write(f'up to {i}\n'.encode() + os.urandom(20480))
-                t = random.random() * 0.2
-                self._wal_make()
-                self._sock.send(f'{t}\n'.encode())
+                if random.randrange(10) == 0:
+                    print('>>> killing snapshot')
+                    self._sock.send(b'k\n')
+                else:
+                    t = random.random() * 0.2
+                    self._wal_make()
+                    self._sock.send(f'{t}\n'.encode())
                 continue
 
             raise RuntimeError(f'bad command {op!r}')
@@ -154,10 +161,11 @@ class Test(unittest.TestCase):
         while end in wals:
             end += 1
         end -= 1
-        if end < begin + 1 and not self._bak_relax:
+        if end < begin + 1:
             raise RuntimeError('missing first WAL')
 
-        self._bak_list.append(end)
+        assert len(self._bak_list) == 1 or begin <= self._bak_list[-1][1]
+        self._bak_list.append((begin, end))
 
     def _run(self, *args):
         env = {**os.environ}
@@ -170,19 +178,12 @@ class Test(unittest.TestCase):
             i = self._wal_make()
             if random.randrange(20) == 0:
                 self._run('wait')
-                if not self._bak_relax:
-                    self.assertGreaterEqual(self._bak_list[-1], i)
-                self._bak_relax = False
-
-            if random.randrange(30) == 0:
-                # Relax the backup consistency requirement until the end of the next sync
-                self._bak_relax = True
-                try:
-                    os.unlink(os.path.join(self._bak_path, 'current'))
-                except FileNotFoundError:
-                    pass
-                else:
-                    print('>>> forcing full backup')
+                self.assertGreaterEqual(self._bak_list[-1][1], i)
+
+            elif random.randrange(30) == 0:
+                print('>>> forcing full backup')
+                self._run('full-sync')
+                self.assertGreaterEqual(self._bak_list[-1][1], i)
 
 if __name__ == '__main__':
     unittest.main()