diff --git a/tests/test_backup.py b/tests/test_backup.py index 1fea7ecd..150b9e16 100644 --- a/tests/test_backup.py +++ b/tests/test_backup.py @@ -11,10 +11,48 @@ BackupGetError, BackupRestoreError, BackupUploadError, + ReplicationClusterInventoryError, ) from tests.helpers import assert_raises +def wait_for_cluster_resilient(sys_db): + collections_in_sync = False + max_attempts = 100 + + while not collections_in_sync and max_attempts > 0: + count_in_sync = 0 + count_still_waiting = 0 + + try: + inventory = sys_db.replication.cluster_inventory(include_system=True) + except ReplicationClusterInventoryError: + print("Failed to get cluster inventory, retrying...") + time.sleep(1) + max_attempts -= 1 + continue + + collections_in_sync = True + for col in inventory["collections"]: + if not col["all_in_sync"]: + count_still_waiting += 1 + collections_in_sync = False + else: + count_in_sync += 1 + + if not collections_in_sync: + if max_attempts % 50 == 0: + print(inventory) + print(f"In sync: {count_in_sync}") + print(f"Still not in sync: {count_still_waiting}") + time.sleep(1) + + max_attempts -= 1 + + if not collections_in_sync: + raise Exception("Collections didn't come in sync!") + + def test_backup_management(sys_db, bad_db, cluster, skip_tests, db_version): if "enterprise" in skip_tests: pytest.skip("Only for ArangoDB enterprise edition") @@ -112,8 +150,7 @@ def test_backup_management(sys_db, bad_db, cluster, skip_tests, db_version): result = sys_db.backup.restore(backup_id_foo) assert isinstance(result, dict) - # Wait for restore to complete - time.sleep(10) + wait_for_cluster_resilient(sys_db) # Test restore backup with bad database. with assert_raises(BackupRestoreError) as err: