start_cluster = True, cluster = <cluster.Cluster object at 0x7fe02f8d3390>
backup_type = 'physical'

    @pytest.mark.timeout(900,func_only=True)
    @pytest.mark.parametrize('backup_type',['logical','physical'])
    def test_pitr_PBM_T268(start_cluster,cluster,backup_type):
        def insert_docs():
            client=pymongo.MongoClient(cluster.connection)
            for i in range(1500):
                client['test']['test'].insert_one({"doc":i})
                time.sleep(0.1)
    
        cluster.check_pbm_status()
        base_backup=cluster.make_backup(backup_type)
        cluster.enable_pitr(pitr_extra_args="--set pitr.oplogSpanMin=0.5")
        Cluster.log("Start inserting docs in the background")
        background_insert = threading.Thread(target=insert_docs)
        background_insert.start()
        time.sleep(60)
        Cluster.log("Check if PITR is running")
        if not cluster.check_pitr():
            logs=cluster.exec_pbm_cli("logs -sD -t0")
            assert False, logs.stdout
        time.sleep(60)
        background_insert.join()
        time.sleep(30)
        assert pymongo.MongoClient(cluster.connection)["test"]["test"].count_documents({}) == 1500
        pitr = datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S")
        Cluster.log("Time for PITR is " + pitr)
        time.sleep(60)
        cluster.disable_pitr()
        pymongo.MongoClient(cluster.connection).drop_database('test')
        backup="--time=" + pitr
        if backup_type == 'logical':
            cluster.make_restore(backup, check_pbm_status=True)
        else:
>           cluster.make_restore(backup, restart_cluster=True, check_pbm_status=True)

test_PBM-1211.py:79: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <cluster.Cluster object at 0x7fe02f8d3390>
name = '--time=2024-11-06T14:51:57'
kwargs = {'check_pbm_status': True, 'restart_cluster': True}
client = MongoClient(host=['mongos:27017'], document_class=dict, tz_aware=False, connect=True)
result = CommandResult(backend=<testinfra.backend.docker.DockerBackend object at 0x7fe031bfcc90>, exit_status=1, command=b'time...AAAAABJ0AP//////////AA==","MultiOpType":null} | merr <nil>: applyOps: (NotWritablePrimary) not primary\n', _stderr=b'')
n = <testinfra.host.Host docker://rscfg01>, timeout = 240
error = '{"t":{"$date":"2024-11-06T14:53:42.913+00:00"},"s":"I",  "c":"NETWORK",  "id":4915701, "ctx":"main","msg":"Initialize...ckpointer","msg":"Frame","attr":{"frame":{"a":"7F68DFF368D3","b":"7F68DFEFD000","o":"398D3","s":"clone","s+":"43"}}}\n'
host = 'rscfg01', container = <Container: 051f1369b71c>

    def make_restore(self, name, **kwargs):
        if self.layout == "sharded":
            client = pymongo.MongoClient(self.connection)
            result = client.admin.command("balancerStop")
            client.close()
            Cluster.log("Stopping balancer: " + str(result))
            self.stop_mongos()
        self.stop_arbiters()
        n = testinfra.get_host("docker://" + self.pbm_cli)
        timeout = time.time() + 60
    
        while True:
            if not self.get_status()['running']:
                break
            if time.time() > timeout:
                assert False, "Cannot start restore, another operation running"
            time.sleep(1)
        Cluster.log("Restore started")
        timeout=kwargs.get('timeout', 240)
        result = n.run('timeout ' + str(timeout) + ' pbm restore ' + name + ' --wait')
    
        if result.rc == 0:
            Cluster.log(result.stdout)
        else:
            # try to catch possible failures if timeout exceeded
            error=''
            for host in self.mongod_hosts:
                try:
                    container = docker.from_env().containers.get(host)
                    get_logs = container.exec_run(
                        'cat /var/lib/mongo/pbm.restore.log', stderr=False)
                    if get_logs.exit_code == 0:
                        Cluster.log(
                            "!!!!Possible failure on {}, file pbm.restore.log was found:".format(host))
                        logs = get_logs.output.decode('utf-8')
                        Cluster.log(logs)
                        if '"s":"F"' in logs:
                            error = logs
                except docker.errors.APIError:
                    pass
            if error:
>               assert False, result.stdout + result.stderr + "\n" + error
E               AssertionError: Starting restore 2024-11-06T14:53:08.930922178Z to point-in-time 2024-11-06T14:51:57 from '2024-11-06T14:48:16Z'..........................Started physical restore.
E               Waiting to finish.........Error: operation failed with: replay pitr oplog: reply oplog: replay chunk 1730904499.1730904562: apply oplog for chunk: applying an entry: op: {"Timestamp":{"T":1730904533,"I":16},"Term":1,"Hash":null,"Version":2,"Operation":"i","Namespace":"test.test","Object":[{"Key":"_id","Value":"672b81d5c1a5ff9f1b4b2ef9"},{"Key":"doc","Value":0}],"Query":null,"UI":{"Subtype":4,"Data":"ju3DVLYqQV2c/njZQjRlBQ=="},"LSID":"SAAAAAVpZAAQAAAABFfyUVAR30DhkkIJwKE1+7EFdWlkACAAAAAAY5mrDaxi8gv8RmdTsQ+1j7fmkr7JUsabhNmXAheU0fgA","TxnNumber":1,"PrevOpTime":"HAAAABF0cwAAAAAAAAAAABJ0AP//////////AA==","MultiOpType":null} | merr <nil>: applyOps: (NotWritablePrimary) not primary
E               
E               {"t":{"$date":"2024-11-06T14:53:42.913+00:00"},"s":"I",  "c":"NETWORK",  "id":4915701, "ctx":"main","msg":"Initialized wire specification","attr":{"spec":{"incomingExternalClient":{"minWireVersion":0,"maxWireVersion":13},"incomingInternalClient":{"minWireVersion":0,"maxWireVersion":13},"outgoing":{"minWireVersion":0,"maxWireVersion":13},"isInternalClient":true}}}
E               {"t":{"$date":"2024-11-06T14:53:42.916+00:00"},"s":"I",  "c":"CONTROL",  "id":23285,   "ctx":"main","msg":"Automatically disabling TLS 1.0, to force-enable TLS 1.0 specify --sslDisabledProtocols 'none'"}
E               {"t":{"$date":"2024-11-06T14:53:42.916+00:00"},"s":"W",  "c":"ASIO",     "id":22601,   "ctx":"main","msg":"No TransportLayer configured during NetworkInterface startup"}
E               {"t":{"$date":"2024-11-06T14:53:42.916+00:00"},"s":"I",  "c":"NETWORK",  "id":4648601, "ctx":"main","msg":"Implicit TCP FastOpen unavailable. If TCP FastOpen is required, set tcpFastOpenServer, tcpFastOpenClient, and tcpFastOpenQueueSize."}
E               {"t":{"$date":"2024-11-06T14:53:42.917+00:00"},"s":"W",  "c":"ASIO",     "id":22601,   "ctx":"main","msg":"No TransportLayer configured during NetworkInterface startup"}
E               {"t":{"$date":"2024-11-06T14:53:42.917+00:00"},"s":"W",  "c":"ASIO",     "id":22601,   "ctx":"main","msg":"No TransportLayer configured during NetworkInterface startup"}
E               {"t":{"$date":"2024-11-06T14:53:42.917+00:00"},"s":"I",  "c":"REPL",     "id":5123008, "ctx":"main","msg":"Successfully registered PrimaryOnlyService","attr":{"service":"TenantMigrationDonorService","ns":"config.tenantMigrationDonors"}}
E               {"t":{"$date":"2024-11-06T14:53:42.917+00:00"},"s":"I",  "c":"REPL",     "id":5123008, "ctx":"main","msg":"Successfully registered PrimaryOnlyService","attr":{"service":"TenantMigrationRecipientService","ns":"config.tenantMigrationRecipients"}}
E               {"t":{"$date":"2024-11-06T14:53:42.917+00:00"},"s":"I",  "c":"CONTROL",  "id":5945603, "ctx":"main","msg":"Multi threading initialized"}
E               {"t":{"$date":"2024-11-06T14:53:42.917+00:00"},"s":"I",  "c":"CONTROL",  "id":4615611, "ctx":"initandlisten","msg":"MongoDB starting","attr":{"pid":512,"port":28064,"dbPath":"/var/lib/mongo","architecture":"64-bit","host":"rs101"}}
E               {"t":{"$date":"2024-11-06T14:53:42.917+00:00"},"s":"I",  "c":"CONTROL",  "id":23403,   "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"5.0.29-25","gitVersion":"6ed5ea3065f1daafa7661d1c05d20c50b614a96a","openSSLVersion":"OpenSSL 1.1.1k  FIPS 25 Mar 2021","modules":[],"allocator":"tcmalloc","environment":{"distarch":"x86_64","target_arch":"x86_64"}}}}
E               {"t":{"$date":"2024-11-06T14:53:42.917+00:00"},"s":"I",  "c":"CONTROL",  "id":51765,   "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"Oracle Linux Server release 8.10","version":"Kernel 6.5.0-1025-azure"}}}
E               {"t":{"$date":"2024-11-06T14:53:42.917+00:00"},"s":"I",  "c":"CONTROL",  "id":21951,   "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"config":"/tmp/pbmMongdTmpConf2797833918","net":{"bindIp":"localhost","port":28064},"replication":{"replSet":"rs1"},"setParameter":{"disableLogicalSessionCacheRefresh":"true","takeUnstableCheckpointOnShutdown":"true"},"storage":{"dbPath":"/var/lib/mongo","directoryPerDB":false,"wiredTiger":{"collectionConfig":{"blockCompressor":"snappy"},"engineConfig":{"directoryForIndexes":false,"journalCompressor":"snappy"},"indexConfig":{"prefixCompression":true}}},"systemLog":{"destination":"file","path":"/var/lib/mongo/pbm.restore.log"}}}}
E               {"t":{"$date":"2024-11-06T14:53:42.918+00:00"},"s":"I",  "c":"STORAGE",  "id":22270,   "ctx":"initandlisten","msg":"Storage engine to use detected by data files","attr":{"dbpath":"/var/lib/mongo","storageEngine":"wiredTiger"}}
E               {"t":{"$date":"2024-11-06T14:53:42.918+00:00"},"s":"I",  "c":"STORAGE",  "id":22315,   "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=7478M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=2000),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress,compact_progress],"}}
E               {"t":{"$date":"2024-11-06T14:53:43.136+00:00"},"s":"I",  "c":"STORAGE",  "id":22430,   "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1730904823:136060][512:0x7f68e2b67c00], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 5 through 6"}}
E               {"t":{"$date":"2024-11-06T14:53:43.163+00:00"},"s":"I",  "c":"STORAGE",  "id":22430,   "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1730904823:163391][512:0x7f68e2b67c00], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 6 through 6"}}
E               {"t":{"$date":"2024-11-06T14:53:43.218+00:00"},"s":"I",  "c":"STORAGE",  "id":22430,   "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1730904823:218269][512:0x7f68e2b67c00], txn-recover: [WT_VERB_RECOVERY_ALL] Main recovery loop: starting at 5/47232 to 6/256"}}
E               {"t":{"$date":"2024-11-06T14:53:43.265+00:00"},"s":"I",  "c":"STORAGE",  "id":22430,   "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1730904823:265353][512:0x7f68e2b67c00], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 5 through 6"}}
E               {"t":{"$date":"2024-11-06T14:53:43.302+00:00"},"s":"I",  "c":"STORAGE",  "id":22430,   "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1730904823:302312][512:0x7f68e2b67c00], txn-recover: [WT_VERB_RECOVERY_PROGRESS] Recovering log 6 through 6"}}
E               {"t":{"$date":"2024-11-06T14:53:43.325+00:00"},"s":"I",  "c":"STORAGE",  "id":22430,   "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1730904823:325704][512:0x7f68e2b67c00], txn-recover: [WT_VERB_RECOVERY_PROGRESS] recovery log replay has successfully finished and ran for 189 milliseconds"}}
E               {"t":{"$date":"2024-11-06T14:53:43.325+00:00"},"s":"I",  "c":"STORAGE",  "id":22430,   "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1730904823:325773][512:0x7f68e2b67c00], txn-recover: [WT_VERB_RECOVERY_ALL] Set global recovery timestamp: (0, 0)"}}
E               {"t":{"$date":"2024-11-06T14:53:43.325+00:00"},"s":"I",  "c":"STORAGE",  "id":22430,   "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1730904823:325785][512:0x7f68e2b67c00], txn-recover: [WT_VERB_RECOVERY_ALL] Set global oldest timestamp: (0, 0)"}}
E               {"t":{"$date":"2024-11-06T14:53:43.326+00:00"},"s":"I",  "c":"STORAGE",  "id":22430,   "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1730904823:326699][512:0x7f68e2b67c00], txn-recover: [WT_VERB_RECOVERY_PROGRESS] recovery rollback to stable has successfully finished and ran for 0 milliseconds"}}
E               {"t":{"$date":"2024-11-06T14:53:43.328+00:00"},"s":"I",  "c":"STORAGE",  "id":22430,   "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1730904823:328382][512:0x7f68e2b67c00], WT_SESSION.checkpoint: [WT_VERB_CHECKPOINT_PROGRESS] saving checkpoint snapshot min: 1, snapshot max: 1 snapshot count: 0, oldest timestamp: (0, 0) , meta checkpoint timestamp: (0, 0) base write gen: 42"}}
E               {"t":{"$date":"2024-11-06T14:53:43.330+00:00"},"s":"I",  "c":"STORAGE",  "id":22430,   "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1730904823:330764][512:0x7f68e2b67c00], txn-recover: [WT_VERB_RECOVERY_PROGRESS] recovery checkpoint has successfully finished and ran for 3 milliseconds"}}
E               {"t":{"$date":"2024-11-06T14:53:43.330+00:00"},"s":"I",  "c":"STORAGE",  "id":22430,   "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1730904823:330800][512:0x7f68e2b67c00], txn-recover: [WT_VERB_RECOVERY_PROGRESS] recovery was completed successfully and took 194ms, including 189ms for the log replay, 0ms for the rollback to stable, and 3ms for the checkpoint."}}
E               {"t":{"$date":"2024-11-06T14:53:43.331+00:00"},"s":"I",  "c":"STORAGE",  "id":4795906, "ctx":"initandlisten","msg":"WiredTiger opened","attr":{"durationMillis":413}}
E               {"t":{"$date":"2024-11-06T14:53:43.331+00:00"},"s":"I",  "c":"RECOVERY", "id":23987,   "ctx":"initandlisten","msg":"WiredTiger recoveryTimestamp","attr":{"recoveryTimestamp":{"$timestamp":{"t":0,"i":0}}}}
E               {"t":{"$date":"2024-11-06T14:53:43.338+00:00"},"s":"I",  "c":"STORAGE",  "id":22383,   "ctx":"initandlisten","msg":"The size storer reports that the oplog contains","attr":{"numRecords":47,"dataSize":13138}}
E               {"t":{"$date":"2024-11-06T14:53:43.338+00:00"},"s":"I",  "c":"STORAGE",  "id":22384,   "ctx":"initandlisten","msg":"Scanning the oplog to determine where to place markers for truncation"}
E               {"t":{"$date":"2024-11-06T14:53:43.339+00:00"},"s":"I",  "c":"STORAGE",  "id":22382,   "ctx":"initandlisten","msg":"WiredTiger record store oplog processing finished","attr":{"durationMillis":0}}
E               {"t":{"$date":"2024-11-06T14:53:43.352+00:00"},"s":"I",  "c":"STORAGE",  "id":22262,   "ctx":"initandlisten","msg":"Timestamp monitor starting"}
E               {"t":{"$date":"2024-11-06T14:53:43.353+00:00"},"s":"W",  "c":"CONTROL",  "id":22120,   "ctx":"initandlisten","msg":"Access control is not enabled for the database. Read and write access to data and configuration is unrestricted. You can use percona-server-mongodb-enable-auth.sh to fix it","tags":["startupWarnings"]}
E               {"t":{"$date":"2024-11-06T14:53:43.353+00:00"},"s":"W",  "c":"CONTROL",  "id":22178,   "ctx":"initandlisten","msg":"/sys/kernel/mm/transparent_hugepage/enabled is 'always'. We suggest setting it to 'never' in this binary version","tags":["startupWarnings"]}
E               {"t":{"$date":"2024-11-06T14:53:43.360+00:00"},"s":"I",  "c":"NETWORK",  "id":4915702, "ctx":"initandlisten","msg":"Updated wire specification","attr":{"oldSpec":{"incomingExternalClient":{"minWireVersion":0,"maxWireVersion":13},"incomingInternalClient":{"minWireVersion":0,"maxWireVersion":13},"outgoing":{"minWireVersion":0,"maxWireVersion":13},"isInternalClient":true},"newSpec":{"incomingExternalClient":{"minWireVersion":0,"maxWireVersion":13},"incomingInternalClient":{"minWireVersion":13,"maxWireVersion":13},"outgoing":{"minWireVersion":13,"maxWireVersion":13},"isInternalClient":true}}}
E               {"t":{"$date":"2024-11-06T14:53:43.360+00:00"},"s":"I",  "c":"STORAGE",  "id":5071100, "ctx":"initandlisten","msg":"Clearing temp directory"}
E               {"t":{"$date":"2024-11-06T14:53:43.378+00:00"},"s":"I",  "c":"CONTROL",  "id":20536,   "ctx":"initandlisten","msg":"Flow Control is enabled on this deployment"}
E               {"t":{"$date":"2024-11-06T14:53:43.379+00:00"},"s":"W",  "c":"SHARDING", "id":22075,   "ctx":"initandlisten","msg":"Not started with --shardsvr, but a shardIdentity document was found on disk","attr":{"namespace":"admin.system.version","shardIdentityDocument":{"_id":"shardIdentity","shardName":"rs1","clusterId":{"$oid":"672b819df2d5aa210a81b4e1"},"configsvrConnectionString":"rscfg/rscfg01:27017"}}}
E               {"t":{"$date":"2024-11-06T14:53:43.379+00:00"},"s":"I",  "c":"FTDC",     "id":20625,   "ctx":"initandlisten","msg":"Initializing full-time diagnostic data capture","attr":{"dataDirectory":"/var/lib/mongo/diagnostic.data"}}
E               {"t":{"$date":"2024-11-06T14:53:43.381+00:00"},"s":"I",  "c":"-",        "id":4939300, "ctx":"monitoring-keys-for-HMAC","msg":"Failed to refresh key cache","attr":{"error":"ReadConcernMajorityNotAvailableYet: Read concern majority reads are currently not possible.","nextWakeupMillis":200}}
E               {"t":{"$date":"2024-11-06T14:53:43.381+00:00"},"s":"I",  "c":"REPL",     "id":6015317, "ctx":"initandlisten","msg":"Setting new configuration state","attr":{"newState":"ConfigStartingUp","oldState":"ConfigPreStart"}}
E               {"t":{"$date":"2024-11-06T14:53:43.382+00:00"},"s":"I",  "c":"REPL",     "id":4280500, "ctx":"initandlisten","msg":"Attempting to create internal replication collections"}
E               {"t":{"$date":"2024-11-06T14:53:43.384+00:00"},"s":"I",  "c":"REPL",     "id":4280501, "ctx":"initandlisten","msg":"Attempting to load local voted for document"}
E               {"t":{"$date":"2024-11-06T14:53:43.384+00:00"},"s":"I",  "c":"REPL",     "id":21311,   "ctx":"initandlisten","msg":"Did not find local initialized voted for document at startup"}
E               {"t":{"$date":"2024-11-06T14:53:43.384+00:00"},"s":"I",  "c":"REPL",     "id":4280502, "ctx":"initandlisten","msg":"Searching for local Rollback ID document"}
E               {"t":{"$date":"2024-11-06T14:53:43.385+00:00"},"s":"I",  "c":"REPL",     "id":21529,   "ctx":"initandlisten","msg":"Initializing rollback ID","attr":{"rbid":1}}
E               {"t":{"$date":"2024-11-06T14:53:43.385+00:00"},"s":"I",  "c":"REPL",     "id":4280504, "ctx":"initandlisten","msg":"Cleaning up any partially applied oplog batches & reading last op from oplog"}
E               {"t":{"$date":"2024-11-06T14:53:43.386+00:00"},"s":"I",  "c":"REPL",     "id":21546,   "ctx":"initandlisten","msg":"Recovering from an unstable checkpoint","attr":{"topOfOplog":{"ts":{"$timestamp":{"t":1730904496,"i":44}},"t":1},"appliedThrough":{"ts":{"$timestamp":{"t":1730904496,"i":44}},"t":1}}}
E               {"t":{"$date":"2024-11-06T14:53:43.386+00:00"},"s":"I",  "c":"REPL",     "id":21548,   "ctx":"initandlisten","msg":"Starting recovery oplog application at the appliedThrough through the top of the oplog","attr":{"appliedThrough":{"ts":{"$timestamp":{"t":1730904496,"i":44}},"t":1},"topOfOplog":{"ts":{"$timestamp":{"t":1730904496,"i":44}},"t":1}}}
E               {"t":{"$date":"2024-11-06T14:53:43.386+00:00"},"s":"I",  "c":"REPL",     "id":21549,   "ctx":"initandlisten","msg":"No oplog entries to apply for recovery. Start point is at the top of the oplog"}
E               {"t":{"$date":"2024-11-06T14:53:43.387+00:00"},"s":"I",  "c":"STORAGE",  "id":22430,   "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1730904823:387789][512:0x7f68e2b67c00], WT_SESSION.checkpoint: [WT_VERB_CHECKPOINT_PROGRESS] saving checkpoint snapshot min: 37, snapshot max: 37 snapshot count: 0, oldest timestamp: (1730904496, 44) , meta checkpoint timestamp: (0, 0) base write gen: 42"}}
E               {"t":{"$date":"2024-11-06T14:53:43.392+00:00"},"s":"I",  "c":"REPL",     "id":4280505, "ctx":"initandlisten","msg":"Creating any necessary TenantMigrationAccessBlockers for unfinished migrations"}
E               {"t":{"$date":"2024-11-06T14:53:43.392+00:00"},"s":"I",  "c":"REPL",     "id":4280506, "ctx":"initandlisten","msg":"Reconstructing prepared transactions"}
E               {"t":{"$date":"2024-11-06T14:53:43.393+00:00"},"s":"I",  "c":"REPL",     "id":4280507, "ctx":"initandlisten","msg":"Loaded replica set config, scheduled callback to set local config"}
E               {"t":{"$date":"2024-11-06T14:53:43.393+00:00"},"s":"I",  "c":"REPL",     "id":4280508, "ctx":"ReplCoord-0","msg":"Attempting to set local replica set config; validating config for startup"}
E               {"t":{"$date":"2024-11-06T14:53:43.394+00:00"},"s":"I",  "c":"REPL",     "id":40440,   "ctx":"initandlisten","msg":"Starting the TopologyVersionObserver"}
E               {"t":{"$date":"2024-11-06T14:53:43.394+00:00"},"s":"I",  "c":"REPL",     "id":40445,   "ctx":"TopologyVersionObserver","msg":"Started TopologyVersionObserver"}
E               {"t":{"$date":"2024-11-06T14:53:43.394+00:00"},"s":"I",  "c":"NETWORK",  "id":23015,   "ctx":"listener","msg":"Listening on","attr":{"address":"/tmp/mongodb-28064.sock"}}
E               {"t":{"$date":"2024-11-06T14:53:43.394+00:00"},"s":"I",  "c":"NETWORK",  "id":23015,   "ctx":"listener","msg":"Listening on","attr":{"address":"127.0.0.1"}}
E               {"t":{"$date":"2024-11-06T14:53:43.394+00:00"},"s":"I",  "c":"NETWORK",  "id":23016,   "ctx":"listener","msg":"Waiting for connections","attr":{"port":28064,"ssl":"off"}}
E               {"t":{"$date":"2024-11-06T14:53:43.422+00:00"},"s":"I",  "c":"NETWORK",  "id":22943,   "ctx":"listener","msg":"Connection accepted","attr":{"remote":"127.0.0.1:53040","uuid":"341ae524-712d-46fd-b58e-5c0890f843c1","connectionId":1,"connectionCount":1}}
E               {"t":{"$date":"2024-11-06T14:53:43.422+00:00"},"s":"I",  "c":"NETWORK",  "id":22943,   "ctx":"listener","msg":"Connection accepted","attr":{"remote":"127.0.0.1:53056","uuid":"554cbd88-05a3-49b9-9d6a-67b9560b040f","connectionId":3,"connectionCount":2}}
E               {"t":{"$date":"2024-11-06T14:53:43.422+00:00"},"s":"I",  "c":"NETWORK",  "id":51800,   "ctx":"conn1","msg":"client metadata","attr":{"remote":"127.0.0.1:53040","client":"conn1","negotiatedCompressors":[],"doc":{"application":{"name":"pbm-physical-restore"},"driver":{"name":"mongo-go-driver","version":"1.16.0"},"os":{"type":"linux","architecture":"amd64"},"platform":"go1.22.8","env":{"container":{"runtime":"docker"}}}}}
E               {"t":{"$date":"2024-11-06T14:53:43.422+00:00"},"s":"W",  "c":"COMMAND",  "id":5578800, "ctx":"conn3","msg":"Deprecated operation requested. The client driver may require an upgrade in order to ensure compatibility with future server versions. For more details see https://dochub.mongodb.org/core/legacy-opcode-compatibility","attr":{"op":"query","clientInfo":{"address":"127.0.0.1:53056"}}}
E               {"t":{"$date":"2024-11-06T14:53:43.422+00:00"},"s":"I",  "c":"REPL",     "id":4280509, "ctx":"ReplCoord-0","msg":"Local configuration validated for startup"}
E               {"t":{"$date":"2024-11-06T14:53:43.422+00:00"},"s":"I",  "c":"REPL",     "id":6015317, "ctx":"ReplCoord-0","msg":"Setting new configuration state","attr":{"newState":"ConfigSteady","oldState":"ConfigStartingUp"}}
E               {"t":{"$date":"2024-11-06T14:53:43.422+00:00"},"s":"I",  "c":"NETWORK",  "id":22944,   "ctx":"conn3","msg":"Connection ended","attr":{"remote":"127.0.0.1:53056","uuid":"554cbd88-05a3-49b9-9d6a-67b9560b040f","connectionId":3,"connectionCount":1}}
E               {"t":{"$date":"2024-11-06T14:53:43.422+00:00"},"s":"I",  "c":"REPL",     "id":21392,   "ctx":"ReplCoord-0","msg":"New replica set config in use","attr":{"config":{"_id":"rs1","version":1,"members":[{"_id":0,"host":"localhost:28064","arbiterOnly":false,"buildIndexes":true,"hidden":false,"priority":1,"tags":{},"secondaryDelaySecs":0,"votes":1}],"protocolVersion":1,"writeConcernMajorityJournalDefault":true,"settings":{"chainingAllowed":true,"heartbeatIntervalMillis":2000,"heartbeatTimeoutSecs":10,"electionTimeoutMillis":10000,"catchUpTimeoutMillis":-1,"catchUpTakeoverDelayMillis":30000,"getLastErrorModes":{},"getLastErrorDefaults":{"w":1,"wtimeout":0}}}}}
E               {"t":{"$date":"2024-11-06T14:53:43.422+00:00"},"s":"I",  "c":"REPL",     "id":21393,   "ctx":"ReplCoord-0","msg":"Found self in config","attr":{"hostAndPort":"localhost:28064"}}
E               {"t":{"$date":"2024-11-06T14:53:43.422+00:00"},"s":"I",  "c":"REPL",     "id":21358,   "ctx":"ReplCoord-0","msg":"Replica set state transition","attr":{"newState":"STARTUP2","oldState":"STARTUP"}}
E               {"t":{"$date":"2024-11-06T14:53:43.423+00:00"},"s":"I",  "c":"REPL",     "id":21320,   "ctx":"ReplCoord-0","msg":"Updated term","attr":{"term":1}}
E               {"t":{"$date":"2024-11-06T14:53:43.423+00:00"},"s":"I",  "c":"REPL",     "id":21306,   "ctx":"ReplCoord-0","msg":"Starting replication storage threads"}
E               {"t":{"$date":"2024-11-06T14:53:43.423+00:00"},"s":"I",  "c":"NETWORK",  "id":22943,   "ctx":"listener","msg":"Connection accepted","attr":{"remote":"127.0.0.1:53060","uuid":"16902c67-420d-4f4b-9e3e-80a3cf95cfae","connectionId":4,"connectionCount":2}}
E               {"t":{"$date":"2024-11-06T14:53:43.423+00:00"},"s":"I",  "c":"NETWORK",  "id":22943,   "ctx":"listener","msg":"Connection accepted","attr":{"remote":"127.0.0.1:53066","uuid":"3707688b-d709-408f-b269-c25ec42a08c8","connectionId":5,"connectionCount":3}}
E               {"t":{"$date":"2024-11-06T14:53:43.423+00:00"},"s":"I",  "c":"NETWORK",  "id":51800,   "ctx":"conn4","msg":"client metadata","attr":{"remote":"127.0.0.1:53060","client":"conn4","negotiatedCompressors":[],"doc":{"application":{"name":"pbm-physical-restore"},"driver":{"name":"mongo-go-driver","version":"1.16.0"},"os":{"type":"linux","architecture":"amd64"},"platform":"go1.22.8","env":{"container":{"runtime":"docker"}}}}}
E               {"t":{"$date":"2024-11-06T14:53:43.423+00:00"},"s":"I",  "c":"REPL",     "id":4280512, "ctx":"ReplCoord-0","msg":"No initial sync required. Attempting to begin steady replication"}
E               {"t":{"$date":"2024-11-06T14:53:43.423+00:00"},"s":"I",  "c":"REPL",     "id":21358,   "ctx":"ReplCoord-0","msg":"Replica set state transition","attr":{"newState":"RECOVERING","oldState":"STARTUP2"}}
E               {"t":{"$date":"2024-11-06T14:53:43.423+00:00"},"s":"I",  "c":"STORAGE",  "id":20320,   "ctx":"ReplCoord-0","msg":"createCollection","attr":{"namespace":"local.replset.initialSyncId","uuidDisposition":"generated","uuid":{"uuid":{"$uuid":"d07873ce-14a9-4a11-b146-34beab6f0179"}},"options":{}}}
E               {"t":{"$date":"2024-11-06T14:53:43.423+00:00"},"s":"I",  "c":"NETWORK",  "id":51800,   "ctx":"conn5","msg":"client metadata","attr":{"remote":"127.0.0.1:53066","client":"conn5","negotiatedCompressors":[],"doc":{"application":{"name":"pbm-physical-restore"},"driver":{"name":"mongo-go-driver","version":"1.16.0"},"os":{"type":"linux","architecture":"amd64"},"platform":"go1.22.8","env":{"container":{"runtime":"docker"}}}}}
E               {"t":{"$date":"2024-11-06T14:53:43.430+00:00"},"s":"I",  "c":"INDEX",    "id":20345,   "ctx":"ReplCoord-0","msg":"Index build: done building","attr":{"buildUUID":null,"namespace":"local.replset.initialSyncId","index":"_id_","commitTimestamp":null}}
E               {"t":{"$date":"2024-11-06T14:53:43.430+00:00"},"s":"I",  "c":"REPL",     "id":21299,   "ctx":"ReplCoord-0","msg":"Starting replication fetcher thread"}
E               {"t":{"$date":"2024-11-06T14:53:43.430+00:00"},"s":"I",  "c":"REPL",     "id":21300,   "ctx":"ReplCoord-0","msg":"Starting replication applier thread"}
E               {"t":{"$date":"2024-11-06T14:53:43.430+00:00"},"s":"I",  "c":"REPL",     "id":21301,   "ctx":"ReplCoord-0","msg":"Starting replication reporter thread"}
E               {"t":{"$date":"2024-11-06T14:53:43.430+00:00"},"s":"I",  "c":"REPL",     "id":4280511, "ctx":"ReplCoord-0","msg":"Set local replica set config"}
E               {"t":{"$date":"2024-11-06T14:53:43.430+00:00"},"s":"I",  "c":"REPL",     "id":21224,   "ctx":"OplogApplier-0","msg":"Starting oplog application"}
E               {"t":{"$date":"2024-11-06T14:53:43.431+00:00"},"s":"I",  "c":"REPL",     "id":21358,   "ctx":"OplogApplier-0","msg":"Replica set state transition","attr":{"newState":"SECONDARY","oldState":"RECOVERING"}}
E               {"t":{"$date":"2024-11-06T14:53:43.431+00:00"},"s":"I",  "c":"ELECTION", "id":4615652, "ctx":"OplogApplier-0","msg":"Starting an election, since we've seen no PRIMARY in election timeout period","attr":{"electionTimeoutPeriodMillis":10000}}
E               {"t":{"$date":"2024-11-06T14:53:43.431+00:00"},"s":"I",  "c":"ELECTION", "id":21438,   "ctx":"OplogApplier-0","msg":"Conducting a dry run election to see if we could be elected","attr":{"currentTerm":1}}
E               {"t":{"$date":"2024-11-06T14:53:43.431+00:00"},"s":"I",  "c":"ELECTION", "id":21444,   "ctx":"ReplCoord-0","msg":"Dry election run succeeded, running for election","attr":{"newTerm":2}}
E               {"t":{"$date":"2024-11-06T14:53:43.431+00:00"},"s":"I",  "c":"ELECTION", "id":6015300, "ctx":"ReplCoord-0","msg":"Storing last vote document in local storage for my election","attr":{"lastVote":{"term":2,"candidateIndex":0}}}
E               {"t":{"$date":"2024-11-06T14:53:43.432+00:00"},"s":"I",  "c":"ELECTION", "id":21450,   "ctx":"ReplCoord-0","msg":"Election succeeded, assuming primary role","attr":{"term":2}}
E               {"t":{"$date":"2024-11-06T14:53:43.432+00:00"},"s":"I",  "c":"REPL",     "id":21358,   "ctx":"ReplCoord-0","msg":"Replica set state transition","attr":{"newState":"PRIMARY","oldState":"SECONDARY"}}
E               {"t":{"$date":"2024-11-06T14:53:43.432+00:00"},"s":"I",  "c":"REPL",     "id":21106,   "ctx":"ReplCoord-0","msg":"Resetting sync source to empty","attr":{"previousSyncSource":":27017"}}
E               {"t":{"$date":"2024-11-06T14:53:43.432+00:00"},"s":"I",  "c":"REPL",     "id":21359,   "ctx":"ReplCoord-0","msg":"Entering primary catch-up mode"}
E               {"t":{"$date":"2024-11-06T14:53:43.432+00:00"},"s":"I",  "c":"REPL",     "id":6015304, "ctx":"ReplCoord-0","msg":"Skipping primary catchup since we are the only node in the replica set."}
E               {"t":{"$date":"2024-11-06T14:53:43.432+00:00"},"s":"I",  "c":"REPL",     "id":21363,   "ctx":"ReplCoord-0","msg":"Exited primary catch-up mode"}
E               {"t":{"$date":"2024-11-06T14:53:43.432+00:00"},"s":"I",  "c":"REPL",     "id":21107,   "ctx":"ReplCoord-0","msg":"Stopping replication producer"}
E               {"t":{"$date":"2024-11-06T14:53:43.432+00:00"},"s":"I",  "c":"REPL",     "id":21239,   "ctx":"ReplBatcher","msg":"Oplog buffer has been drained","attr":{"term":2}}
E               {"t":{"$date":"2024-11-06T14:53:43.432+00:00"},"s":"I",  "c":"REPL",     "id":21343,   "ctx":"RstlKillOpThread","msg":"Starting to kill user operations"}
E               {"t":{"$date":"2024-11-06T14:53:43.432+00:00"},"s":"I",  "c":"REPL",     "id":21344,   "ctx":"RstlKillOpThread","msg":"Stopped killing user operations"}
E               {"t":{"$date":"2024-11-06T14:53:43.432+00:00"},"s":"I",  "c":"REPL",     "id":21340,   "ctx":"RstlKillOpThread","msg":"State transition ops metrics","attr":{"metrics":{"lastStateTransition":"stepUp","userOpsKilled":0,"userOpsRunning":1}}}
E               {"t":{"$date":"2024-11-06T14:53:43.432+00:00"},"s":"I",  "c":"REPL",     "id":6015310, "ctx":"OplogApplier-0","msg":"Starting to transition to primary."}
E               {"t":{"$date":"2024-11-06T14:53:43.433+00:00"},"s":"I",  "c":"REPL",     "id":6015309, "ctx":"OplogApplier-0","msg":"Logging transition to primary to oplog on stepup"}
E               {"t":{"$date":"2024-11-06T14:53:43.433+00:00"},"s":"I",  "c":"STORAGE",  "id":20657,   "ctx":"OplogApplier-0","msg":"IndexBuildsCoordinator::onStepUp - this node is stepping up to primary"}
E               {"t":{"$date":"2024-11-06T14:53:43.433+00:00"},"s":"I",  "c":"REPL",     "id":21331,   "ctx":"OplogApplier-0","msg":"Transition to primary complete; database writes are now permitted"}
E               {"t":{"$date":"2024-11-06T14:53:43.434+00:00"},"s":"I",  "c":"STORAGE",  "id":20320,   "ctx":"monitoring-keys-for-HMAC","msg":"createCollection","attr":{"namespace":"admin.system.keys","uuidDisposition":"generated","uuid":{"uuid":{"$uuid":"50ff9a95-8be6-42e3-b2f2-b65bfe37274f"}},"options":{}}}
E               {"t":{"$date":"2024-11-06T14:53:43.435+00:00"},"s":"I",  "c":"STORAGE",  "id":22310,   "ctx":"JournalFlusher","msg":"Triggering the first stable checkpoint","attr":{"initialDataTimestamp":{"$timestamp":{"t":1730904496,"i":44}},"prevStableTimestamp":{"$timestamp":{"t":0,"i":0}},"currStableTimestamp":{"$timestamp":{"t":1730904823,"i":2}}}}
E               {"t":{"$date":"2024-11-06T14:53:43.435+00:00"},"s":"I",  "c":"STORAGE",  "id":20320,   "ctx":"TenantMigrationRecipientService-0","msg":"createCollection","attr":{"namespace":"config.tenantMigrationRecipients","uuidDisposition":"generated","uuid":{"uuid":{"$uuid":"f389babb-b8a5-4fb5-912a-08c096d294f1"}},"options":{}}}
E               {"t":{"$date":"2024-11-06T14:53:43.435+00:00"},"s":"I",  "c":"STORAGE",  "id":20320,   "ctx":"TenantMigrationDonorService-0","msg":"createCollection","attr":{"namespace":"config.tenantMigrationDonors","uuidDisposition":"generated","uuid":{"uuid":{"$uuid":"d23f1336-046b-41c2-a3af-a30796ea4c28"}},"options":{}}}
E               {"t":{"$date":"2024-11-06T14:53:43.438+00:00"},"s":"E",  "c":"STORAGE",  "id":22435,   "ctx":"Checkpointer","msg":"WiredTiger error","attr":{"error":2,"message":"[1730904823:438424][512:0x7f68caa72700], file:collection-2--3772753621548582318.wt, WT_SESSION.checkpoint: __posix_fs_size, 296: /var/lib/mongo/WiredTigerHS.wt: file-size: stat: No such file or directory"}}
E               {"t":{"$date":"2024-11-06T14:53:43.438+00:00"},"s":"E",  "c":"STORAGE",  "id":22435,   "ctx":"Checkpointer","msg":"WiredTiger error","attr":{"error":2,"message":"[1730904823:438471][512:0x7f68caa72700], file:collection-2--3772753621548582318.wt, WT_SESSION.checkpoint: __reconcile, 351: reconciliation failed after building the disk image: No such file or directory"}}
E               {"t":{"$date":"2024-11-06T14:53:43.438+00:00"},"s":"E",  "c":"STORAGE",  "id":22435,   "ctx":"Checkpointer","msg":"WiredTiger error","attr":{"error":-31804,"message":"[1730904823:438490][512:0x7f68caa72700], file:collection-2--3772753621548582318.wt, WT_SESSION.checkpoint: __reconcile, 351: the process must exit and restart: WT_PANIC: WiredTiger library panic"}}
E               {"t":{"$date":"2024-11-06T14:53:43.438+00:00"},"s":"F",  "c":"-",        "id":23089,   "ctx":"Checkpointer","msg":"Fatal assertion","attr":{"msgid":50853,"file":"src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp","line":574}}
E               {"t":{"$date":"2024-11-06T14:53:43.438+00:00"},"s":"F",  "c":"-",        "id":23090,   "ctx":"Checkpointer","msg":"\n\n***aborting after fassert() failure\n\n"}
E               {"t":{"$date":"2024-11-06T14:53:43.438+00:00"},"s":"F",  "c":"CONTROL",  "id":6384300, "ctx":"Checkpointer","msg":"Writing fatal message","attr":{"message":"Got signal: 6 (Aborted).\n"}}
E               {"t":{"$date":"2024-11-06T14:53:43.443+00:00"},"s":"I",  "c":"NETWORK",  "id":22944,   "ctx":"conn5","msg":"Connection ended","attr":{"remote":"127.0.0.1:53066","uuid":"3707688b-d709-408f-b269-c25ec42a08c8","connectionId":5,"connectionCount":1}}
E               {"t":{"$date":"2024-11-06T14:53:43.443+00:00"},"s":"I",  "c":"NETWORK",  "id":22944,   "ctx":"conn4","msg":"Connection ended","attr":{"remote":"127.0.0.1:53060","uuid":"16902c67-420d-4f4b-9e3e-80a3cf95cfae","connectionId":4,"connectionCount":2}}
E               {"t":{"$date":"2024-11-06T14:53:43.444+00:00"},"s":"I",  "c":"-",        "id":20883,   "ctx":"conn1","msg":"Interrupted operation as its client disconnected","attr":{"opId":39}}
E               {"t":{"$date":"2024-11-06T14:53:43.445+00:00"},"s":"I",  "c":"NETWORK",  "id":22944,   "ctx":"conn1","msg":"Connection ended","attr":{"remote":"127.0.0.1:53040","uuid":"341ae524-712d-46fd-b58e-5c0890f843c1","connectionId":1,"connectionCount":0}}
E               {"t":{"$date":"2024-11-06T14:53:43.451+00:00"},"s":"I",  "c":"INDEX",    "id":20345,   "ctx":"monitoring-keys-for-HMAC","msg":"Index build: done building","attr":{"buildUUID":null,"namespace":"admin.system.keys","index":"_id_","commitTimestamp":{"$timestamp":{"t":1730904823,"i":3}}}}
E               {"t":{"$date":"2024-11-06T14:53:43.453+00:00"},"s":"I",  "c":"INDEX",    "id":20345,   "ctx":"TenantMigrationDonorService-0","msg":"Index build: done building","attr":{"buildUUID":null,"namespace":"config.tenantMigrationDonors","index":"_id_","commitTimestamp":{"$timestamp":{"t":1730904823,"i":6}}}}
E               {"t":{"$date":"2024-11-06T14:53:43.453+00:00"},"s":"I",  "c":"INDEX",    "id":20345,   "ctx":"TenantMigrationDonorService-0","msg":"Index build: done building","attr":{"buildUUID":null,"namespace":"config.tenantMigrationDonors","index":"TenantMigrationDonorTTLIndex","commitTimestamp":{"$timestamp":{"t":1730904823,"i":6}}}}
E               {"t":{"$date":"2024-11-06T14:53:43.454+00:00"},"s":"I",  "c":"STORAGE",  "id":20320,   "ctx":"TenantMigrationDonorService-1","msg":"createCollection","attr":{"namespace":"config.external_validation_keys","uuidDisposition":"generated","uuid":{"uuid":{"$uuid":"092025f6-5f9c-40b8-8119-209ee0336fa4"}},"options":{}}}
E               {"t":{"$date":"2024-11-06T14:53:43.459+00:00"},"s":"I",  "c":"INDEX",    "id":20345,   "ctx":"TenantMigrationRecipientService-0","msg":"Index build: done building","attr":{"buildUUID":null,"namespace":"config.tenantMigrationRecipients","index":"_id_","commitTimestamp":{"$timestamp":{"t":1730904823,"i":7}}}}
E               {"t":{"$date":"2024-11-06T14:53:43.459+00:00"},"s":"I",  "c":"INDEX",    "id":20345,   "ctx":"TenantMigrationRecipientService-0","msg":"Index build: done building","attr":{"buildUUID":null,"namespace":"config.tenantMigrationRecipients","index":"TenantMigrationRecipientTTLIndex","commitTimestamp":{"$timestamp":{"t":1730904823,"i":7}}}}
E               {"t":{"$date":"2024-11-06T14:53:43.459+00:00"},"s":"I",  "c":"REPL",     "id":5123005, "ctx":"TenantMigrationRecipientService-0","msg":"Rebuilding PrimaryOnlyService due to stepUp","attr":{"service":"TenantMigrationRecipientService"}}
E               {"t":{"$date":"2024-11-06T14:53:43.468+00:00"},"s":"I",  "c":"INDEX",    "id":20345,   "ctx":"TenantMigrationDonorService-1","msg":"Index build: done building","attr":{"buildUUID":null,"namespace":"config.external_validation_keys","index":"_id_","commitTimestamp":{"$timestamp":{"t":1730904823,"i":11}}}}
E               {"t":{"$date":"2024-11-06T14:53:43.468+00:00"},"s":"I",  "c":"INDEX",    "id":20345,   "ctx":"TenantMigrationDonorService-1","msg":"Index build: done building","attr":{"buildUUID":null,"namespace":"config.external_validation_keys","index":"ExternalKeysTTLIndex","commitTimestamp":{"$timestamp":{"t":1730904823,"i":11}}}}
E               {"t":{"$date":"2024-11-06T14:53:43.468+00:00"},"s":"I",  "c":"REPL",     "id":5123005, "ctx":"TenantMigrationDonorService-1","msg":"Rebuilding PrimaryOnlyService due to stepUp","attr":{"service":"TenantMigrationDonorService"}}
E               {"t":{"$date":"2024-11-06T14:53:43.494+00:00"},"s":"I",  "c":"CONTROL",  "id":31380,   "ctx":"Checkpointer","msg":"BACKTRACE","attr":{"bt":{"backtrace":[{"a":"55A27229F9C3","b":"55A26E082000","o":"421D9C3","s":"_ZN5mongo34StackTraceAddressMetadataGenerator4loadEPv","s+":"FC3"},{"a":"55A2722A2417","b":"55A26E082000","o":"4220417","s":"_ZN5mongo15printStackTraceEv","s+":"37"},{"a":"55A27229A5B6","b":"55A26E082000","o":"42185B6","s":"abruptQuit","s+":"66"},{"a":"7F68E02E5D70","b":"7F68E02D3000","o":"12D70","s":"funlockfile","s+":"50"},{"a":"7F68DFF4B5EF","b":"7F68DFEFD000","o":"4E5EF","s":"gsignal","s+":"10F"},{"a":"7F68DFF1EE65","b":"7F68DFEFD000","o":"21E65","s":"abort","s+":"127"},{"a":"55A26F600D6D","b":"55A26E082000","o":"157ED6D","s":"_ZN5mongo25fassertFailedWithLocationEiPKcj","s+":"F6"},{"a":"55A26F0BCCF4","b":"55A26E082000","o":"103ACF4","s":"_ZZN5mongo8RecordIdC4EPKciENKUlvE0_clEv","s+":"30B8"},{"a":"55A26F9659F3","b":"55A26E082000","o":"18E39F3","s":"__wt_event_handler_set","s+":"4A3"},{"a":"55A26F0CF3E3","b":"55A26E082000","o":"104D3E3","s":"__wt_panic_func","s+":"114"},{"a":"55A26F0CBA8D","b":"55A26E082000","o":"1049A8D","s":"__wt_ext_map_windows_error","s+":"25CA"},{"a":"55A26F9FADFA","b":"55A26E082000","o":"1978DFA","s":"__wt_sync_file","s+":"6AA"},{"a":"55A26F98413B","b":"55A26E082000","o":"190213B","s":"__wt_checkpoint_tree_reconcile_update","s+":"1CB"},{"a":"55A26F98523F","b":"55A26E082000","o":"190323F","s":"__wt_checkpoint_sync","s+":"99F"},{"a":"55A26F986B76","b":"55A26E082000","o":"1904B76","s":"__wt_txn_checkpoint","s+":"246"},{"a":"55A26F0CE6EF","b":"55A26E082000","o":"104C6EF","s":"__wt_session_notsup","s+":"FCC"},{"a":"55A26F86B586","b":"55A26E082000","o":"17E9586","s":"_ZN5mongo18WiredTigerKVEngine11_checkpointEP12__wt_sessionb","s+":"46"},{"a":"55A26F878988","b":"55A26E082000","o":"17F6988","s":"_ZN5mongo18WiredTigerKVEngine11_checkpointEP12__wt_session","s+":"2B8"},{"a":"55A26F879050","b":"55A26E082000","o":"17F7050","s":"_ZN5mongo18WiredTigerKVEngine10checkpointEv","s+":"40"},{"a":"55A27081E459","b":"55A26E082000","o":"279C459","s":"_ZN5mongo12Checkpointer3runEv","s+":"E59"},{"a":"55A27205AE32","b":"55A26E082000","o":"3FD8E32","s":"_ZN5mongo13BackgroundJob7jobBodyEv","s+":"E2"},{"a":"55A27205B54C","b":"55A26E082000","o":"3FD954C","s":"_ZN5mongo13BackgroundJob7jobBodyEv","s+":"7FC"},{"a":"7F68E0B50B23","b":"7F68E0A8E000","o":"C2B23","s":"_ZNKSt10error_code23default_error_conditionEv","s+":"33"},{"a":"7F68E02DB1DA","b":"7F68E02D3000","o":"81DA","s":"start_thread","s+":"EA"},{"a":"7F68DFF368D3","b":"7F68DFEFD000","o":"398D3","s":"clone","s+":"43"}],"processInfo":{"mongodbVersion":"5.0.29-25","gitVersion":"6ed5ea3065f1daafa7661d1c05d20c50b614a96a","compiledModules":[],"uname":{"sysname":"Linux","release":"6.5.0-1025-azure","version":"#26~22.04.1-Ubuntu SMP Thu Jul 11 22:33:04 UTC 2024","machine":"x86_64"},"somap":[{"b":"55A26E082000","elfType":3,"buildId":"539E2C825673723B04D19874D3C9DF6EB9A69E4F"},{"b":"7F68E0A8E000","path":"/lib64/libstdc++.so.6","elfType":3,"buildId":"41809D0B18272BC525772714DA54108BF1C59253"},{"b":"7F68E02D3000","path":"/lib64/libpthread.so.0","elfType":3,"buildId":"E144DC0B1B2E7DA6C5B2191AC90367D60923235E"},{"b":"7F68DFEFD000","path":"/lib64/libc.so.6","elfType":3,"buildId":"A3904D14B666122259AB466667C8100B8295FE16"}]}}}}
E               {"t":{"$date":"2024-11-06T14:53:43.494+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"Checkpointer","msg":"Frame","attr":{"frame":{"a":"55A27229F9C3","b":"55A26E082000","o":"421D9C3","s":"_ZN5mongo34StackTraceAddressMetadataGenerator4loadEPv","s+":"FC3"}}}
E               {"t":{"$date":"2024-11-06T14:53:43.494+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"Checkpointer","msg":"Frame","attr":{"frame":{"a":"55A2722A2417","b":"55A26E082000","o":"4220417","s":"_ZN5mongo15printStackTraceEv","s+":"37"}}}
E               {"t":{"$date":"2024-11-06T14:53:43.494+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"Checkpointer","msg":"Frame","attr":{"frame":{"a":"55A27229A5B6","b":"55A26E082000","o":"42185B6","s":"abruptQuit","s+":"66"}}}
E               {"t":{"$date":"2024-11-06T14:53:43.494+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"Checkpointer","msg":"Frame","attr":{"frame":{"a":"7F68E02E5D70","b":"7F68E02D3000","o":"12D70","s":"funlockfile","s+":"50"}}}
E               {"t":{"$date":"2024-11-06T14:53:43.494+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"Checkpointer","msg":"Frame","attr":{"frame":{"a":"7F68DFF4B5EF","b":"7F68DFEFD000","o":"4E5EF","s":"gsignal","s+":"10F"}}}
E               {"t":{"$date":"2024-11-06T14:53:43.494+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"Checkpointer","msg":"Frame","attr":{"frame":{"a":"7F68DFF1EE65","b":"7F68DFEFD000","o":"21E65","s":"abort","s+":"127"}}}
E               {"t":{"$date":"2024-11-06T14:53:43.494+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"Checkpointer","msg":"Frame","attr":{"frame":{"a":"55A26F600D6D","b":"55A26E082000","o":"157ED6D","s":"_ZN5mongo25fassertFailedWithLocationEiPKcj","s+":"F6"}}}
E               {"t":{"$date":"2024-11-06T14:53:43.494+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"Checkpointer","msg":"Frame","attr":{"frame":{"a":"55A26F0BCCF4","b":"55A26E082000","o":"103ACF4","s":"_ZZN5mongo8RecordIdC4EPKciENKUlvE0_clEv","s+":"30B8"}}}
E               {"t":{"$date":"2024-11-06T14:53:43.494+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"Checkpointer","msg":"Frame","attr":{"frame":{"a":"55A26F9659F3","b":"55A26E082000","o":"18E39F3","s":"__wt_event_handler_set","s+":"4A3"}}}
E               {"t":{"$date":"2024-11-06T14:53:43.494+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"Checkpointer","msg":"Frame","attr":{"frame":{"a":"55A26F0CF3E3","b":"55A26E082000","o":"104D3E3","s":"__wt_panic_func","s+":"114"}}}
E               {"t":{"$date":"2024-11-06T14:53:43.494+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"Checkpointer","msg":"Frame","attr":{"frame":{"a":"55A26F0CBA8D","b":"55A26E082000","o":"1049A8D","s":"__wt_ext_map_windows_error","s+":"25CA"}}}
E               {"t":{"$date":"2024-11-06T14:53:43.494+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"Checkpointer","msg":"Frame","attr":{"frame":{"a":"55A26F9FADFA","b":"55A26E082000","o":"1978DFA","s":"__wt_sync_file","s+":"6AA"}}}
E               {"t":{"$date":"2024-11-06T14:53:43.494+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"Checkpointer","msg":"Frame","attr":{"frame":{"a":"55A26F98413B","b":"55A26E082000","o":"190213B","s":"__wt_checkpoint_tree_reconcile_update","s+":"1CB"}}}
E               {"t":{"$date":"2024-11-06T14:53:43.494+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"Checkpointer","msg":"Frame","attr":{"frame":{"a":"55A26F98523F","b":"55A26E082000","o":"190323F","s":"__wt_checkpoint_sync","s+":"99F"}}}
E               {"t":{"$date":"2024-11-06T14:53:43.494+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"Checkpointer","msg":"Frame","attr":{"frame":{"a":"55A26F986B76","b":"55A26E082000","o":"1904B76","s":"__wt_txn_checkpoint","s+":"246"}}}
E               {"t":{"$date":"2024-11-06T14:53:43.494+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"Checkpointer","msg":"Frame","attr":{"frame":{"a":"55A26F0CE6EF","b":"55A26E082000","o":"104C6EF","s":"__wt_session_notsup","s+":"FCC"}}}
E               {"t":{"$date":"2024-11-06T14:53:43.494+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"Checkpointer","msg":"Frame","attr":{"frame":{"a":"55A26F86B586","b":"55A26E082000","o":"17E9586","s":"_ZN5mongo18WiredTigerKVEngine11_checkpointEP12__wt_sessionb","s+":"46"}}}
E               {"t":{"$date":"2024-11-06T14:53:43.494+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"Checkpointer","msg":"Frame","attr":{"frame":{"a":"55A26F878988","b":"55A26E082000","o":"17F6988","s":"_ZN5mongo18WiredTigerKVEngine11_checkpointEP12__wt_session","s+":"2B8"}}}
E               {"t":{"$date":"2024-11-06T14:53:43.494+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"Checkpointer","msg":"Frame","attr":{"frame":{"a":"55A26F879050","b":"55A26E082000","o":"17F7050","s":"_ZN5mongo18WiredTigerKVEngine10checkpointEv","s+":"40"}}}
E               {"t":{"$date":"2024-11-06T14:53:43.494+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"Checkpointer","msg":"Frame","attr":{"frame":{"a":"55A27081E459","b":"55A26E082000","o":"279C459","s":"_ZN5mongo12Checkpointer3runEv","s+":"E59"}}}
E               {"t":{"$date":"2024-11-06T14:53:43.494+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"Checkpointer","msg":"Frame","attr":{"frame":{"a":"55A27205AE32","b":"55A26E082000","o":"3FD8E32","s":"_ZN5mongo13BackgroundJob7jobBodyEv","s+":"E2"}}}
E               {"t":{"$date":"2024-11-06T14:53:43.494+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"Checkpointer","msg":"Frame","attr":{"frame":{"a":"55A27205B54C","b":"55A26E082000","o":"3FD954C","s":"_ZN5mongo13BackgroundJob7jobBodyEv","s+":"7FC"}}}
E               {"t":{"$date":"2024-11-06T14:53:43.494+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"Checkpointer","msg":"Frame","attr":{"frame":{"a":"7F68E0B50B23","b":"7F68E0A8E000","o":"C2B23","s":"_ZNKSt10error_code23default_error_conditionEv","s+":"33"}}}
E               {"t":{"$date":"2024-11-06T14:53:43.494+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"Checkpointer","msg":"Frame","attr":{"frame":{"a":"7F68E02DB1DA","b":"7F68E02D3000","o":"81DA","s":"start_thread","s+":"EA"}}}
E               {"t":{"$date":"2024-11-06T14:53:43.494+00:00"},"s":"I",  "c":"CONTROL",  "id":31445,   "ctx":"Checkpointer","msg":"Frame","attr":{"frame":{"a":"7F68DFF368D3","b":"7F68DFEFD000","o":"398D3","s":"clone","s+":"43"}}}

cluster.py:462: AssertionError

start_cluster = True, cluster = <cluster.Cluster object at 0x7fe030d36810>
backup_type = 'physical', base_snapshot = 'base'

    @pytest.mark.timeout(900,func_only=True)
    @pytest.mark.parametrize('backup_type',['logical','physical'])
    @pytest.mark.parametrize('base_snapshot',['base','profile'])
    def test_pitr_PBM_T256(start_cluster,cluster,backup_type,base_snapshot):
        def insert_docs():
            client=pymongo.MongoClient(cluster.connection)
            for i in range(1500):
                client['test']['test'].insert_one({"doc":i})
                time.sleep(0.1)
    
        cluster.check_pbm_status()
        base_backup=cluster.make_backup(backup_type)
        cluster.enable_pitr(pitr_extra_args="--set pitr.oplogSpanMin=0.5")
        Cluster.log("Start inserting docs in the background")
        background_insert = threading.Thread(target=insert_docs)
        background_insert.start()
        time.sleep(60)
        Cluster.log("Check if PITR is running")
        if not cluster.check_pitr():
            logs=cluster.exec_pbm_cli("logs -sD -t0")
            assert False, logs.stdout
        Cluster.log("Setup filesystem profile")
        profile=cluster.exec_pbm_cli("profile add filesystem /etc/pbm-fs-profile.conf --wait")
        assert profile.rc==0, profile.stderr
        assert "OK" in profile.stdout, profile.stdout
        Cluster.log(profile.stdout)
        Cluster.log("Perform backup on fs profile")
        fs_backup=cluster.make_backup(backup_type + " --profile filesystem")
        Cluster.log("Check if the backup actually was created on the filesystem storage")
        assert os.path.isdir("/backups/" + fs_backup)
        assert os.path.isfile("/backups/" + fs_backup + ".pbm.json")
        time.sleep(60)
        background_insert.join()
        time.sleep(30)
        assert pymongo.MongoClient(cluster.connection)["test"]["test"].count_documents({}) == 1500
        pitr = datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S")
        Cluster.log("Time for PITR is " + pitr)
        time.sleep(60)
        cluster.disable_pitr()
        pymongo.MongoClient(cluster.connection).drop_database('test')
        if base_snapshot == 'base':
            backup="--time=" + pitr + " --base-snapshot=" + base_backup
        else:
            backup="--time=" + pitr + " --base-snapshot=" + fs_backup
        Cluster.log("Attempt to PITR restore with the backup on the " + base_snapshot + " storage")
        if backup_type == 'logical':
            cluster.make_restore(backup, check_pbm_status=True)
        else:
            cluster.make_restore(backup, restart_cluster=True, check_pbm_status=True)
        time.sleep(60)
>       assert pymongo.MongoClient(cluster.connection)["test"]["test"].count_documents({}) == 1500
E       AssertionError: assert 734 == 1500
E        +  where 734 = count_documents({})
E        +    where count_documents = Collection(Database(MongoClient(host=['mongos:27017'], document_class=dict, tz_aware=False, connect=True), 'test'), 'test').count_documents

test_PBM-1228.py:95: AssertionError