Description
Hi
one of our sharding part went down
Now the logs say:duplicate key error collection mongodb
and this is some error before this:
I COMMAND [conn381] command config command: dbStats { dbStats: 1, $clusterTime: { clusterTime: Timestamp(1584280395, 751), signature: { hash: BinData(0, ), keyId: 74634 } }, $configServerState: { opTime: { ts: Timestamp(1584280395, 4), t: 19 } }, $db: "config" } numYields:0 reslen:472 locks:{ Global: { acquireCount: { r: 2 } }, Database: { acquireCount: { R: 1 } } } protocol:op_msg 1ms
2020-03-:23:15.161+0330 F - [sessionCatalogMigration-rs8_rs9_5e6e323ce159e8a61] Invariant failure oplogEntry.getWallClockTime() src/mongo/db/s/session_catalog_migration_destination.cpp 246
2020-03-:23:15.161+0330 F - [sessionCatalogMigration-rs8_rs9_5e6e323ce159e8a61]
***aborting after invariant() failure
----- BEGIN BACKTRACE -----
{"backtrace":[{"b":"55856C0FF000","o":"21F9B51","s":"_ZN5mongo15printStackTraceERSo"},{"b":"55856C0FF000","o":"21F8D69"},{"b":"55856C0FF000","o":"21F924D"},{"b":"7F981BF30000","o":"F5D0"},{"b":"7F981BB63000","o":"36207","s":"gsignal"},{"b":"7F981BB63000","o":"378F8","s":"abort"},{"b":"55856C0FF000","o":"98B224","s":"_ZN5mongo17invariantOKFailedEPKcRKNS_6StatusES1_j"},{"b":"55856C0FF000","o":"1051128","s":"_ZN5mongo34SessionCatalogMigrationDestination31_retrieveSessionStateFromSourceEPNS_14ServiceContextE"},{"b":"55856C0FF000","o":"2307530"},{"b":"7F981BF30000","o":"7DD5"},{"b":"7F981BB63000","o":"FDEAD","s":"clone"}],"processInfo":{ "mongodbVersion" : "3.6.0", "gitVersion" : "a57d8e71e6998a2d0afde7edc11bd23e5661c915", "compiledModules" : [], "uname" : { "sysname" : "Linux", "release" : "3.10.0-957.12.2.el7.x86_64", "version" : "#1 SMP Tue May 14 21:24:32 UTC 2019", "machine" : "x86_64" }, "somap" : [ { "b" : "55856C0FF000", "elfType" : 3, "buildId" : "DE8018F36C6B9F029DEF9254662EDB5AF914ED1A" }, { "b" : "7FFE0EFF7000", "elfType" : 3, "buildId" : "D149FABD4DF969958863221F946E26C130432B47" }, { "b" : "7F981D144000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "C444AE61E7CBB716FD9C18A0B46A7FE8F4FCF3E5" }, { "b" : "7F981CED2000", "path" : "/lib64/libssl.so.10", "elfType" : 3, "buildId" : "AEF5E6F2240B55F90E9DF76CFBB8B9D9F5286583" }, { "b" : "7F981CA70000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "3593FA778645A59EA272DBBB59D318C60940E792" }, { "b" : "7F981C868000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "EFDE2029C9A4A20BE5B8D8AE7E6551FF9B5755D2" }, { "b" : "7F981C664000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "357693C8F1F49D93010C4E31529C07CDD2BD3D08" }, { "b" : "7F981C362000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "5B14BE4D749631673523A61074C10959D50F5455" }, { "b" : "7F981C14C000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "179F202998E429AA1215907F6D4C5C1BB9C90136" }, { "b" : "7F981BF30000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "96900CB0FF25B26F2BBDF247DE1408242E4773D8" }, { "b" : "7F981BB63000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "EB9F22A3891E5FD3494DFD9ED199E20AE71BB08D" }, { "b" : "7F981D35D000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "A527FE72908703C5972AE384E78D1850D1881EE7" }, { "b" : "7F981B916000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "BCC30853830CD911E58700591830DF51ABCBD7BA" }, { "b" : "7F981B62D000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "45BAB0BB455BDFA960FDA22E4124CF17B67CC930" }, { "b" : "7F981B429000", "path" : "/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "B4BE1023D9606A88169DF411BF94AF417D7BA1A0" }, { "b" : "7F981B1F6000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "A9B3906192687CC45D483AE3C58C8AF745A6726A" }, { "b" : "7F981AFE0000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "B9D5F73428BD6AD68C96986B57BEA3B7CEDB9745" }, { "b" : "7F981ADD0000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "94B3BCB669126166B77CDCE6092679A6AA2004C8" }, { "b" : "7F981ABCC000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "2E01D5AC08C1280D013AAB96B292AC58BC30A263" }, { "b" : "7F981A9A5000", "path" : "/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "D2DD4DA3FDE1477D25BFFF80F3A25FDB541A8179" }, { "b" : "7F981A743000", "path" : "/lib64/libpcre.so.1", "elfType" : 3, "buildId" : "9CA3D11F018BEEB719CDB34BE800BF1641350D0A" } ] }}
mongod(_ZN5mongo15printStackTraceERSo+0x41) [0x55856e2f8b51]
mongod(+0x21F8D69) [0x55856e2f7d69]
mongod(+0x21F924D) [0x55856e2f824d]
libpthread.so.0(+0xF5D0) [0x7f981bf3f5d0]
libc.so.6(gsignal+0x37) [0x7f981bb99207]
libc.so.6(abort+0x148) [0x7f981bb9a8f8]
mongod(_ZN5mongo17invariantOKFailedEPKcRKNS_6StatusES1_j+0x0) [0x55856ca8a224]
mongod(_ZN5mongo34SessionCatalogMigrationDestination31_retrieveSessionStateFromSourceEPNS_14ServiceContextE+0x22D8) [0x55856d150128]
mongod(+0x2307530) [0x55856e406530]
libpthread.so.0(+0x7DD5) [0x7f981bf37dd5]
libc.so.6(clone+0x6D) [0x7f981bc60ead]
----- END BACKTRACE -----