Failed to assign buffers. All buffers(2) are being used #414
Description
Describe the bug
running kvcache and met fail to assign buffer issue, tried to incrase but not able to increase, it seems it is stuck at 2 in the following
I0828 11:59:32.640034 22913 Cache.h:711] "navyConfig::blockCacheNumInMemBuffers": "2",
Expected behavior
Please provide some guidance on how to increase this "blockCacheNumInMemBuffers"
Related log
===JSON Config===
{
"cache_config": {
"cacheSizeMB": 43000,
"cacheDir": "/var/tmp/cachelib_meta",
"allocFactor": 1.08,
"maxAllocSize": 524288,
"minAllocSize": 64,
"nvmCachePaths": ["/dev/nvme7n1"],
"nvmCacheSizeMB": 952320,
"navyBlockDevice": true,
"navyEnableIoUring": false,
"navyQDepth": 32,
"deviceMaxWriteSize": 196608,
"navyReaderThreads": 32,
"navyWriterThreads": 16,
"navyBigHashBucketSize": 4096,
"navyBigHashSizePct": 10,
"navySmallItemMaxSize": 640,
"navyBlockSize": 4096,
"navyParcelMemoryMB": 6048,
"nvmAdmissionRetentionTimeThreshold": 7200,
"enableChainedItem": true,
"htBucketPower": 28,
"moveOnSlabRelease": false,
"poolRebalanceIntervalSec": 2,
"poolResizeIntervalSec": 2,
"rebalanceStrategy": "hits",
"writeAmpDeviceList": ["nvme7n1"]
},
"test_config": {
"opRatePerSec": 550000,
"opRateBurstSize": 200,
"generator": "replay",
"replayGeneratorConfig": { "ampFactor": 100 },
"repeatTraceReplay": true,
"repeatOpCount": true,
"onlySetIfMiss": false,
"numOps": 100000000000,
"numThreads": 10,
"prepopulateCache": true,
"enableLookaside": false,
"traceFileNames": [
"kvcache_traces_1.csv",
"kvcache_traces_2.csv",
"kvcache_traces_3.csv",
"kvcache_traces_4.csv",
"kvcache_traces_5.csv"
]
}
}
Welcome to OSS version of cachebench
I0828 11:59:32.596024 22913 KVReplayGenerator.h:106] Started KVReplayGenerator (amp factor 100, # of stressor threads 10)
I0828 11:59:32.596048 22914 ReplayGeneratorBase.h:218] [0] Opened trace file kvcache_traces_1.csv
I0828 11:59:32.596574 22914 ReplayGeneratorBase.h:179] New header detected: header "key,op,size,op_count,key_size" field map key -> 0, op -> 1, size -> 2, op_count -> 3, key_size -> 4
E0828 11:59:32.638914 22913 Cache.h:498] Exception fetching nand writes for nvme7n1. Msg: Vendor not recogized in device model number solidigm snbph27x076tgf
I0828 11:59:32.639419 22913 Cache.h:622] Configuring NVM cache: simple file /dev/nvme7n1 size 952320 MB
I0828 11:59:32.640034 22913 Cache.h:711] Using the following nvm config{
I0828 11:59:32.640034 22913 Cache.h:711] "navyConfig::QDepth": "32",
I0828 11:59:32.640034 22913 Cache.h:711] "navyConfig::admissionPolicy": "",
I0828 11:59:32.640034 22913 Cache.h:711] "navyConfig::admissionProbBaseSize": "0",
I0828 11:59:32.640034 22913 Cache.h:711] "navyConfig::admissionProbFactorLowerBound": "0",
I0828 11:59:32.640034 22913 Cache.h:711] "navyConfig::admissionProbFactorUpperBound": "0",
I0828 11:59:32.640034 22913 Cache.h:711] "navyConfig::admissionProbability": "0",
I0828 11:59:32.640034 22913 Cache.h:711] "navyConfig::admissionSuffixLen": "0",
I0828 11:59:32.640034 22913 Cache.h:711] "navyConfig::admissionWriteRate": "0",
I0828 11:59:32.640034 22913 Cache.h:711] "navyConfig::bigHashBucketBfSize": "8",
I0828 11:59:32.640034 22913 Cache.h:711] "navyConfig::bigHashBucketSize": "4096",
I0828 11:59:32.640034 22913 Cache.h:711] "navyConfig::bigHashSizePct": "10",
I0828 11:59:32.640034 22913 Cache.h:711] "navyConfig::bigHashSmallItemMaxSize": "640",
I0828 11:59:32.640034 22913 Cache.h:711] "navyConfig::blockCacheCleanRegionThreads": "1",
I0828 11:59:32.640034 22913 Cache.h:711] "navyConfig::blockCacheCleanRegions": "1",
I0828 11:59:32.640034 22913 Cache.h:711] "navyConfig::blockCacheDataChecksum": "true",
I0828 11:59:32.640034 22913 Cache.h:711] "navyConfig::blockCacheLru": "true",
I0828 11:59:32.640034 22913 Cache.h:711] "navyConfig::blockCacheNumInMemBuffers": "2",
I0828 11:59:32.640034 22913 Cache.h:711] "navyConfig::blockCacheRegionSize": "16777216",
I0828 11:59:32.640034 22913 Cache.h:711] "navyConfig::blockCacheReinsertionHitsThreshold": "0",
I0828 11:59:32.640034 22913 Cache.h:711] "navyConfig::blockCacheReinsertionPctThreshold": "0",
I0828 11:59:32.640034 22913 Cache.h:711] "navyConfig::blockCacheSegmentedFifoSegmentRatio": "",
I0828 11:59:32.640034 22913 Cache.h:711] "navyConfig::blockSize": "4096",
I0828 11:59:32.640034 22913 Cache.h:711] "navyConfig::deviceMaxWriteSize": "196608",
I0828 11:59:32.640034 22913 Cache.h:711] "navyConfig::deviceMetadataSize": "0",
I0828 11:59:32.640034 22913 Cache.h:711] "navyConfig::enableFDP": "0",
I0828 11:59:32.640034 22913 Cache.h:711] "navyConfig::fileName": "/dev/nvme7n1",
I0828 11:59:32.640034 22913 Cache.h:711] "navyConfig::fileSize": "998579896320",
I0828 11:59:32.640034 22913 Cache.h:711] "navyConfig::ioEngine": "libaio",
I0828 11:59:32.640034 22913 Cache.h:711] "navyConfig::maxConcurrentInserts": "1000000",
I0828 11:59:32.640034 22913 Cache.h:711] "navyConfig::maxNumReads": "0",
I0828 11:59:32.640034 22913 Cache.h:711] "navyConfig::maxNumWrites": "0",
I0828 11:59:32.640034 22913 Cache.h:711] "navyConfig::maxParcelMemoryMB": "6048",
I0828 11:59:32.640034 22913 Cache.h:711] "navyConfig::maxWriteRate": "0",
I0828 11:59:32.640034 22913 Cache.h:711] "navyConfig::navyReqOrderingShards": "21",
I0828 11:59:32.640034 22913 Cache.h:711] "navyConfig::raidPaths": "",
I0828 11:59:32.640034 22913 Cache.h:711] "navyConfig::readerThreads": "32",
I0828 11:59:32.640034 22913 Cache.h:711] "navyConfig::stackSize": "16384",
I0828 11:59:32.640034 22913 Cache.h:711] "navyConfig::truncateFile": "false",
I0828 11:59:32.640034 22913 Cache.h:711] "navyConfig::writerThreads": "16"
I0828 11:59:32.640034 22913 Cache.h:711] }
I0828 11:59:32.641212 22913 Cache.h:750] Failed to attach for reason: Unable to find any segment with name shm_info
E0828 11:59:37.637370 22913 NvmCacheState.cpp:135] unable to deserialize nvm metadata file: no content in file: /var/tmp/cachelib_meta/NvmCacheState
I0828 11:59:37.647208 22913 Device.cpp:1085] Cache file: /dev/nvme7n1 size: 998579896320 truncate: 0
I0828 11:59:37.647334 22913 Device.cpp:970] Created device with num_devices 1 size 998579896320 block_size 4096,stripe_size 0 max_write_size 196608 max_io_size 196608 io_engine libaio qdepth 32,num_fdp_devices 0
I0828 11:59:37.813466 22913 NavySetup.cpp:243] metadataSize: 4992897024
I0828 11:59:37.813526 22913 NavySetup.cpp:245] Setting up engine pair 0
I0828 11:59:37.813547 22913 NavySetup.cpp:111] bighashStartingLimit: 4992897024 bigHashCacheOffset: 898721906688 bigHashCacheSize: 99857989632
I0828 11:59:37.813560 22913 NavySetup.cpp:259] blockCacheSize 893729009664
I0828 11:59:37.813572 22913 NavySetup.cpp:156] blockcache: starting offset: 4992897024, block cache size: 893722296320
I0828 11:59:37.813708 22913 LruPolicy.cpp:35] LRU policy: expected 53270 regions
I0828 11:59:37.960658 22913 BigHash.cpp:93] BigHash created: buckets: 24379392, bucket size: 4096, base offset: 898721906688
I0828 11:59:37.960726 22913 BigHash.cpp:102] Reset BigHash
I0828 11:59:37.985409 22913 RegionManager.cpp:50] 53270 regions, 16777216 bytes each
I0828 11:59:38.013070 22964 RegionManager.cpp:68] region_manager_0 started
I0828 11:59:38.064168 22913 Allocator.cpp:39] Enable priority-based allocation for Allocator. Number of priorities: 1
I0828 11:59:38.064239 22913 BlockCache.cpp:145] Block cache created
I0828 11:59:38.064573 22913 Driver.cpp:70] Max concurrent inserts: 1000000
I0828 11:59:38.064597 22913 Driver.cpp:71] Max parcel memory: 6341787648
I0828 11:59:38.064612 22913 Driver.cpp:72] Use Write Estimated Size: false
I0828 11:59:38.064634 22913 Driver.cpp:209] Reset Navy
I0828 11:59:38.064662 22913 BigHash.cpp:102] Reset BigHash
I0828 11:59:38.080072 22913 BlockCache.cpp:707] Reset block cache
Total 1000000.00M ops to be run
11:59:38 0.00M ops completed. Hit Ratio 90.00% (RAM 91.00%, NVM 0.00%)
12:00:38 32.31M ops completed. Hit Ratio 9.86% (RAM 9.86%, NVM 0.00%)
12:01:38 64.62M ops completed. Hit Ratio 20.65% (RAM 20.65%, NVM -0.00%)
12:02:38 96.86M ops completed. Hit Ratio 33.11% (RAM 33.11%, NVM 0.00%)
12:03:38 129.39M ops completed. Hit Ratio 34.99% (RAM 34.99%, NVM -0.00%)
12:04:38 161.69M ops completed. Hit Ratio 29.53% (RAM 29.53%, NVM 0.00%)
12:05:38 194.06M ops completed. Hit Ratio 35.63% (RAM 35.63%, NVM -0.00%)
12:06:38 226.39M ops completed. Hit Ratio 37.71% (RAM 37.71%, NVM 0.00%)
12:07:38 258.86M ops completed. Hit Ratio 39.27% (RAM 39.27%, NVM 0.00%)
12:08:38 291.29M ops completed. Hit Ratio 33.95% (RAM 33.95%, NVM 0.00%)
12:09:38 323.45M ops completed. Hit Ratio 39.75% (RAM 39.75%, NVM -0.00%)
12:10:39 355.93M ops completed. Hit Ratio 43.17% (RAM 43.17%, NVM 0.00%)
12:11:39 388.36M ops completed. Hit Ratio 37.14% (RAM 37.14%, NVM 0.00%)
12:12:39 420.80M ops completed. Hit Ratio 44.84% (RAM 44.84%, NVM -0.00%)
12:13:39 453.37M ops completed. Hit Ratio 44.40% (RAM 44.40%, NVM 0.00%)
12:14:39 485.79M ops completed. Hit Ratio 44.13% (RAM 44.13%, NVM 0.00%)
I0828 12:15:10.938798 22964 Device.cpp:780] [ctx_0] Created new async io context with qdepth 32 io_engine libaio
E0828 12:15:12.335407 22948 RegionManager.cpp:181] Failed to assign buffers. All buffers(2) are being used
E0828 12:15:25.528841 22954 RegionManager.cpp:181] Failed to assign buffers. All buffers(2) are being used
E0828 12:15:36.335482 22953 RegionManager.cpp:181] Failed to assign buffers. All buffers(2) are being used
12:15:39 517.90M ops completed. Hit Ratio 42.02% (RAM 42.01%, NVM 0.01%)
E0828 12:15:53.231555 22960 RegionManager.cpp:181] Failed to assign buffers. All buffers(2) are being used
E0828 12:16:10.277482 22961 RegionManager.cpp:181] Failed to assign buffers. All buffers(2) are being used
E0828 12:16:20.276806 22961 RegionManager.cpp:181] Failed to assign buffers. All buffers(2) are being used
E0828 12:16:30.998905 22954 RegionManager.cpp:181] Failed to assign buffers. All buffers(2) are being used
12:16:39 549.37M ops completed. Hit Ratio 45.38% (RAM 45.34%, NVM 0.07%)
E0828 12:16:42.982179 22959 RegionManager.cpp:181] Failed to assign buffers. All buffers(2) are being used
E0828 12:16:58.429018 22954 RegionManager.cpp:181] Failed to assign buffers. All buffers(2) are being used
E0828 12:17:08.687214 22961 RegionManager.cpp:181] Failed to assign buffers. All buffers(2) are being used
E0828 12:17:21.591422 22950 RegionManager.cpp:181] Failed to assign buffers. All buffers(2) are being used
E0828 12:17:36.023783 22950 RegionManager.cpp:181] Failed to assign buffers. All buffers(2) are being used
12:17:39 580.88M ops completed. Hit Ratio 48.28% (RAM 48.21%, NVM 0.14%)
E0828 12:17:47.105793 22962 RegionManager.cpp:181] Failed to assign buffers. All buffers(2) are being used
E0828 12:18:01.100442 22950 RegionManager.cpp:181] Failed to assign buffers. All buffers(2) are being used
E0828 12:18:11.100159 22954 RegionManager.cpp:181] Failed to assign buffers. All buffers(2) are being used
E0828 12:18:31.403505 22948 RegionManager.cpp:181] Failed to assign buffers. All buffers(2) are being used
12:18:39 612.28M ops completed. Hit Ratio 36.50% (RAM 36.39%, NVM 0.17%)
E0828 12:18:41.835194 22948 RegionManager.cpp:181] Failed to assign buffers. All buffers(2) are being used
E0828 12:18:55.052945 22957 RegionManager.cpp:181] Failed to assign buffers. All buffers(2) are being used
E0828 12:19:13.595917 22961 RegionManager.cpp:181] Failed to assign buffers. All buffers(2) are being used
E0828 12:19:25.874168 22949 RegionManager.cpp:181] Failed to assign buffers. All buffers(2) are being used
E0828 12:19:37.042677 22963 RegionManager.cpp:181] Failed to assign buffers. All buffers(2) are being used
12:19:39 643.29M ops completed. Hit Ratio 45.51% (RAM 45.38%, NVM 0.24%)
E0828 12:19:47.044314 22952 RegionManager.cpp:181] Failed to assign buffers. All buffers(2) are being used
E0828 12:19:57.098433 22960 RegionManager.cpp:181] Failed to assign buffers. All buffers(2) are being used
E0828 12:20:13.160506 22960 RegionManager.cpp:181] Failed to assign buffers. All buffers(2) are being used
E0828 12:20:28.029697 22951 RegionManager.cpp:181] Failed to assign buffers. All buffers(2) are being used
E0828 12:20:38.397520 22955 RegionManager.cpp:181] Failed to assign buffers. All buffers(2) are being used
12:20:39 674.23M ops completed. Hit Ratio 45.55% (RAM 45.33%, NVM 0.40%)
E0828 12:20:49.040838 22961 RegionManager.cpp:181] Failed to assign buffers. All buffers(2) are being used
E0828 12:21:01.804480 22963 RegionManager.cpp:181] Failed to assign buffers. All buffers(2) are being used
E0828 12:21:12.660606 22956 RegionManager.cpp:181] Failed to assign buffers. All buffers(2) are being used
E0828 12:21:31.857285 22949 RegionManager.cpp:181] Failed to assign buffers. All buffers(2) are being used
12:21:39 704.84M ops completed. Hit Ratio 45.94% (RAM 45.71%, NVM 0.42%)
E0828 12:21:43.260118 22958 RegionManager.cpp:181] Failed to assign buffers. All buffers(2) are being used
E0828 12:21:58.208551 22963 RegionManager.cpp:181] Failed to assign buffers. All buffers(2) are being used
E0828 12:22:08.663174 22948 RegionManager.cpp:181] Failed to assign buffers. All buffers(2) are being used
E0828 12:22:24.253245 22954 RegionManager.cpp:181] Failed to assign buffers. All buffers(2) are being used
12:22:40 735.80M ops completed. Hit Ratio 51.25% (RAM 50.99%, NVM 0.53%)
E0828 12:22:44.863645 22952 RegionManager.cpp:181] Failed to assign buffers. All buffers(2) are being used
E0828 12:22:56.354596 22952 RegionManager.cpp:181] Failed to assign buffers. All buffers(2) are being used
E0828 12:23:07.300051 22951 RegionManager.cpp:181] Failed to assign buffers. All buffers(2) are being used
E0828 12:23:18.585624 22955 RegionManager.cpp:181] Failed to assign buffers. All buffers(2) are being used
E0828 12:23:37.090524 22950 RegionManager.cpp:181] Failed to assign buffers. All buffers(2) are being used
12:23:40 766.29M ops completed. Hit Ratio 44.19% (RAM 43.90%, NVM 0.52%)
E0828 12:23:49.779486 22963 RegionManager.cpp:181] Failed to assign buffers. All buffers(2) are being used
E0828 12:24:03.425178 22953 RegionManager.cpp:181] Failed to assign buffers. All buffers(2) are being used
E0828 12:24:19.538671 22952 RegionManager.cpp:181] Failed to assign buffers. All buffers(2) are being used