2012-11-22 14:14:09.330212 osd.10 192.168.30.11:6800/2875 365 : [WRN] 1 slow requests, 1 included below; oldest blocked for > 30.678784 secs
2012-11-22 14:14:09.330216 osd.10 192.168.30.11:6800/2875 366 : [WRN] slow request 30.678784 seconds old, received at 2012-11-22 14:13:38.651386: osd_op(client.7564.0:103733 rbd_data.1b1d413902f9.00000000000003dd [write 720896~4096] 2.c40372a6 snapc d=[d]) v4 currently delayed
2012-11-22 14:14:09.758117 osd.11 192.168.30.11:6803/2950 341 : [WRN] 1 slow requests, 1 included below; oldest blocked for > 30.003742 secs
2012-11-22 14:14:09.758124 osd.11 192.168.30.11:6803/2950 342 : [WRN] slow request 30.003742 seconds old, received at 2012-11-22 14:13:39.754328: osd_op(client.7346.0:423426 rbd_data.1b335245df3e.0000000000000721 [write 851968~61440] 2.33695508 snapc c=[c]) v4 currently delayed
2012-11-22 14:14:39.476619 osd.10 192.168.30.11:6800/2875 381 : [WRN] 1 slow requests, 1 included below; oldest blocked for > 60.825212 secs
2012-11-22 14:14:39.476624 osd.10 192.168.30.11:6800/2875 382 : [WRN] slow request 60.825212 seconds old, received at 2012-11-22 14:13:38.651386: osd_op(client.7564.0:103733 rbd_data.1b1d413902f9.00000000000003dd [write 720896~4096] 2.c40372a6 snapc d=[d]) v4 currently delayed
2012-11-22 14:14:39.892445 osd.11 192.168.30.11:6803/2950 351 : [WRN] 1 slow requests, 1 included below; oldest blocked for > 60.138089 secs
2012-11-22 14:14:39.892452 osd.11 192.168.30.11:6803/2950 352 : [WRN] slow request 60.138089 seconds old, received at 2012-11-22 14:13:39.754328: osd_op(client.7346.0:423426 rbd_data.1b335245df3e.0000000000000721 [write 851968~61440] 2.33695508 snapc c=[c]) v4 currently delayed
2012-11-22 14:15:38.717242 osd.10 192.168.30.11:6800/2875 386 : [WRN] 1 slow requests, 1 included below; oldest blocked for > 120.065830 secs
2012-11-22 14:15:38.717247 osd.10 192.168.30.11:6800/2875 387 : [WRN] slow request 120.065830 seconds old, received at 2012-11-22 14:13:38.651386: osd_op(client.7564.0:103733 rbd_data.1b1d413902f9.00000000000003dd [write 720896~4096] 2.c40372a6 snapc d=[d]) v4 currently delayed
2012-11-22 14:15:40.137194 osd.11 192.168.30.11:6803/2950 376 : [WRN] 1 slow requests, 1 included below; oldest blocked for > 120.382837 secs
2012-11-22 14:15:40.137202 osd.11 192.168.30.11:6803/2950 377 : [WRN] slow request 120.382837 seconds old, received at 2012-11-22 14:13:39.754328: osd_op(client.7346.0:423426 rbd_data.1b335245df3e.0000000000000721 [write 851968~61440] 2.33695508 snapc c=[c]) v4 currently delayed
2012-11-22 14:17:39.238906 osd.10 192.168.30.11:6800/2875 388 : [WRN] 1 slow requests, 1 included below; oldest blocked for > 240.587493 secs
2012-11-22 14:17:39.238912 osd.10 192.168.30.11:6800/2875 389 : [WRN] slow request 240.587493 seconds old, received at 2012-11-22 14:13:38.651386: osd_op(client.7564.0:103733 rbd_data.1b1d413902f9.00000000000003dd [write 720896~4096] 2.c40372a6 snapc d=[d]) v4 currently delayed
2012-11-22 14:17:40.643054 osd.11 192.168.30.11:6803/2950 378 : [WRN] 1 slow requests, 1 included below; oldest blocked for > 240.888699 secs
2012-11-22 14:17:40.643059 osd.11 192.168.30.11:6803/2950 379 : [WRN] slow request 240.888699 seconds old, received at 2012-11-22 14:13:39.754328: osd_op(client.7346.0:423426 rbd_data.1b335245df3e.0000000000000721 [write 851968~61440] 2.33695508 snapc c=[c]) v4 currently delayed
2012-11-22 14:21:39.266295 osd.10 192.168.30.11:6800/2875 575 : [WRN] 1 slow requests, 1 included below; oldest blocked for > 480.614882 secs
2012-11-22 14:21:39.266300 osd.10 192.168.30.11:6800/2875 576 : [WRN] slow request 480.614882 seconds old, received at 2012-11-22 14:13:38.651386: osd_op(client.7564.0:103733 rbd_data.1b1d413902f9.00000000000003dd [write 720896~4096] 2.c40372a6 snapc d=[d]) v4 currently delayed
2012-11-22 14:21:40.650230 osd.11 192.168.30.11:6803/2950 579 : [WRN] 1 slow requests, 1 included below; oldest blocked for > 480.895878 secs
2012-11-22 14:21:40.650235 osd.11 192.168.30.11:6803/2950 580 : [WRN] slow request 480.895878 seconds old, received at 2012-11-22 14:13:39.754328: osd_op(client.7346.0:423426 rbd_data.1b335245df3e.0000000000000721 [write 851968~61440] 2.33695508 snapc c=[c]) v4 currently delayed
2012-11-22 14:22:40.735235 osd.7 192.168.30.11:6809/3236 571 : [WRN] 1 slow requests, 1 included below; oldest blocked for > 30.187211 secs
2012-11-22 14:22:40.735241 osd.7 192.168.30.11:6809/3236 572 : [WRN] slow request 30.187211 seconds old, received at 2012-11-22 14:22:10.547972: osd_op(client.7343.0:86890 rbd_data.1b203a24ea54.000000000000043c [write 1372160~4096] 2.711430d4 snapc b=[b]) v4 currently delayed
2012-11-22 14:23:10.857628 osd.7 192.168.30.11:6809/3236 573 : [WRN] 1 slow requests, 1 included below; oldest blocked for > 60.309625 secs
2012-11-22 14:23:10.857635 osd.7 192.168.30.11:6809/3236 574 : [WRN] slow request 60.309625 seconds old, received at 2012-11-22 14:22:10.547972: osd_op(client.7343.0:86890 rbd_data.1b203a24ea54.000000000000043c [write 1372160~4096] 2.711430d4 snapc b=[b]) v4 currently delayed
2012-11-22 14:23:12.657833 osd.10 192.168.30.11:6800/2875 578 : [WRN] 3 slow requests, 3 included below; oldest blocked for > 30.244264 secs
2012-11-22 14:23:12.657837 osd.10 192.168.30.11:6800/2875 579 : [WRN] slow request 30.244264 seconds old, received at 2012-11-22 14:22:42.413529: osd_op(client.7564.0:103763 rbd_data.1b1d413902f9.00000000000003dd [write 720896~4096] 2.c40372a6 snapc d=[d]) v4 currently delayed
2012-11-22 14:23:12.657841 osd.10 192.168.30.11:6800/2875 580 : [WRN] slow request 30.244163 seconds old, received at 2012-11-22 14:22:42.413630: osd_op(client.7564.0:103764 rbd_data.1b1d413902f9.00000000000003dd [write 1306624~4096] 2.c40372a6 snapc d=[d]) v4 currently delayed
2012-11-22 14:23:12.657845 osd.10 192.168.30.11:6800/2875 581 : [WRN] slow request 30.244129 seconds old, received at 2012-11-22 14:22:42.413664: osd_op(client.7564.0:103765 rbd_data.1b1d413902f9.00000000000003dd [write 1810432~4096] 2.c40372a6 snapc d=[d]) v4 currently delayed
{ "num_ops": 1,
"ops": [
{ "description": "osd_op(client.7564.0:103733 rbd_data.1b1d413902f9.00000000000003dd [write 720896~4096] 2.c40372a6 snapc d=[d])",
"received_at": "2012-11-22 14:13:38.651386",
"age": "474.525834",
"duration": "0.000120",
"flag_point": "delayed",
"client_info": { "client": "client.7564",
"tid": 103733},
"events": [
{ "time": "2012-11-22 14:13:38.651476",
"event": "waiting_for_osdmap"},
{ "time": "2012-11-22 14:13:38.651506",
"event": "reached_pg"}]}]}
{ "num_ops": 1,
"ops": [
{ "description": "osd_op(client.7346.0:423426 rbd_data.1b335245df3e.0000000000000721 [write 851968~61440] 2.33695508 snapc c=[c])",
"received_at": "2012-11-22 14:13:39.754328",
"age": "479.784385",
"duration": "0.000278",
"flag_point": "delayed",
"client_info": { "client": "client.7346",
"tid": 423426},
"events": [
{ "time": "2012-11-22 14:13:39.754575",
"event": "waiting_for_osdmap"},
{ "time": "2012-11-22 14:13:39.754606",
"event": "reached_pg"}]}]}
{ "num_ops": 1,
"ops": [
{ "description": "osd_op(client.7343.0:86890 rbd_data.1b203a24ea54.000000000000043c [write 1372160~4096] 2.711430d4 snapc b=[b])",
"received_at": "2012-11-22 14:22:10.547972",
"age": "69.295647",
"duration": "0.000157",
"flag_point": "delayed",
"client_info": { "client": "client.7343",
"tid": 86890},
"events": [
{ "time": "2012-11-22 14:22:10.548079",
"event": "waiting_for_osdmap"},
{ "time": "2012-11-22 14:22:10.548129",
"event": "reached_pg"}]}]}