1. Nova 配置 disk_cachemodes = "network=writeback” (启用)
    改为 disk_cachemodes = “network=none"(关闭)

  2. Ceph 配置 1. 开启 ceph rbd cache client] rbd_cache = true rbd_cache_writethrough_until_flush = true admin_socket = /var/run/ceph/guests/$cluster-$type.$id.$pid.$cctid.asok log_file = /var/log/qemu/qemu-guest-$pid.log rbd_concurrent_management_ops = 20

     2. 关闭ceph rbd cache 
     [client]
     #rbd_cache = true
     #rbd_cache_writethrough_until_flush = true
     admin_socket = /var/run/ceph/guests/$cluster-$type.$id.$pid.$cctid.asok
     #log_file = /var/log/qemu/qemu-guest-$pid.log
     #rbd_concurrent_management_ops = 20
    
  3. 验证cache是否生效

    1. 验证 生效命令 [root@l2-4-30 guests]# ceph --admin-daemo ceph-client.cinder.27346.94699498717184.asok perf dump { "finisher-librbd::TaskFinisher::m_finisher": { "queue_len": 0, "complete_latency": { "avgcount": 0, "sum": 0.000000000 } }, "librbd-1373b25a3be0cd-glance_images-86e30f80-5a93-47ee-88c9-de5c1db6df10": { "rd": 32853, "rd_bytes": 2060131840, "rd_latency": { "avgcount": 32853, "sum": 42.348302430 }, "wr": 0, "wr_bytes": 0, "wr_latency": { "avgcount": 0, "sum": 0.000000000 }, "discard": 0, "discard_bytes": 0, "discard_latency": { "avgcount": 0, "sum": 0.000000000 }, "flush": 0, "aio_flush": 0, "aio_flush_latency": { "avgcount": 0, "sum": 0.000000000 }, "snap_create": 0, "snap_remove": 0, "snap_rollback": 0, "snap_rename": 0, "notify": 0, "resize": 0, "readahead": 103, "readahead_bytes": 27451904, "invalidate_cache": 0 }, "librbd-bf1708496d088e-nova_instances-555153af-f088-4aec-9bae-e4d6389b4d98_disk": { "rd": 40116, "rd_bytes": 650777600, "rd_latency": { "avgcount": 40116, "sum": 38.547805260 }, "wr": 7348, "wr_bytes": 253340672, "wr_latency": { "avgcount": 7348, "sum": 55.895274365 }, "discard": 0, "discard_bytes": 0, "discard_latency": { "avgcount": 0, "sum": 0.000000000 }, "flush": 0, "aio_flush": 434, "aio_flush_latency": { "avgcount": 434, "sum": 0.540130077 }, "snap_create": 0, "snap_remove": 0, "snap_rollback": 0, "snap_rename": 0, "notify": 0, "resize": 0, "readahead": 130, "readahead_bytes": 37353472, "invalidate_cache": 0 }, "objectcacher-librbd-1373b25a3be0cd-glance_images-86e30f80-5a93-47ee-88c9-de5c1db6df10": { "cache_ops_hit": 2761, "cache_ops_miss": 30195, "cache_bytes_hit": 1500713472, "cache_bytes_miss": 2063611392, "data_read": 2087583744, "data_written": 0, "data_flushed": 0, "data_overwritten_while_flushing": 0, "write_ops_blocked": 0, "write_bytes_blocked": 0, "write_time_blocked": 0.000000000 }, "objectcacher-librbd-bf1708496d088e-nova_instances-555153af-f088-4aec-9bae-e4d6389b4d98_disk": { "cache_ops_hit": 829, "cache_ops_miss": 7979, "cache_bytes_hit": 191516672, "cache_bytes_miss": 153311232, "data_read": 168047616, "data_written": 253340672, "data_flushed": 253340672, "data_overwritten_while_flushing": 0, "write_ops_blocked": 0, "write_bytes_blocked": 0, "write_time_blocked": 0.000000000 }, "objecter": { "op_active": 0, "op_laggy": 0, "op_send": 45229, "op_send_bytes": 0, "op_resend": 0, "op_ack": 45105, "op_commit": 7893, "op": 45229, "op_r": 37337, "op_w": 7892, "op_rmw": 0, "op_pg": 0, "osdop_stat": 7045, "osdop_create": 0, "osdop_read": 37319, "osdop_write": 7519, "osdop_writefull": 0, "osdop_append": 0, "osdop_zero": 0, "osdop_truncate": 0, "osdop_delete": 0, "osdop_mapext": 0, "osdop_sparse_read": 0, "osdop_clonerange": 0, "osdop_getxattr": 0, "osdop_setxattr": 0, "osdop_cmpxattr": 0, "osdop_rmxattr": 0, "osdop_resetxattrs": 0, "osdop_tmap_up": 0, "osdop_tmap_put": 0, "osdop_tmap_get": 0, "osdop_call": 1117, "osdop_watch": 122, "osdop_notify": 1, "osdop_src_cmpxattr": 0, "osdop_pgls": 0, "osdop_pgls_filter": 0, "osdop_other": 1, "linger_active": 1, "linger_send": 2, "linger_resend": 0, "linger_ping": 121, "poolop_active": 0, "poolop_send": 0, "poolop_resend": 0, "poolstat_active": 0, "poolstat_send": 0, "poolstat_resend": 0, "statfs_active": 0, "statfs_send": 0, "statfs_resend": 0, "command_active": 0, "command_send": 0, "command_resend": 0, "map_epoch": 0, "map_full": 0, "map_inc": 0, "osd_sessions": 27966, "osd_session_open": 236, "osd_session_close": 0, "osd_laggy": 1, "omap_wr": 0, "omap_rd": 0, "omap_del": 0 }, "throttle-msgr_dispatch_throttler-radosclient": { "val": 0, "max": 104857600, "get": 52511, "get_sum": 2152111179, "get_or_fail_fail": 0, "get_or_fail_success": 0, "take": 0, "take_sum": 0, "put": 52511, "put_sum": 2152111179, "wait": { "avgcount": 0, "sum": 0.000000000 } }, "throttle-objecter_bytes": { "val": 0, "max": 104857600, "get": 45108, "get_sum": 2419831296, "get_or_fail_fail": 0, "get_or_fail_success": 45108, "take": 0, "take_sum": 0, "put": 44725, "put_sum": 2419831296, "wait": { "avgcount": 0, "sum": 0.000000000 } }, "throttle-objecter_ops": { "val": 0, "max": 1024, "get": 45108, "get_sum": 45108, "get_or_fail_fail": 0, "get_or_fail_success": 45108, "take": 0, "take_sum": 0, "put": 45108, "put_sum": 45108, "wait": { "avgcount": 0, "sum": 0.000000000 } } } 完整配置

    过滤 cache 字符串

    	[root@l2-4-30 guests]# ceph --admin-daemo ceph-client.cinder.27346.94699498717184.asok   perf  dump |grep cache -A 5 -B 5
    			"snap_rename": 0,
    			"notify": 0,
    			"resize": 0,
    			"readahead": 103,
    			"readahead_bytes": 27451904,
    			"invalidate_cache": 0
    	},
    	"librbd-bf1708496d088e-nova_instances-555153af-f088-4aec-9bae-e4d6389b4d98_disk": {
    			"rd": 57149,
    			"rd_bytes": 1148467200,
    			"rd_latency": {
    

    -- "snap_rename": 0, "notify": 0, "resize": 0, "readahead": 130, "readahead_bytes": 37353472, "invalidate_cache": 0 }, "objectcacher-librbd-1373b25a3be0cd-glance_images-86e30f80-5a93-47ee-88c9-de5c1db6df10": { "cache_ops_hit": 2791, "cache_ops_miss": 45597, "cache_bytes_hit": 1526398976, "cache_bytes_miss": 3029461504, "data_read": 3057748992, "data_written": 0, "data_flushed": 0, "data_overwritten_while_flushing": 0, "write_ops_blocked": 0, "write_bytes_blocked": 0, "write_time_blocked": 0.000000000 }, "objectcacher-librbd-bf1708496d088e-nova_instances-555153af-f088-4aec-9bae-e4d6389b4d98_disk": { "cache_ops_hit": 886, "cache_ops_miss": 9957, "cache_bytes_hit": 248139264, "cache_bytes_miss": 225620480, "data_read": 240729088, "data_written": 462767104, "data_flushed": 462767104, "data_overwritten_while_flushing": 0, "write_ops_blocked": 0,

    1. 验证 无cache命令 [root@l23-4-40 guests]# ceph --admin-daemo ceph-client.cinder.138869.94547912278016.asok perf dump |grep cache -A 5 -B 5 "snap_rename": 0, "notify": 0, "resize": 0, "readahead": 0, "readahead_bytes": 0, "invalidate_cache": 0 }, "librbd-bee65365887c0b-nova_instances-5cf85c18-6725-4674-88e4-60408c97f487_disk": { "rd": 62937, "rd_bytes": 1261381632, "rd_latency": {

       			"snap_rename": 0,
       			"notify": 0,
       			"resize": 0,
       			"readahead": 0,
       			"readahead_bytes": 0,
       			"invalidate_cache": 0
       	},
       	"objecter": {
       			"op_active": 0,
       			"op_laggy": 0,
       			"op_send": 95327
       	}