Commit 1dd8d470 authored by Xiubo Li's avatar Xiubo Li Committed by Ilya Dryomov

ceph: metrics for opened files, pinned caps and opened inodes

In client for each inode, it may have many opened files and may
have been pinned in more than one MDS servers. And some inodes
are idle, which have no any opened files.

This patch will show these metrics in the debugfs, likes:

item                               total
-----------------------------------------
opened files  / total inodes       14 / 5
pinned i_caps / total inodes       7  / 5
opened inodes / total inodes       3  / 5

Will send these metrics to ceph, which will be used by the `fs top`,
later.

[ jlayton: drop unrelated hunk, count hashed inodes instead of
           allocated ones ]

URL: https://tracker.ceph.com/issues/47005Signed-off-by: default avatarXiubo Li <xiubli@redhat.com>
Signed-off-by: default avatarJeff Layton <jlayton@kernel.org>
Signed-off-by: default avatarIlya Dryomov <idryomov@gmail.com>
parent 2678da88
...@@ -4283,13 +4283,30 @@ void __ceph_touch_fmode(struct ceph_inode_info *ci, ...@@ -4283,13 +4283,30 @@ void __ceph_touch_fmode(struct ceph_inode_info *ci,
void ceph_get_fmode(struct ceph_inode_info *ci, int fmode, int count) void ceph_get_fmode(struct ceph_inode_info *ci, int fmode, int count)
{ {
int i; struct ceph_mds_client *mdsc = ceph_sb_to_mdsc(ci->vfs_inode.i_sb);
int bits = (fmode << 1) | 1; int bits = (fmode << 1) | 1;
bool is_opened = false;
int i;
if (count == 1)
atomic64_inc(&mdsc->metric.opened_files);
spin_lock(&ci->i_ceph_lock); spin_lock(&ci->i_ceph_lock);
for (i = 0; i < CEPH_FILE_MODE_BITS; i++) { for (i = 0; i < CEPH_FILE_MODE_BITS; i++) {
if (bits & (1 << i)) if (bits & (1 << i))
ci->i_nr_by_mode[i] += count; ci->i_nr_by_mode[i] += count;
/*
* If any of the mode ref is larger than 1,
* that means it has been already opened by
* others. Just skip checking the PIN ref.
*/
if (i && ci->i_nr_by_mode[i] > 1)
is_opened = true;
} }
if (!is_opened)
percpu_counter_inc(&mdsc->metric.opened_inodes);
spin_unlock(&ci->i_ceph_lock); spin_unlock(&ci->i_ceph_lock);
} }
...@@ -4300,15 +4317,32 @@ void ceph_get_fmode(struct ceph_inode_info *ci, int fmode, int count) ...@@ -4300,15 +4317,32 @@ void ceph_get_fmode(struct ceph_inode_info *ci, int fmode, int count)
*/ */
void ceph_put_fmode(struct ceph_inode_info *ci, int fmode, int count) void ceph_put_fmode(struct ceph_inode_info *ci, int fmode, int count)
{ {
int i; struct ceph_mds_client *mdsc = ceph_sb_to_mdsc(ci->vfs_inode.i_sb);
int bits = (fmode << 1) | 1; int bits = (fmode << 1) | 1;
bool is_closed = true;
int i;
if (count == 1)
atomic64_dec(&mdsc->metric.opened_files);
spin_lock(&ci->i_ceph_lock); spin_lock(&ci->i_ceph_lock);
for (i = 0; i < CEPH_FILE_MODE_BITS; i++) { for (i = 0; i < CEPH_FILE_MODE_BITS; i++) {
if (bits & (1 << i)) { if (bits & (1 << i)) {
BUG_ON(ci->i_nr_by_mode[i] < count); BUG_ON(ci->i_nr_by_mode[i] < count);
ci->i_nr_by_mode[i] -= count; ci->i_nr_by_mode[i] -= count;
} }
/*
* If any of the mode ref is not 0 after
* decreased, that means it is still opened
* by others. Just skip checking the PIN ref.
*/
if (i && ci->i_nr_by_mode[i])
is_closed = false;
} }
if (is_closed)
percpu_counter_dec(&mdsc->metric.opened_inodes);
spin_unlock(&ci->i_ceph_lock); spin_unlock(&ci->i_ceph_lock);
} }
......
...@@ -148,6 +148,17 @@ static int metric_show(struct seq_file *s, void *p) ...@@ -148,6 +148,17 @@ static int metric_show(struct seq_file *s, void *p)
int nr_caps = 0; int nr_caps = 0;
s64 total, sum, avg, min, max, sq; s64 total, sum, avg, min, max, sq;
sum = percpu_counter_sum(&m->total_inodes);
seq_printf(s, "item total\n");
seq_printf(s, "------------------------------------------\n");
seq_printf(s, "%-35s%lld / %lld\n", "opened files / total inodes",
atomic64_read(&m->opened_files), sum);
seq_printf(s, "%-35s%lld / %lld\n", "pinned i_caps / total inodes",
atomic64_read(&m->total_caps), sum);
seq_printf(s, "%-35s%lld / %lld\n", "opened inodes / total inodes",
percpu_counter_sum(&m->opened_inodes), sum);
seq_printf(s, "\n");
seq_printf(s, "item total avg_lat(us) min_lat(us) max_lat(us) stdev(us)\n"); seq_printf(s, "item total avg_lat(us) min_lat(us) max_lat(us) stdev(us)\n");
seq_printf(s, "-----------------------------------------------------------------------------------\n"); seq_printf(s, "-----------------------------------------------------------------------------------\n");
......
...@@ -42,10 +42,13 @@ static void ceph_inode_work(struct work_struct *work); ...@@ -42,10 +42,13 @@ static void ceph_inode_work(struct work_struct *work);
static int ceph_set_ino_cb(struct inode *inode, void *data) static int ceph_set_ino_cb(struct inode *inode, void *data)
{ {
struct ceph_inode_info *ci = ceph_inode(inode); struct ceph_inode_info *ci = ceph_inode(inode);
struct ceph_mds_client *mdsc = ceph_sb_to_mdsc(inode->i_sb);
ci->i_vino = *(struct ceph_vino *)data; ci->i_vino = *(struct ceph_vino *)data;
inode->i_ino = ceph_vino_to_ino_t(ci->i_vino); inode->i_ino = ceph_vino_to_ino_t(ci->i_vino);
inode_set_iversion_raw(inode, 0); inode_set_iversion_raw(inode, 0);
percpu_counter_inc(&mdsc->metric.total_inodes);
return 0; return 0;
} }
...@@ -538,11 +541,14 @@ void ceph_free_inode(struct inode *inode) ...@@ -538,11 +541,14 @@ void ceph_free_inode(struct inode *inode)
void ceph_evict_inode(struct inode *inode) void ceph_evict_inode(struct inode *inode)
{ {
struct ceph_inode_info *ci = ceph_inode(inode); struct ceph_inode_info *ci = ceph_inode(inode);
struct ceph_mds_client *mdsc = ceph_sb_to_mdsc(inode->i_sb);
struct ceph_inode_frag *frag; struct ceph_inode_frag *frag;
struct rb_node *n; struct rb_node *n;
dout("evict_inode %p ino %llx.%llx\n", inode, ceph_vinop(inode)); dout("evict_inode %p ino %llx.%llx\n", inode, ceph_vinop(inode));
percpu_counter_dec(&mdsc->metric.total_inodes);
truncate_inode_pages_final(&inode->i_data); truncate_inode_pages_final(&inode->i_data);
clear_inode(inode); clear_inode(inode);
...@@ -558,7 +564,6 @@ void ceph_evict_inode(struct inode *inode) ...@@ -558,7 +564,6 @@ void ceph_evict_inode(struct inode *inode)
* caps in i_snap_caps. * caps in i_snap_caps.
*/ */
if (ci->i_snap_realm) { if (ci->i_snap_realm) {
struct ceph_mds_client *mdsc = ceph_sb_to_mdsc(inode->i_sb);
if (ceph_snap(inode) == CEPH_NOSNAP) { if (ceph_snap(inode) == CEPH_NOSNAP) {
struct ceph_snap_realm *realm = ci->i_snap_realm; struct ceph_snap_realm *realm = ci->i_snap_realm;
dout(" dropping residual ref to snap realm %p\n", dout(" dropping residual ref to snap realm %p\n",
......
...@@ -192,11 +192,23 @@ int ceph_metric_init(struct ceph_client_metric *m) ...@@ -192,11 +192,23 @@ int ceph_metric_init(struct ceph_client_metric *m)
m->total_metadatas = 0; m->total_metadatas = 0;
m->metadata_latency_sum = 0; m->metadata_latency_sum = 0;
atomic64_set(&m->opened_files, 0);
ret = percpu_counter_init(&m->opened_inodes, 0, GFP_KERNEL);
if (ret)
goto err_opened_inodes;
ret = percpu_counter_init(&m->total_inodes, 0, GFP_KERNEL);
if (ret)
goto err_total_inodes;
m->session = NULL; m->session = NULL;
INIT_DELAYED_WORK(&m->delayed_work, metric_delayed_work); INIT_DELAYED_WORK(&m->delayed_work, metric_delayed_work);
return 0; return 0;
err_total_inodes:
percpu_counter_destroy(&m->opened_inodes);
err_opened_inodes:
percpu_counter_destroy(&m->i_caps_mis);
err_i_caps_mis: err_i_caps_mis:
percpu_counter_destroy(&m->i_caps_hit); percpu_counter_destroy(&m->i_caps_hit);
err_i_caps_hit: err_i_caps_hit:
...@@ -212,6 +224,8 @@ void ceph_metric_destroy(struct ceph_client_metric *m) ...@@ -212,6 +224,8 @@ void ceph_metric_destroy(struct ceph_client_metric *m)
if (!m) if (!m)
return; return;
percpu_counter_destroy(&m->total_inodes);
percpu_counter_destroy(&m->opened_inodes);
percpu_counter_destroy(&m->i_caps_mis); percpu_counter_destroy(&m->i_caps_mis);
percpu_counter_destroy(&m->i_caps_hit); percpu_counter_destroy(&m->i_caps_hit);
percpu_counter_destroy(&m->d_lease_mis); percpu_counter_destroy(&m->d_lease_mis);
......
...@@ -115,6 +115,13 @@ struct ceph_client_metric { ...@@ -115,6 +115,13 @@ struct ceph_client_metric {
ktime_t metadata_latency_min; ktime_t metadata_latency_min;
ktime_t metadata_latency_max; ktime_t metadata_latency_max;
/* The total number of directories and files that are opened */
atomic64_t opened_files;
/* The total number of inodes that have opened files or directories */
struct percpu_counter opened_inodes;
struct percpu_counter total_inodes;
struct ceph_mds_session *session; struct ceph_mds_session *session;
struct delayed_work delayed_work; /* delayed work */ struct delayed_work delayed_work; /* delayed work */
}; };
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment