Commit 46535e9f authored by Vincent Pelletier's avatar Vincent Pelletier

Add stats on most frequent user agents.

To find which user agents should be ignored (bots, monitoring, ...).
parent a7e0a644
...@@ -79,6 +79,7 @@ US_PER_S = 10 ** 6 ...@@ -79,6 +79,7 @@ US_PER_S = 10 ** 6
N_SLOWEST = 20 N_SLOWEST = 20
N_ERROR_URL = 10 N_ERROR_URL = 10
N_REFERRER_PER_ERROR_URL = 5 N_REFERRER_PER_ERROR_URL = 5
N_USER_AGENT = 20
ITEMGETTER0 = itemgetter(0) ITEMGETTER0 = itemgetter(0)
ITEMGETTER1 = itemgetter(1) ITEMGETTER1 = itemgetter(1)
APDEX_TOLERATING_COEF = 4 APDEX_TOLERATING_COEF = 4
...@@ -285,6 +286,7 @@ class GenericSiteStats(object): ...@@ -285,6 +286,7 @@ class GenericSiteStats(object):
self.error_url_count = defaultdict(partial(defaultdict, list)) self.error_url_count = defaultdict(partial(defaultdict, list))
self.url_apdex = defaultdict(partial(APDEXStats, threshold, getDuration)) self.url_apdex = defaultdict(partial(APDEXStats, threshold, getDuration))
self.apdex = defaultdict(partial(APDEXStats, threshold, getDuration)) self.apdex = defaultdict(partial(APDEXStats, threshold, getDuration))
self.user_agent_counter = Counter()
def rescale(self, convert, getDuration): def rescale(self, convert, getDuration):
for status, date_dict in self.status.iteritems(): for status, date_dict in self.status.iteritems():
...@@ -310,6 +312,7 @@ class GenericSiteStats(object): ...@@ -310,6 +312,7 @@ class GenericSiteStats(object):
if self.error_detail and statusIsError(status): if self.error_detail and statusIsError(status):
# XXX: can eat memory if there are many errors on many different urls # XXX: can eat memory if there are many errors on many different urls
self.error_url_count[status][url].append(match.group('referer')) self.error_url_count[status][url].append(match.group('referer'))
self.user_agent_counter[match.group('agent')] += 1
def getApdexData(self): def getApdexData(self):
return getDataPoints(self.apdex) return getDataPoints(self.apdex)
...@@ -334,6 +337,12 @@ class GenericSiteStats(object): ...@@ -334,6 +337,12 @@ class GenericSiteStats(object):
append(data.asHTML(self.threshold)) append(data.asHTML(self.threshold))
append('<td class="text">%s</td></tr>' % unquoteToHtml(url, encoding)) append('<td class="text">%s</td></tr>' % unquoteToHtml(url, encoding))
append('</table>') append('</table>')
append('<h2>User agents</h2><table class="stats"><tr><th>hits</th>'
'<th>user agent</th></tr>')
for user_agent, hit in self.user_agent_counter.most_common(N_USER_AGENT):
# XXX: s/escape/unquoteToHtml/ ?
append('<tr><td>%s</td><td class="text">%s</td></tr>' % (hit, escape(user_agent)))
append('</table>')
column_set = set() column_set = set()
filtered_status = defaultdict(partial(defaultdict, int)) filtered_status = defaultdict(partial(defaultdict, int))
for status, date_dict in self.status.iteritems(): for status, date_dict in self.status.iteritems():
...@@ -416,6 +425,7 @@ class GenericSiteStats(object): ...@@ -416,6 +425,7 @@ class GenericSiteStats(object):
status = result.status status = result.status
for status_code, date_dict in state['status'].iteritems(): for status_code, date_dict in state['status'].iteritems():
status[status_code].update(date_dict) status[status_code].update(date_dict)
result.user_agent_counter.update(state['user_agent_counter'])
return result return result
def asJSONState(self): def asJSONState(self):
...@@ -426,6 +436,7 @@ class GenericSiteStats(object): ...@@ -426,6 +436,7 @@ class GenericSiteStats(object):
'url_apdex': _APDEXDateDictAsJSONState(self.url_apdex), 'url_apdex': _APDEXDateDictAsJSONState(self.url_apdex),
'apdex': _APDEXDateDictAsJSONState(self.apdex), 'apdex': _APDEXDateDictAsJSONState(self.apdex),
'status': self.status, 'status': self.status,
'user_agent_counter': self.user_agent_counter,
} }
def accumulateFrom(self, other): def accumulateFrom(self, other):
...@@ -445,6 +456,7 @@ class GenericSiteStats(object): ...@@ -445,6 +456,7 @@ class GenericSiteStats(object):
date_dict = status[status_code] date_dict = status[status_code]
for status_date, count in other_date_dict.iteritems(): for status_date, count in other_date_dict.iteritems():
date_dict[status_date] += count date_dict[status_date] += count
self.user_agent_counter.update(other.user_agent_counter)
class ERP5SiteStats(GenericSiteStats): class ERP5SiteStats(GenericSiteStats):
""" """
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment