Commit d7bf976c authored by Alain Takoudjou's avatar Alain Takoudjou Committed by Tristan Cavelier

monitor: improve changes for log rss

monitor: do not check list of rss item with start_date, only consider limit
parent 3df4f460
...@@ -139,7 +139,7 @@ mode = 0644 ...@@ -139,7 +139,7 @@ mode = 0644
recipe = hexagonit.recipe.download recipe = hexagonit.recipe.download
url = ${:_profile_base_location_}/${:filename} url = ${:_profile_base_location_}/${:filename}
download-only = true download-only = true
md5sum = f880212b392a937a637f6eb244f0e6b5 md5sum = 91a1a46bcdb7024035a734482c8c7bc9
filename = logTools.py filename = logTools.py
mode = 0644 mode = 0644
...@@ -147,7 +147,7 @@ mode = 0644 ...@@ -147,7 +147,7 @@ mode = 0644
recipe = hexagonit.recipe.download recipe = hexagonit.recipe.download
url = ${:_profile_base_location_}/${:filename} url = ${:_profile_base_location_}/${:filename}
download-only = true download-only = true
md5sum = 49eb40b5515db17bf9c63d87162ad4ad md5sum = e723f929b3afd44924a55910954a1a06
filename = errorlog2rss.py.in filename = errorlog2rss.py.in
mode = 0644 mode = 0644
......
...@@ -61,25 +61,25 @@ for name, log_path in log_file_list.items(): ...@@ -61,25 +61,25 @@ for name, log_path in log_file_list.items():
filter_with=log_filter, start_date=start_date, filter_with=log_filter, start_date=start_date,
date_format=date_format) date_format=date_format)
if item_list: if item_list:
start = 0 items = item_list
size = len(item_list) size = len(item_list)
if limit and size > limit: if limit and size > limit:
start = size - limit - 1 items = item_list[size-limit-1:-1]
print "Found %s log entries matching date upper than %s..." % (size, start_date) print "%s: Found %s log entries matching date upper than %s..." % (name,
size, start_date)
logTools.insertRssDb(db_path, item_list[start:-1], name) logTools.insertRssDb(db_path, items, name)
print "%s items inserted into database..." % limit
print "Generating RSS entries with %s items..." % limit print "Generating RSS entries with maximum %s items..." % limit
logTools.generateRSS(db_path, name, rss_path, start_date, base_link+name, limit=limit) logTools.generateRSS(db_path, name, rss_path, base_link+name, limit=limit)
else: else:
print "No log entries with date upper than %s..." % start_date print "%s: No log entries with date upper than %s..." % (name, start_date)
try: try:
# Remove olders entries from database # Remove olders entries from database
logTools.truncateRssDb(db_path, to_date) logTools.truncateRssDb(db_path, to_date)
except Exception,e: except Exception,e:
print "Database not exist!! ", str(e) print "Database not exist!! \n", str(e)
...@@ -40,7 +40,7 @@ def getZopeParser(): ...@@ -40,7 +40,7 @@ def getZopeParser():
serverDateTime = Combine(integer + "-" + integer + "-" + integer + " " + serverDateTime = Combine(integer + "-" + integer + "-" + integer + " " +
integer + ":" + integer + ":" + integer + "," + integer) integer + ":" + integer + ":" + integer + "," + integer)
status = Word(string.uppercase, max=7, min=3) status = Word(string.uppercase, max=7, min=3)
word = Word( alphas+nums+"@._-" ) word = Word( alphas+nums+"@._-:/#" )
message = Regex(".*") message = Regex(".*")
bnf = serverDateTime.setResultsName("timestamp") + \ bnf = serverDateTime.setResultsName("timestamp") + \
status.setResultsName("statusCode") + \ status.setResultsName("statusCode") + \
...@@ -64,21 +64,24 @@ def parseLog(path, parserbnf, method, filter_with="ERROR", start_date="", date_f ...@@ -64,21 +64,24 @@ def parseLog(path, parserbnf, method, filter_with="ERROR", start_date="", date_f
log_result = [] log_result = []
if not date_format: if not date_format:
date_format = "%Y-%m-%d %H:%M:%S,%f" date_format = "%Y-%m-%d %H:%M:%S,%f"
skip_entry = False
with open(path, 'r') as logfile: with open(path, 'r') as logfile:
index = 0 index = 0
for line in logfile: for line in logfile:
regex = method(line) regex = method(line)
if not regex: if not regex:
if index == 0 or line.strip() == "------": if index == 0 or line.strip() == "------" or skip_entry:
continue continue
# Add this line to log content # Add this line to log content, if entry is not skipped
log_result[index - 1]['content'] += ("\n" + line) log_result[index - 1]['content'] += ("\n" + line)
else: else:
try: try:
fields = parserbnf.parseString(line) fields = parserbnf.parseString(line)
if filter_with and not fields.statusCode == filter_with: skip_entry = filter_with and not fields.statusCode == filter_with
if skip_entry:
continue continue
if start_date and regex.group() < start_date: skip_entry = start_date and regex.group() < start_date
if skip_entry:
continue continue
log_result.append(dict(datetime=datetime.datetime.strptime( log_result.append(dict(datetime=datetime.datetime.strptime(
fields.timestamp , date_format), fields.timestamp , date_format),
...@@ -89,7 +92,8 @@ def parseLog(path, parserbnf, method, filter_with="ERROR", start_date="", date_f ...@@ -89,7 +92,8 @@ def parseLog(path, parserbnf, method, filter_with="ERROR", start_date="", date_f
content=fields.get('content', fields.title))) content=fields.get('content', fields.title)))
index += 1 index += 1
except Exception: except Exception:
raise continue
#raise
# print "WARNING: Could not parse log line. %s \n << %s >>" % (str(e), line) # print "WARNING: Could not parse log line. %s \n << %s >>" % (str(e), line)
return log_result return log_result
...@@ -123,17 +127,17 @@ def selectRssDb(db_path, rss_name, start_date, limit=0): ...@@ -123,17 +127,17 @@ def selectRssDb(db_path, rss_name, start_date, limit=0):
return rows return rows
return [] return []
def generateRSS(db_path, name, rss_path, start_date, url_link, limit=0): def generateRSS(db_path, name, rss_path, url_link, limit=10):
items = [] items = []
db = sqlite3.connect(db_path) db = sqlite3.connect(db_path)
query = "select name, datetime, status, method, title, url, content from rss_entry " query = "select name, datetime, status, method, title, url, content from rss_entry "
query += "where name=? and datetime>=? order by datetime DESC" query += "where name=? order by datetime DESC"
if limit: if limit:
query += " limit ?" query += " limit ?"
entry_list = db.execute(query, (name, start_date, limit)) entry_list = db.execute(query, (name, limit))
else: else:
entry_list = db.execute(query, (name, start_date)) entry_list = db.execute(query, (name,))
for entry in entry_list: for entry in entry_list:
name, rss_date, status, method, title, url, content = entry name, rss_date, status, method, title, url, content = entry
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment