1+ #!/usr/bin/env python3
2+
3+ import argparse
4+ import os
5+ import json
6+ import sqlite3
7+ from tabulate import tabulate
8+
9+ def parse_log_line (line ):
10+ try :
11+ return json .loads (line )
12+ except json .JSONDecodeError :
13+ return None
14+
15+ def extract_query_data (data ):
16+ if data ["msg" ] == "Slow query" and "attr" in data and "queryHash" in data ["attr" ] and data ["attr" ]["queryHash" ]:
17+ return {
18+ "hash" : data ["attr" ]["queryHash" ],
19+ "durationMillis" : data ["attr" ]["durationMillis" ],
20+ "ns" : data ["attr" ]["ns" ],
21+ "planSummary" : data ["attr" ]["planSummary" ],
22+ "command" : data ["attr" ]["command" ] if "command" in data ["attr" ] else None
23+ }
24+ return None
25+
26+ def create_or_update_result (result , query_data ):
27+ hash_key = query_data ["hash" ]
28+ result ["durationMillis_" + hash_key ] = result .get ("durationMillis_" + hash_key , 0 ) + query_data ["durationMillis" ]
29+ result ["count_" + hash_key ] = result .get ("count_" + hash_key , 0 ) + 1
30+ result .setdefault ("ns_" + hash_key , query_data ["ns" ])
31+ result .setdefault ("planSummary_" + hash_key , []).append (query_data ["planSummary" ])
32+ result ["avgDurationMillis_" + hash_key ] = result ["durationMillis_" + hash_key ] / result ["count_" + hash_key ] if result ["durationMillis_" + hash_key ] and result ["count_" + hash_key ] > 0 else 0
33+ if query_data ["command" ] and "command_" + hash_key not in result :
34+ result ["command_" + hash_key ] = query_data ["command" ]
35+
36+ def process_slow_log (data , db , limit , char_limit , count , query_condition ):
37+ hashes = set ()
38+ result = {}
39+
40+ for line in data :
41+ parsed_data = parse_log_line (line )
42+ if parsed_data :
43+ query_data = extract_query_data (parsed_data )
44+ if query_data :
45+ hash_key = query_data ["hash" ]
46+ if hash_key not in hashes :
47+ hashes .add (hash_key )
48+ create_or_update_result (result , query_data )
49+
50+ if os .path .exists (db ):
51+ os .remove (db )
52+ print (f"Old database file { db } )
53+
54+ connection = sqlite3 .connect (db )
55+ cursor = connection .cursor ()
56+
57+ cursor .execute ('''
58+ CREATE TABLE IF NOT EXISTS results (
59+ hash TEXT PRIMARY KEY,
60+ durationMillis INTEGER,
61+ count INTEGER,
62+ avgDurationMillis REAL,
63+ ns STRING,
64+ planSummary String,
65+ command STRING
66+ )
67+ ''' )
68+
69+ for hash_key in hashes :
70+ cursor .execute ('''
71+ INSERT OR REPLACE INTO results (hash, durationMillis, count, avgDurationMillis, ns, planSummary, command)
72+ VALUES (?, ?, ?, ?, ?, ?, ?)
73+ ''' , (hash_key , result .get ("durationMillis_" + hash_key , 0 ), result .get ("count_" + hash_key , 0 ), result .get ("avgDurationMillis_" + hash_key , 0 ), result .get ("ns_" + hash_key , '' ),
74+ str (result .get ("planSummary_" + hash_key , '' )), str (result .get ("command_" + hash_key , '' ))))
75+ connection .commit ()
76+
77+ cursor .execute (f"PRAGMA table_info(results);" )
78+ columns = cursor .fetchall ()
79+ column_names = [column_info [1 ] for column_info in columns ]
80+
81+ cursor .execute (f'''
82+ SELECT hash, durationMillis, count, avgDurationMillis, ns, SUBSTR(planSummary, 1, { char_limit }
83+ SUBSTR(command, 1, { char_limit }
84+ FROM results WHERE count >= { count } { query_condition }
85+ ORDER BY avgDurationMillis DESC LIMIT { limit }
86+ ''' )
87+
88+ rows = cursor .fetchall ()
89+ table_data = [column_names ] + list (rows )
90+
91+ print (tabulate (table_data , headers = "firstrow" , tablefmt = "fancy_grid" ))
92+ connection .close ()
93+
94+ def print_sql_info (db , limit , char_limit , count , query_condition ):
95+ print (f'sqlite3 { db } )
96+ print ('.mode column' )
97+ print (f"SELECT hash, durationMillis, count, avgDurationMillis, ns, SUBSTR(planSummary, 1, { char_limit }
98+ f"SUBSTR(command, 1, { char_limit } { count } { query_condition } { limit } )
99+ print ("SELECT hash, durationMillis, count, avgDurationMillis, ns, planSummary, command FROM results ORDER BY avgDurationMillis DESC;" )
100+ exit ()
101+
102+ def main ():
103+ parser = argparse .ArgumentParser (description = "Process MongoDB slow log file" )
104+
105+ parser .add_argument ("log" , nargs = "?" , default = "/var/log/mongod.log" , help = "Path to the mongodb log file (default: /var/log/mongod.log)" )
106+ parser .add_argument ("--db" , default = "./mongo_slow_logs.sql" , help = "Path to the SQLite database file (default: ./mongodb-slow-log.sql)" )
107+ parser .add_argument ("--limit" , default = 10 , type = int , help = "Limit the number of rows in SQL output (default: 10)" )
108+ parser .add_argument ("--char-limit" , default = 100 , type = int , help = "Limit the number of characters in SQL strings output (default: 100)" )
109+ parser .add_argument ("--count" , default = 1 , type = int , help = "Filter queries that appear less than this count in the log (default: 1)" )
110+ parser .add_argument ("--collscan" , action = "store_true" , help = "Filter queries with COLLSCAN in the results (default: no filters)" )
111+ parser .add_argument ("--sql" , action = "store_true" , help = "Print useful SQL information and exit" )
112+
113+ args = parser .parse_args ()
114+
115+ query_condition = ' AND planSummary LIKE \' %COLLSCAN%\' ' if args .collscan else ''
116+
117+ if args .sql :
118+ print_sql_info (args .db , args .limit , args .char_limit , args .count , query_condition )
119+
120+ try :
121+ with open (args .log , "r" ) as log_file :
122+ process_slow_log (log_file , args .db , args .limit , args .char_limit , args .count , query_condition )
123+
124+ except FileNotFoundError :
125+ print (f"The file '{ args .log } )
126+ except Exception as e :
127+ print (f"An error occurred: { e } )
128+
129+ if __name__ == "__main__" :
130+ main ()
0 commit comments