ksol
/
dmarc-analyzer
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Heading towards report sending

This commit is contained in:
László Károlyi 2021-11-03 18:33:29 +01:00
parent 94d8748fd8
commit d229e1b2bf
Signed by: karolyi
GPG Key ID: 2DCAF25E55735BFE
1 changed files with 97 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

144
analyze.py
View File

@ -1,6 +1,7 @@
#!/usr/bin/env python
from argparse import ArgumentParser, Namespace
from collections import namedtuple
from datetime import datetime
from email import message_from_bytes
from email.header import decode_header
@ -17,14 +18,29 @@ from zipfile import ZipFile, is_zipfile
from yaml import CLoader as Loader
from yaml import load
_INSERTED_FIELDS = (
'checked_at, org_name, domain, report_id, unixtime_start, unixtime_end, '
'offending_ip, count, failed_types')
_FETCHED_FIELDS = f'rowid, {_INSERTED_FIELDS}'
LogRow = namedtuple(typename='LogRow', field_names=_FETCHED_FIELDS)
_CFG_TEMPLATE = """\
sqlite_path: dmarc-analyzer.db
# Imap account where the aggregate reports are stored in
imap:
ssl_host: mail.example.com
host: mail.example.com
username: imap-username
password: my-super-secret-password
folder_path: INBOX
# Host settings for sending email reports
smtp:
host: mail.example.com
port: 587
login: sender-login
password: password
"""
_DESCRIPTION = (
'DMARC Report analyzer and reporter. '
'The default mode is to analyze what\'s in the IMAP folder.')
_STARTTIME = time()
@ -40,6 +56,43 @@ class NotAReport(DmarcReporterBase):
'Raised when the passed email is not a DMARC report.'
def _get_sql_connection(config: dict) -> Connection:
'Return an sqlite `Connection` object from the settings.'
return connect(database=Path(config['sqlite_path']).absolute())
def _get_loaded_cfg(parsed_args: Namespace) -> dict:
with Path(parsed_args.cfg_file).absolute().open('r') as fd:
data = load(stream=fd, Loader=Loader)
return data
def _init_cfg(parsed_args: Namespace):
'Initialize a config file template.'
path_cfg = Path(parsed_args.cfg_file).absolute()
with path_cfg.open('w') as fd:
fd.write(_CFG_TEMPLATE)
print((
f'Config file template written to {path_cfg}. Please edit it '
'before running this program.'))
def _init_db(parsed_args: Namespace, config: dict):
'Initialize a blank DB.'
conn = connect(database=Path(config['sqlite_path']).absolute())
cursor = conn.cursor()
cursor.execute('DROP TABLE IF EXISTS reports')
cursor.execute(
'CREATE TABLE reports (checked_at INTEGER, org_name TEXT, '
'domain TEXT, report_id TEXT, unixtime_start INTEGER, '
'unixtime_end INTEGER, offending_ip TEXT, count INTEGER, '
'failed_types TEXT)')
conn.commit()
conn.close()
print(
'Database initialized. You can now run the program without --init-db')
class XmlParser(object):
'Parse one report here.'
@ -52,10 +105,8 @@ class XmlParser(object):
def _parse_header(self):
'Parse data headers.'
date_range = self._root.find(path='./report_metadata/date_range')
self._datetime_start = datetime.utcfromtimestamp(int(
date_range.find(path='begin').text))
self._datetime_end = datetime.utcfromtimestamp(int(
date_range.find(path='end').text))
self._unixtime_start = int(date_range.find(path='begin').text)
self._unixtime_end = int(date_range.find(path='end').text)
self._org_name = self._root.find(
path='./report_metadata/org_name').text
self._domain = self._root.find(path='./policy_published/domain').text
@ -65,11 +116,10 @@ class XmlParser(object):
def _note_failed_records(self, ip: str, failed: list, count: int):
'Add the failed records to the sqlite DB.'
self._cursor.execute(
'INSERT INTO reports(checked_at, org_name, domain, report_id, '
'datetime_start, datetime_end, offending_ip, count, failed_types) '
'VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)', (
f'INSERT INTO reports({_INSERTED_FIELDS}) VALUES (?, ?, ?, ?, ?, '
'?, ?, ?, ?)', (
_STARTTIME, self._org_name, self._domain, self._report_id,
self._datetime_start, self._datetime_end, ip, count,
self._unixtime_start, self._unixtime_end, ip, count,
', '.join(failed)))
self._sql_conn.commit()
@ -88,12 +138,14 @@ class XmlParser(object):
for record in self._root.findall(path='record'):
self._parse_record(record=record)
def start(self):
'Start parsing.'
def process(self):
'Start processing.'
self._root = XML(text=self._content)
self._parse_header()
print('\n', self._datetime_start.strftime(format='%c'))
print(self._datetime_end.strftime(format='%c'))
print('\n', datetime.fromtimestamp(
self._unixtime_start).strftime(format='%c'))
print(datetime.fromtimestamp(self._unixtime_end).strftime(format='%c'))
print(self._org_name, self._domain, self._report_id)
self._parse_records()
@ -104,8 +156,7 @@ class ImapHandler(object):
def __init__(self, config: dict):
self._config = config
self._sql_conn = connect(
database=Path(config['sqlite_path']).absolute())
self._sql_conn = _get_sql_connection(config=config)
def _get_subject(self, email: Message) -> str:
'Extract and return the subject.'
@ -183,50 +234,43 @@ class ImapHandler(object):
parser = XmlParser(
content=content_item, config=self._config,
sql_conn=self._sql_conn)
parser.start()
parser.process()
def start(self):
def process(self):
'Start processing.'
with IMAP4_SSL(host=self._config['imap']['ssl_host']) as self._conn:
with IMAP4_SSL(host=self._config['imap']['host']) as self._conn:
self._login_and_run()
self._sql_conn.close()
def _get_loaded_cfg(parsed_args: Namespace):
with Path(parsed_args.cfg_file).absolute().open('r') as fd:
data = load(stream=fd, Loader=Loader)
return data
class ReportSender(object):
'Sending reports per email.'
def __init__(self, config: dict):
self._config = config
self._sql_conn = _get_sql_connection(config=config)
self._cursor = self._sql_conn.cursor()
def _init_cfg(parsed_args: Namespace):
'Initialize a config file template.'
path_cfg = Path(parsed_args.cfg_file).absolute()
with path_cfg.open('w') as fd:
fd.write(_CFG_TEMPLATE)
print((
f'Config file template written to {path_cfg}. Please edit it '
'before running this program.'))
def _read_records(self):
'Read records from the DB.'
self._cursor.execute(f'SELECT {_FETCHED_FIELDS} FROM reports')
rows = list()
while True:
row = self._cursor.fetchone()
if row is None:
break
rows.append(LogRow(*row))
return rows
def _init_db(parsed_args: Namespace, config: dict):
'Initialize a blank DB.'
conn = connect(database=Path(config['sqlite_path']).absolute())
cursor = conn.cursor()
cursor.execute('DROP TABLE IF EXISTS reports')
cursor.execute(
'CREATE TABLE reports (checked_at INTEGER, org_name TEXT, '
'domain TEXT, report_id TEXT, datetime_start INTEGER, '
'datetime_end INTEGER, offending_ip TEXT, count INTEGER, '
'failed_types TEXT)')
conn.commit()
conn.close()
print(
'Database initialized. You can now run the program without --init-db')
def process(self):
'Start processing.'
rows = self._read_records()
# self._filter_and_sort_rows()
def main():
'Startup of program.'
parser = ArgumentParser(description='DMARC Report analyzer')
parser = ArgumentParser(description=_DESCRIPTION)
parser.add_argument(
'-c', '--config', required=True, help='Configuration file path',
dest='cfg_file')
@ -236,14 +280,20 @@ def main():
parser.add_argument(
'--recreate', action='store_true', required=False,
help='Recreate the config file')
parser.add_argument(
'-r', '--report', action='store_true', required=False,
help='Report from the collected logs')
parsed_args = parser.parse_args()
if parsed_args.recreate:
return _init_cfg(parsed_args=parsed_args)
config = _get_loaded_cfg(parsed_args=parsed_args)
if parsed_args.init_db:
return _init_db(parsed_args=parsed_args, config=config)
if parsed_args.report:
report_sender = ReportSender(config=config)
return report_sender.process()
handler = ImapHandler(config=config)
handler.start()
handler.process()
if __name__ == '__main__':