Commit 1612fa08 authored by Patrik Dufresne's avatar Patrik Dufresne

WIP

parent ce6e563c
Pipeline #489 failed with stages
in 48 seconds
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright (C) 2018 Patrik Dufresne Service Logiciel inc. All rights reserved.
# Patrik Dufresne Service Logiciel PROPRIETARY/CONFIDENTIAL.
# Use is subject to license terms.
import argparse
import os
import osec.rest
def main():
parser = argparse.ArgumentParser(description='Open Source Event Correlation')
parser.add_argument('action', metavar='ACTION', type=str, nargs='?',
help='Define the components to be started: rest, enrich')
# Rest
parser.add_argument('--host', default=os.environ.get('OSEC_HOST', '127.0.0.1'),
help='Host name to listen for RESTful API. Default to: 127.0.0.1. OSEC_HOST')
parser.add_argument('--port', type=int, default=os.environ.get('OSEC_PORT', '8081'),
help='Port to listen for RESTful API. Default to: 8081 OSEC_PORT')
parser.add_argument('--secret', default=os.environ.get('OSEC_SECRET', 'osec'),
help='Secret used as a password for the RESTful API. Default: osec. OSEC_SECRET')
args = parser.parse_args()
if args.action == 'rest':
osec.rest.run(args.host, args.port, args.secret)
elif args.action == 'enrich':
pass
else:
print('unknown action: %s' % args.action)
exit(1)
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright (C) 2018 Patrik Dufresne Service Logiciel inc. All rights reserved.
# Patrik Dufresne Service Logiciel PROPRIETARY/CONFIDENTIAL.
# Use is subject to license terms.
"""
Created on Sep 26, 2018
'''
Created on Sep 28, 2018
@author: patrik dufresne
"""
@author: ikus060
'''
from osec.app import app
from osec import main
if __name__ == '__main__': # Script executed directly?
main()
app.main()
\ No newline at end of file
'''
Created on Sep 26, 2018
@author: ikus060
'''
class Alarm(dict):
'''
Representation of an alarm.
'''
def __init__(self):
'''
Constructor
'''
'''
Created on Oct 4, 2018
@author: ikus060
'''
from osec.alarms.models import Alarm, PerceivedSeverity
from osec.app import app, updates_topic, alarms_topic
from osec.updates.models import UpdateType
@app.agent(updates_topic)
async def alarms_tags(stream):
"""
This agents is responsible to enriched the alarms with tags
"""
'''
Created on Sep 28, 2018
@author: ikus060
'''
from enum import Enum
import faust
class PerceivedSeverity(Enum):
"""
An enumerated value that describes the severity of the
Alert Indication from the notifier\'s point of view:
1 - Other, by CIM convention, is used to indicate that
the Severity\'s value can be found in the OtherSeverity
property.
3 - Degraded/Warning should be used when its appropriate
to let the user decide if action is needed.
4 - Minor should be used to indicate action is needed,
but the situation is not serious at this time.
5 - Major should be used to indicate action is needed
NOW.
6 - Critical should be used to indicate action is needed
NOW and the scope is broad (perhaps an imminent outage
to a critical resource will result).
7 - Fatal/NonRecoverable should be used to indicate an
error occurred, but it\'s too late to take remedial
action.
2 and 0 - Information and Unknown (respectively) follow
common usage. Literally, the AlertIndication is purely
informational or its severity is simply unknown.
"""
cleared = -1
unknown = 0
other = 1
information = 2
warning = 3
minor = 4
major = 5
critical = 6
fatal = 7
class Alarm(faust.Record):
"""
This model represent an instance of an alarm.
"""
id: str
"""
The alarm id. Generated from <orgid>:<localid>
"""
managedElement: str = ""
perceivedSeverity: PerceivedSeverity = PerceivedSeverity.critical
# TODO complete this.
@property
def orgid(self):
"""
Return the <orgid> from the alarm id.
See id
"""
return self.id.split(':')[0]
@property
def localid(self):
"""
Return the <localid> from the alarm id.
See id
"""
return self.id.split(':')[1]
def update(self, new_alarm):
"""
Update this alarm with new alarm information. Return true if any fields was update.
"""
changed = False
for k, v in new_alarm.__dict__.items():
default_value = getattr(Alarm, k).default
current_value = getattr(self, k)
if v != default_value and v != current_value:
setattr(self, k, v)
changed = True
return changed
'''
Created on Oct 3, 2018
@author: ikus060
'''
import pytest
from osec.alarms.models import Alarm
def test_alarm_orgid():
alarm = Alarm('test:1')
assert alarm.orgid == 'test'
assert alarm.localid == '1'
'''
Created on Sep 28, 2018
@author: ikus060
'''
import logging
import faust
from osec.alarms.models import Alarm
from osec.updates.models import Update
app = faust.App(
'osec',
version=1,
autodiscover=True,
origin='osec',
broker='kafka://veras.patrikdufresne.com:9092',
)
updates_topic = app.topic(
'updates_topic',
key_type=str,
value_type=Update,
)
alarms_topic = app.topic(
'alarms_topic',
key_type=str,
value_type=Alarm,
)
def main():
# TODO Configure logging
logging.basicConfig(level=logging.DEBUG)
app.main()
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright (C) 2018 Patrik Dufresne Service Logiciel inc. All rights reserved.
# Patrik Dufresne Service Logiciel PROPRIETARY/CONFIDENTIAL.
# Use is subject to license terms.
'''
Created on Sep 26, 2018
@author: ikus060
'''
import os
import cherrypy
from kafka.producer import KafkaProducer
from osec.rest.alarms import Alarms
USERS = {'osec': os.environ.get('OSEC_SECRET', 'osec')}
# TODO Must allow creation of users.
def validate_password(realm, username, password):
if username in USERS and USERS[username] == password:
return True
return False
@cherrypy.tools.auth_basic(realm='osec-api', checkpassword=validate_password)
class Root(object):
alarms = Alarms()
@cherrypy.expose
def index(self):
return 'OSEC RESTful API'
def run(host, port, secret):
cherrypy.config.update({
'server.socket_host': host,
'server.socket_port': port,
})
cherrypy.quickstart(Root(), '/')
'''
Created on Sep 26, 2018
@author: ikus060
'''
import cherrypy
@cherrypy.expose
class Alarms(object):
@cherrypy.expose
@cherrypy.tools.json_out()
@cherrypy.tools.json_in()
def index(self):
if cherrypy.request.method in ['PUT', 'POST']:
return self.alarms_post()
else:
return self.alarms_get()
def alarms_get(self):
return ""
def alarms_post(self):
cherrypy.log('post alarms')
return ""
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright (C) 2018 Patrik Dufresne Service Logiciel inc. All rights reserved.
# Patrik Dufresne Service Logiciel PROPRIETARY/CONFIDENTIAL.
# Use is subject to license terms.
'''
Created on Sep 26, 2018
@author: ikus060
'''
from base64 import b64encode
import unittest
import cherrypy
from cherrypy.test import helper
from osec.alarm import Alarm
import osec.rest
import json
class TestRest(helper.CPWebCase):
def setup_server():
cherrypy.tree.mount(osec.rest.Root())
setup_server = staticmethod(setup_server)
interactive = False
def test_alarm_post(self):
a = Alarm()
a['identifier'] = "A::B::C"
# Make the query
body = json.dumps(a)
headers = [
("Authorization", "Basic " + b64encode(b"osec:osec").decode('ascii')),
("Content-Type", "application/json"),
("Content-Length", str(len(body)))
]
self.getPage('/alarms/', method='POST', headers=headers, body=body)
self.assertStatus(200)
pass
if __name__ == "__main__":
# import sys;sys.argv = ['', 'Test.testName']
unittest.main()
'''
Created on Sep 28, 2018
@author: ikus060
'''
import logging
from osec.alarms.models import Alarm, PerceivedSeverity
from osec.app import app, updates_topic, alarms_topic
from osec.updates.models import UpdateType
alarm_table = app.Table(
'alarm_table',
key_type=str,
value_type=Alarm,
help="""This table contains the original alarms receive by the monitoring tool without enrichedment."""
)
alarm_status = app.Table(
'alarm_status',
key_type=str,
value_type=int,
help="""Identify list of alarm status to be resync."""
)
@app.agent(updates_topic)
async def updates_aggregator(stream):
"""Aggregate the update updates messages into alarms."""
async for update in stream:
try:
assert update.id, "update is missing `id`"
assert update.type, "update is missing `type`"
assert update.alarm, "update is missing `alarm`"
logging.debug("processing update [%s][%s]" % (update.id, update.type))
if update.type == UpdateType.begin:
# Mark all the associated alarm as 'dirty'
for k, v in alarm_table.items():
if k.startswith(update.alarm.orgid + ':'):
alarm_status[k] = 1
yield None
elif update.type == UpdateType.end:
# When receiving a end. we need to delete all the alarm that wasn't received.
for k, v in alarm_status.items():
if k.startswith(update.alarm.orgid + ':') and v:
alarm = alarm_table.get(k)
if alarm:
logging.info("delete obsolete [%s]" % alarm.id)
alarm.perceivedSeverity = PerceivedSeverity.cleared
alarm_table[alarm.id] = alarm
await alarms_topic.send(alarm.id, alarm)
yield None
elif update.type == UpdateType.creation:
# Need to replace the previous data.
i = update.alarm.id
alarm_table[i] = update.alarm
alarm_status[i] = 0
await alarms_topic.send(i, update.alarm)
yield update.alarm
elif update.type == UpdateType.modification:
alarm = alarm_table.get(update.alarm.id)
if alarm:
logging.info("update [%s]" % alarm.id)
# Update the alarm properties
alarm.update(update.alarm)
# Store it in table and topic.
alarm_table[alarm.id] = alarm
alarm_status[alarm.id] = 0
await alarms_topic.send(alarm.id, alarm)
yield alarm
elif update.type == UpdateType.deletion:
alarm = alarm_table.get(update.alarm.id)
if alarm:
logging.info("delete [%s]" % alarm.id)
alarm.perceivedSeverity = PerceivedSeverity.cleared
alarm_table[alarm.id] = alarm
await alarms_topic.send(alarm.id, alarm)
yield alarm
except:
logging.exception('fail processing: %s' % update)
yield None
'''
Created on Sep 28, 2018
@author: ikus060
'''
import faust
from osec.alarms.models import Alarm
from enum import Enum
class UpdateType(Enum):
"""
Define the alarm update type.
begin: define the beginning of the resynch
end: define the end of the resynch
create: create or replace the alarm
delete: remove the alarm (clear the alarm)
update: update the alarm
"""
begin = 0
end = 1
creation = 2
deletion = 3
modification = 4
class Update(faust.Record):
id: str
type: UpdateType
alarm: Alarm
'''
Created on Oct 2, 2018
@author: ikus060
'''
import pytest
from osec.alarms.models import Alarm, PerceivedSeverity
from osec.app import app
from osec.updates.agents import updates_aggregator, \
alarm_table
from osec.updates.models import UpdateType, Update
async def _send(update_type, alarm_id, **kwargs):
"""
Send message to the update aggregator agent.
"""
async with updates_aggregator.test_context() as agent: # @UndefinedVariable
alarm = Alarm(
id=alarm_id,
**kwargs
)
update = Update(id=alarm_id, type=update_type, alarm=alarm)
event = await agent.put(update)
return agent.results[event.message.offset]
@pytest.fixture()
def test_app():
app.finalize()
app.conf.store = 'memory://'
app.flow_control.resume()
return app
@pytest.mark.asyncio()
async def test_updates_creation(test_app):
"""
Sending a creation expect the alarm to be created.
"""
alarm = await _send(UpdateType.creation, 'test:1', perceivedSeverity=PerceivedSeverity.warning, managedElement='')
assert alarm.id == 'test:1'
assert alarm.perceivedSeverity == PerceivedSeverity.warning
@pytest.mark.asyncio()
async def test_updates_creation_creation(test_app):
"""
Sending a creation for the same alarm should replace the previous one.
"""
alarm = await _send(UpdateType.creation, 'test:1', perceivedSeverity=PerceivedSeverity.warning, managedElement='entity1')
assert alarm.id == 'test:1'
assert alarm.perceivedSeverity == PerceivedSeverity.warning
assert alarm.managedElement == 'entity1'
alarm = await _send(UpdateType.creation, 'test:1', perceivedSeverity=PerceivedSeverity.critical)
assert alarm.id == 'test:1'
assert alarm.perceivedSeverity == PerceivedSeverity.critical
assert alarm.managedElement == ''
@pytest.mark.asyncio()
async def test_updates_modification(test_app):
"""
Sending a modification should update the alarm field.
"""
# Send creation
alarm = await _send(UpdateType.creation, 'test:1', managedElement='entity1')
assert alarm.id == 'test:1'
assert alarm.managedElement == 'entity1'
# Send modification
alarm = await _send(UpdateType.modification, 'test:1', perceivedSeverity=PerceivedSeverity.critical)
assert alarm.id == 'test:1'
assert alarm.managedElement == 'entity1'
assert alarm.perceivedSeverity == PerceivedSeverity.critical
@pytest.mark.asyncio()
async def test_updates_modification_not_exists(test_app):
"""
Sending a modification should update the alarm field.
"""
# Send creation
alarm = await _send(UpdateType.creation, 'test:1', managedElement='entity1')
assert alarm.id == 'test:1'
assert alarm.managedElement == 'entity1'
# Send modification
alarm = await _send(UpdateType.modification, 'test:2', perceivedSeverity=PerceivedSeverity.critical)
assert alarm is None
@pytest.mark.asyncio()
async def test_updates_creation_deletion(test_app):
"""
Sending creation then a deletion.
Expect the alarm to be cleared.
"""
alarm = await _send(UpdateType.creation, 'test:1', managedElement='entity1')
assert alarm.id == 'test:1'
assert alarm.managedElement == 'entity1'
alarm = await _send(UpdateType.deletion, 'test:1')
assert alarm.id == 'test:1'
assert alarm.perceivedSeverity == PerceivedSeverity.cleared
@pytest.mark.asyncio()
async def test_updates_deletion_not_exists(test_app):
"""
Sending creation then a deletion.
Expect the alarm to be cleared.
"""
alarm = await _send(UpdateType.deletion, 'test:2')
assert alarm is None
@pytest.mark.asyncio()
async def test_updates_creation_begin_create_end(test_app):
"""
Sending a begin / end message is clearing the memory.
"""
alarm = await _send(UpdateType.creation, 'test:1', managedElement='entity1')
assert alarm.id == 'test:1'
assert alarm.managedElement == 'entity1'
assert alarm_table['test:1']
await _send(UpdateType.begin, 'test:begin')
alarm = await _send(UpdateType.creation, 'test:2', managedElement='entity2')
assert alarm.id == 'test:2'
assert alarm.managedElement == 'entity2'
assert alarm_table['test:1']
assert alarm_table['test:2']
await _send(UpdateType.end, 'test:end')
assert alarm_table['test:1']
assert alarm_table['test:1'].perceivedSeverity == PerceivedSeverity.cleared
assert alarm_table['test:2']
......@@ -17,7 +17,7 @@ setuptools.setup(
url='http://www.patrikdufresne.com/',
packages=['osec'],
include_package_data=True,
python_requires='>=3.6',
python_requires='~=3.6',
setup_requires=[
"setuptools_scm",
],
......@@ -29,7 +29,8 @@ setuptools.setup(
tests_require=[
"mock>=1.3.0",
"pytest",
"pytest-asyncio",
"nose",
],
entry_points={"console_scripts": ["osec = osec:run"], },
entry_points={"console_scripts": [ 'osec = osec.app:main', ] },
)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment