Merge pull request #20 from dbarrosop/v0.17

V0.17

docs/api/api_v1.0.rst

@@ -89,6 +89,7 @@ _________
 * **limit_prefixes**: Optional. Number of top prefixes to retrieve.
 * **net_masks**: Optional. List of prefix lengths to filter in or out.
 * **exclude_net_masks**: Optional. If set to any value it will return prefixes with a prefix length not included in net_masks. If set to 0 it will return only prefixes with a prefix length included in net_masks. Default is 0.
+* **filter_proto**: Optional. If you don't set it you will get both IPv4 and IPv6 prefixes. If you set it to 4 you will get only IPv4 prefixes. Otherwise, if you set it to 6 you will get IPv6 prefixes.
 
 Returns
 _______
@@ -642,3 +643,61 @@ ________
 ::
 
     http://127.0.0.1:5000/api/v1.0/pmacct/raw_bgp?date=2015-07-16T11:00:01
+
+/api/v1.0/pmacct/purge_bgp
+==========================
+
+GET
+---
+
+Description
+___________
+
+Deletes all the BGP data that is older than ``older_than``.
+
+
+Arguments
+_________
+
+* **older_than**: Mandatory. Datetime in unicode string following the format ``'%Y-%m-%dT%H:%M:%S'``.
+
+Returns
+_______
+
+The list of files containing BGP data that was deleted.
+
+Examples
+________
+
+::
+
+    http://127.0.0.1:5000/api/v1.0/pmacct/purge_bgp?older_than=2015-07-29T13:00:01
+
+/api/v1.0/pmacct/purge_flows
+============================
+
+GET
+---
+
+Description
+___________
+
+Deletes all the flows that are older than ``older_than``.
+
+
+Arguments
+_________
+
+* **older_than**: Mandatory. Datetime in unicode string following the format ``'%Y-%m-%dT%H:%M:%S'``.
+
+Returns
+_______
+
+The flows that were deleted.
+
+Examples
+________
+
+::
+
+    http://127.0.0.1:5000/api/v1.0/pmacct/purge_flows?older_than=2015-07-29T13:00:01

sir/agent.py

@@ -185,5 +185,16 @@ def pmacct_data_api_get_raw_bgp():
 def pmacct_data_api_get_bgp_prefixes():
     return jsonify(sir.pmacct_data.api.get_bgp_prefixes(request))
 
+
+@app.route('/api/v1.0/pmacct/purge_bgp', methods=['GET'])
+def pmacct_data_api_purge_bgp():
+    return jsonify(sir.pmacct_data.api.purge_bgp(request))
+
+
+@app.route('/api/v1.0/pmacct/purge_flows', methods=['GET'])
+def pmacct_data_api_purge_flows():
+    return jsonify(sir.pmacct_data.api.purge_flows(request))
+
+
 if __name__ == '__main__':
     app.run(app.config['BIND_IP'], app.config['PORT'])

sir/analytics/api.py

@@ -10,19 +10,22 @@ def top_prefixes(request):
     limit_prefixes = int(request.args.get('limit_prefixes', 0))
     net_masks = request.args.get('net_masks', '')
     exclude_net_masks = request.args.get('exclude_net_masks', False)
+    filter_proto = request.args.get('filter_proto', None)
 
     result = db.aggregate_per_prefix(
         start_time, end_time,
         limit=limit_prefixes,
         net_masks=net_masks,
-        exclude_net_masks=exclude_net_masks)
+        exclude_net_masks=exclude_net_masks,
+        filter_proto=filter_proto)
 
     parameters = {
         'limit_prefixes': limit_prefixes,
         'start_time': start_time,
         'end_time': end_time,
         'net_masks': net_masks,
         'exclude_net_masks': exclude_net_masks,
+        'filter_proto': filter_proto,
     }
     return sir.helpers.api.build_api_response(result, error=False, **parameters)
 

sir/helpers/FSHelper.py

@@ -2,7 +2,7 @@
 from datetime import datetime
 import json
 import ipaddress
-
+import os
 
 class FSHelper:
     def __init__(self, base_path='./data'):
@@ -92,3 +92,21 @@ def find_prefixes_asn(self, asn, date, originate_only=False):
                             if asn in as_path:
                                 prefixes[n].append(data)
         return prefixes
+
+    def purge_bgp(self, older_than):
+        older_than = datetime.strptime(older_than, '%Y-%m-%dT%H:%M:%S')
+        deleted_files = list()
+
+        for n in self.neighbors:
+            for d in self.dates:
+                if older_than > d:
+                    file_name = '{}/bgp-{}-{}.txt'.format(
+                        self.base_path,
+                        n.replace('.', '_'),
+                        d.strftime('%Y_%m_%dT%H_%M_%S')
+                    )
+                    if os.path.isfile(file_name):
+                        os.remove(file_name)
+                        deleted_files.append(file_name)
+
+        return deleted_files

sir/helpers/SQLite3Helper.py

@@ -40,13 +40,17 @@ def _execute_query(self, query, args=(), commit=False):
             raise Exception('The following query failed:\n%s' % query)
         return result
 
-    def aggregate_per_prefix(self, start_time, end_time, limit=0, net_masks='', exclude_net_masks=False):
+    def aggregate_per_prefix(self, start_time, end_time, limit=0, net_masks='', exclude_net_masks=False, filter_proto=None):
         """ Given a time range aggregates bytes per prefix.
 
             Args:
                 start_time: A string representing the starting time of the time range
                 end_time: A string representing the ending time of the time range
                 limit: An optional integer. If it's >0 it will limit the amount of prefixes returned.
+                filter_proto: Can be:
+                    - None: Returns both ipv4 and ipv6
+                    - 4: Returns only ipv4
+                    - 6: Retruns only ipv6
 
             Returns:
                 A list of prefixes sorted by sum_bytes. For example:
@@ -64,14 +68,22 @@ def aggregate_per_prefix(self, start_time, end_time, limit=0, net_masks='', excl
         elif exclude_net_masks:
             net_mask_filter = 'AND mask_dst NOT IN ({})'.format(net_masks)
 
+        if filter_proto is None:
+            proto_filter = ''
+        elif int(filter_proto) == 4:
+            proto_filter = 'AND ip_dst NOT LIKE "%:%"'
+        elif int(filter_proto) == 6:
+            proto_filter = 'AND ip_dst LIKE "%:%"'
+
         query = ''' SELECT ip_dst||'/'||mask_dst as key, SUM(bytes) as sum_bytes, as_dst
                     from acct
                     WHERE
                     datetime(stamp_updated) BETWEEN datetime(?) AND datetime(?, "+1 second")
                     {}
+                    {}
                     GROUP by ip_dst,mask_dst
                     ORDER BY SUM(bytes) DESC
-                '''.format(net_mask_filter)
+                '''.format(net_mask_filter, proto_filter)
 
         if limit > 0:
             query += 'LIMIT %d' % limit
@@ -211,3 +223,20 @@ def get_flows(self, start_time, end_time):
                             ;
                 '''
         return self._execute_query(query, [start_time, end_time])
+
+    def purge_flows(self, older_than):
+        query = ''' SELECT ip_dst, mask_dst, bytes, packets, as_dst, stamp_updated
+                            FROM acct
+                            WHERE
+                            datetime(stamp_updated) < datetime(?)
+                            ;
+                '''
+        purged_flows = self._execute_query(query, [older_than])
+        query = ''' DELETE
+                            FROM acct
+                            WHERE
+                            datetime(stamp_updated) < datetime(?)
+                            ;
+                '''
+        self._execute_query(query, [older_than], commit=True)
+        return purged_flows

sir/pmacct_data/api.py

@@ -22,8 +22,23 @@ def get_bgp_prefixes(request):
     bgp_prefixes = fs.get_bgp_prefixes(date)
     return sir.helpers.api.build_api_response(bgp_prefixes, error=False, date=date)
 
+
 def get_raw_bgp(request):
     fs = getattr(g, 'fs')
     date = request.args.get('date')
     bgp_prefixes = fs.get_raw_bgp(date)
     return sir.helpers.api.build_api_response(bgp_prefixes, error=False, date=date)
+
+
+def purge_bgp(request):
+    fs = getattr(g, 'fs')
+    older_than = request.args.get('older_than')
+    purged_bgp_tables = fs.purge_bgp(older_than)
+    return sir.helpers.api.build_api_response(purged_bgp_tables, error=False, older_than=older_than)
+
+
+def purge_flows(request):
+    db = getattr(g, 'db')
+    older_than = request.args.get('older_than')
+    purged_flows = db.purge_flows(older_than)
+    return sir.helpers.api.build_api_response(purged_flows, error=False, older_than=older_than)