-
+ 5614D6523B1512656953C12732DB5DAA56B49288251B879427A9B8E33DA7DB95847E441D2AD007896182C5ACB27F0ED808B072A25C12B4789CF85CC186E68F68logotron/eat_dump.py(0 . 0)(1 . 104)
542 #!/usr/bin/python
543
544 ##############################################################################
545 import psycopg2, psycopg2.extras
546 import psycopg2.extensions
547 psycopg2.extensions.register_type(psycopg2.extensions.UNICODE)
548 psycopg2.extensions.register_type(psycopg2.extensions.UNICODEARRAY)
549 import re
550 import time
551 import datetime
552 from datetime import datetime
553 import sys
554 import os
555
556 # Debug Knob
557 DB_DEBUG = False
558 ##############################################################################
559
560 ##############################################################################
561 db = psycopg2.connect("dbname=nsalog user=nsabot") ## CHANGE THESE
562
563 def close_db():
564 db.close()
565
566 def exec_db(query, args=()):
567 cur = db.cursor(cursor_factory=psycopg2.extras.RealDictCursor)
568 if (DB_DEBUG): print "query: '{0}'".format(query)
569 if (DB_DEBUG): print "args: '{0}'".format(args)
570 if (DB_DEBUG): print "EXEC:"
571 cur.execute(query, args)
572
573 def rollback_db():
574 cur = db.cursor(cursor_factory=psycopg2.extras.RealDictCursor)
575 cur.execute("ROLLBACK")
576 db.commit()
577
578 def commit_db():
579 cur = db.cursor(cursor_factory=psycopg2.extras.RealDictCursor)
580 db.commit()
581
582 ##############################################################################
583
584 # Eat individual line of a Phf-style log dump
585 def eat_logline(line, chan, era):
586 match = re.search("(\d+)\;(\d+)\;([^;]+)\;(.*$)", line)
587 if match:
588 g = match.groups()
589 self_speak = False
590
591 try:
592 idx = int(g[0]) # Serial Number of Log Line
593 time = int(g[1]) # Unix Epochal Time of Log Line
594 except Exception, e:
595 print("Malformed Line! '" + line +"' ! : " + e)
596 close_db()
597 exit(1)
598
599 speaker = g[2] # Name of Speaker
600 payload = g[3] # Payload (remainder of line)
601
602 ## If spoken line is of form "* user ..." :
603 if speaker == "*":
604 spl = payload.split(' ', 1)
605 speaker = spl[0]
606 payload = spl[1]
607 self_speak = True
608
609 ## Put in DB:
610 try:
611 exec_db('''insert into loglines (idx, t, chan, era, speaker, self, payload)
612 values (%s, %s, %s, %s, %s, %s, %s) ; ''',
613 [int(idx), datetime.fromtimestamp(time), str(chan), int(era), str(speaker),
614 bool(self_speak), str(payload)])
615 commit_db()
616 except psycopg2.IntegrityError as e:
617 rollback_db()
618 print "Dupe Ignored, Idx=", idx
619 else:
620 print("Malformed Line! '" + line +"' !")
621 close_db()
622 exit(1)
623
624
625 # Eat Phf-style log dump at given path
626 def eat_dump(path, chan, era):
627 with open(path) as fp:
628 for line in fp:
629 eat_logline(line, chan, era)
630
631
632 ##############################################################################
633
634 if (len(sys.argv) == 4):
635 logdump = sys.argv[1] # Path to Phf-style log dump
636 chan = sys.argv[2] # Chan Name
637 era = sys.argv[3] # Era (integer)
638 # Eat:
639 eat_dump(logdump, chan, era)
640 close_db()
641 else:
642 print "Usage: ./eat_dump LOGFILE CHAN ERA"
643 exit(0)
644
645 ##############################################################################