bibleanalyze.py

"""
Analyze the Gutenberg Project's Bible text into sentences and words.
We consider a verse to be a sentence: starting at a verse number like
1:23 and ending at the next verse or blank line.

Output an n-gram table.
"""

import itertools
import re
import sys

def main():
    # print_tokens()
    # count_bigrams()
    count_ngrams(4)

def print_tokens():
    for token in enum_bible_tokens(sys.stdin):
        print token

def count_ngrams(n):
    d = {}
    state = ()
    for token in enum_bible_tokens(sys.stdin):
        word = token if token == '<S>' else token.lower()
        if n-1 == len(state):
            key = state + (word,)
            d[key] = d.get(key, 0) + 1
            state = state[1:] + (word,)
        else:
            state = state + (word,)
    for words, count in d.iteritems():
        sys.stdout.write('%s\t%d\n' % (' '.join(words), count))

def enum_bible_tokens(lines):
    for para in enum_paragraphs(lines):
        for verse in enum_verses(para):
            yield '<S>'
            for token in verse:
                yield token

def enum_paragraphs(lines):
    para = []
    for line in lines:
        line = line.strip()
        if line:
            para.append(line)
        elif para:
            yield '\n'.join(para)
            para = []
    if para:
        yield '\n'.join(para)

def enum_verses(paragraph):
    versenum = r'\d+:\d+'
    tokens = re.findall(versenum + r"|['\w]+", paragraph)
    groups = itertools.groupby(tokens, re.compile(versenum).match)
    first_group = True
    for is_versenum, verse_tokens in groups:
        if not first_group:
            if not is_versenum:
                yield [t.strip("'") for t in verse_tokens]
        first_group = False

if __name__ == '__main__':
    main()