document.py

import numpy as np


class Topic(object):
    """Basic data structure of topic, contains topic id and
       corresponding probability.
    """

    def __init__(self, tid, prob):
        self.tid = tid  # topic id
        self.prob = prob  # topic probability


class Token(object):
    """Basic storage unit of LDA documents, contains word id
       and corresponding topic.
    """

    def __init__(self, topic, id):
        self.topic = topic
        self.id = id


class Sentence(object):
    """Basic storage unit of SentenceLDA documents, contains word ids
       of the sentence and its corresponding topic id.
    """

    def __init__(self, topic, tokens):
        self.topic = topic
        self.tokens = tokens


class LDADoc(object):
    """The storage structure of LDA model's inference result.
    """

    def __init__(self):
        self._num_topics = None  # Number of topics.
        self._num_accum = None  # Number of accumulated sample rounds.
        self._alpha = None  # Document prior parameter.
        self._tokens = None  # Storage structure of inference results.
        self._topic_sum = None  # Document's topic sum in one round samples.
        self._accum_topic_sum = None  # Accumulated results of topic sum.

    def init(self, num_topics):
        """Initialize the LDADoc according to num_topics.
        """
        self._num_topics = num_topics
        self._num_accum = 0
        self._tokens = []
        self._topic_sum = np.zeros(self._num_topics)
        self._accum_topic_sum = np.zeros(self._num_topics)

    def add_token(self, token):
        """Add new word to current LDADoc.
        Arg:
            token: Token class object.
        """
        assert token.topic >= 0, "Topic %d out of range!" % token.topic
        assert token.topic < self._num_topics, "Topic %d out of range!" % token.topic
        self._tokens.append(token)
        self._topic_sum[token.topic] += 1

    def token(self, index):
        return self._tokens[index]

    def set_topic(self, index, new_topic):
        """Set the index word's topic to new_topic, and update the corresponding
           topic distribution.
        """
        assert new_topic >= 0, "Topic %d out of range!" % new_topic
        assert new_topic < self._num_topics, "Topic %d out of range!" % new_topic
        old_topic = self._tokens[index].topic
        if new_topic == old_topic:
            return
        self._tokens[index].topic = new_topic
        self._topic_sum[old_topic] -= 1
        self._topic_sum[new_topic] += 1

    def set_alpha(self, alpha):
        self._alpha = alpha

    def size(self):
        """Return number of words in LDADoc.
        """
        return len(self._tokens)

    def topic_sum(self, topic_id):
        return self._topic_sum[topic_id]

    def sparse_topic_dist(self, sort=True):
        """Return the topic distribution of documents in sparse format.
           By default, it is sorted according to the topic probability
           under the descending order.
        """
        topic_dist = []
        sum_ = np.sum(self._accum_topic_sum)
        if sum_ == 0:
            return topic_dist
        for i in range(0, self._num_topics):
            if self._accum_topic_sum[i] == 0:
                continue
            topic_dist.append(Topic(i, self._accum_topic_sum[i] * 1.0 / sum_))
        if sort:

            def take_elem(topic):
                return topic.prob

            topic_dist.sort(key=take_elem, reverse=True)
            if topic_dist is None:
                topic_dist = []

        return topic_dist

    def dense_topic_dist(self):
        """Return the distribution of document topics in dense format,
           taking into account the prior parameter alpha.
        """
        dense_dist = np.zeros(self._num_topics)
        if self.size() == 0:
            return dense_dist
        dense_dist = (self._accum_topic_sum * 1.0 / self._num_accum + self._alpha) / (
            self.size() + self._alpha * self._num_topics)
        return dense_dist

    def accumulate_topic_num(self):
        self._accum_topic_sum += self._topic_sum
        self._num_accum += 1


class SLDADoc(LDADoc):
    """Sentence LDA Document, inherited from LDADoc.
       Add add_sentence interface.
    """

    def __init__(self):
        super().__init__()
        self.__sentences = None

    def init(self, num_topics):
        """Initialize the SLDADoc according to num_topics.
        """
        self._num_topics = num_topics
        self.__sentences = []
        self._num_accum = 0
        self._topic_sum = np.zeros(self._num_topics)
        self._accum_topic_sum = np.zeros(self._num_topics)

    def add_sentence(self, sent):
        """Add new sentence to current SLDADoc.
        Arg:
            sent: Sentence class object.
        """
        assert sent.topic >= 0, "Topic %d out of range!" % (sent.topic)
        assert sent.topic < self._num_topics, "Topic %d out of range!" % (sent.topic)
        self.__sentences.append(sent)
        self._topic_sum[sent.topic] += 1

    def set_topic(self, index, new_topic):
        assert new_topic >= 0, "Topic %d out of range!" % (new_topic)
        assert new_topic < self._num_topics, "Topic %d out of range!" % (new_topic)
        old_topic = self.__sentences[index].topic
        if new_topic == old_topic:
            return
        self.__sentences[index].topic = new_topic
        self._topic_sum[old_topic] -= 1
        self._topic_sum[new_topic] += 1

    def size(self):
        """Return number of sentences in SLDADoc.
        """
        return len(self.__sentences)

    def sent(self, index):
        return self.__sentences[index]