diff --git a/src/widgets/chat.rs b/src/widgets/chat.rs
index 9b141cb..f40b1a1 100644
--- a/src/widgets/chat.rs
+++ b/src/widgets/chat.rs
@@ -833,7 +833,7 @@ fn make_message_list(
     }
 
     // apply our read receipts
-    Message::apply_receipts(&mut messages, receipts);
+    Message::apply_receipts(&mut messages, &mut receipts.get_all());
 
     // update senders to friendly names
     messages.iter_mut().for_each(|m| m.update_senders(members));
diff --git a/src/widgets/message.rs b/src/widgets/message.rs
index 7e3c6c1..a29d607 100644
--- a/src/widgets/message.rs
+++ b/src/widgets/message.rs
@@ -1,5 +1,7 @@
 use chrono::TimeZone;
+use log::info;
 use std::cell::Cell;
+use std::collections::BinaryHeap;
 use std::iter;
 use std::time::{Duration, SystemTime};
 
@@ -27,7 +29,7 @@ use tui::style::{Color, Style};
 use tui::text::{Span, Spans};
 use tui::widgets::ListItem;
 
-use super::receipts::Receipts;
+use super::receipts::Receipt;
 
 // A Message is a line in the chat window; what a user would generally
 // consider a "message". It has reactions, edits, and is generally in a state
@@ -328,10 +330,28 @@ impl Message {
         reply_result
     }
 
-    pub fn apply_receipts(messages: &mut [Message], receipts: &Receipts) {
-        for message in messages.iter_mut() {
-            if let Some(usernames) = receipts.get(&message.id) {
-                message.receipts = usernames.clone()
+    /// Given a binary heap (priority queue) of Receipts, run through the
+    /// the messages, popping off receipts and attaching them. This way we
+    /// only show a single receipt per user, on the latest message they have
+    /// read.
+    pub fn apply_receipts(messages: &mut [Message], heap: &mut BinaryHeap<Receipt>) {
+        for message in messages.iter_mut().rev() {
+            Message::apply_receipts(&mut message.replies, heap);
+
+            loop {
+                if let Some(candidate) = heap.peek() {
+                    if candidate.timestamp > &message.sent {
+                        message
+                            .receipts
+                            .push(Username::new(candidate.user_id.clone()));
+
+                        heap.pop();
+                    } else {
+                        break;
+                    }
+                } else {
+                    return;
+                }
             }
         }
     }
diff --git a/src/widgets/receipts.rs b/src/widgets/receipts.rs
index aa3cc80..021cf18 100644
--- a/src/widgets/receipts.rs
+++ b/src/widgets/receipts.rs
@@ -1,59 +1,64 @@
-use ruma::OwnedUserId;
-use std::collections::HashMap;
+use ruma::{MilliSecondsSinceUnixEpoch, OwnedUserId};
+use std::collections::{btree_map::Entry, BTreeMap, BinaryHeap};
 
-use ruma::{
-    events::receipt::{ReceiptEventContent, ReceiptType},
-    OwnedEventId,
-};
-
-use crate::matrix::username::Username;
+use ruma::events::receipt::{ReceiptEventContent, ReceiptType};
 
 /// A place to put and update read receipts.
 pub struct Receipts {
-    events: HashMap<OwnedEventId, Vec<Username>>,
+    markers: BTreeMap<OwnedUserId, MilliSecondsSinceUnixEpoch>,
     ignore: OwnedUserId,
 }
 
 impl Receipts {
     pub fn new(ignore: OwnedUserId) -> Self {
         Receipts {
-            events: HashMap::default(),
+            markers: BTreeMap::default(),
             ignore,
         }
     }
 
-    pub fn get(&self, event_id: &OwnedEventId) -> Option<&Vec<Username>> {
-        self.events.get(event_id)
-    }
-
     pub fn apply_event(&mut self, event: &ReceiptEventContent) {
-        for (event_id, types) in event.iter() {
+        for types in event.values() {
             if let Some(user_ids) = types.get(&ReceiptType::Read) {
-                for user_id in user_ids.keys() {
-                    self.apply_event_and_user(event_id, user_id);
+                for (user_id, receipt) in user_ids.iter() {
+                    if let Some(ts) = &receipt.ts {
+                        self.apply_timestamp_and_user(ts, user_id);
+                    }
                 }
             }
         }
     }
 
-    fn apply_event_and_user(&mut self, event_id: &OwnedEventId, user_id: &OwnedUserId) {
+    pub fn get_all(&self) -> BinaryHeap<Receipt> {
+        let mut heap = BinaryHeap::with_capacity(self.markers.len());
+
+        heap.extend(self.markers.iter().map(|(k, v)| Receipt {
+            timestamp: v,
+            user_id: k,
+        }));
+
+        heap
+    }
+
+    fn apply_timestamp_and_user(
+        &mut self,
+        timestamp: &MilliSecondsSinceUnixEpoch,
+        user_id: &OwnedUserId,
+    ) {
         if user_id == &self.ignore {
             return;
         }
 
-        // wipe out any previous receipts for this user
-        for (_, usernames) in self.events.iter_mut() {
-            usernames.retain(|u| &u.id != user_id)
-        }
-
-        // add our receipt
-        self.events
-            .entry(event_id.clone())
-            .or_insert_with(|| Vec::with_capacity(1))
-            .push(Username::new(user_id.clone()));
-
-        // and clean up any now-empty vectors
-        self.events.retain(|_, value| !value.is_empty());
+        match self.markers.entry(user_id.clone()) {
+            Entry::Vacant(entry) => {
+                entry.insert(*timestamp);
+            }
+            Entry::Occupied(mut entry) => {
+                if timestamp > entry.get() {
+                    *entry.get_mut() = *timestamp
+                }
+            }
+        };
     }
 
     pub fn get_senders(event: &ReceiptEventContent) -> Vec<&OwnedUserId> {
@@ -70,3 +75,9 @@ impl Receipts {
         ids
     }
 }
+
+#[derive(Eq, PartialEq, Ord, PartialOrd, Debug)]
+pub struct Receipt<'a> {
+    pub timestamp: &'a MilliSecondsSinceUnixEpoch,
+    pub user_id: &'a OwnedUserId,
+}