diff --git a/panel/io/callbacks.py b/panel/io/callbacks.py
index b9c62479ff..d751e3e8df 100644
--- a/panel/io/callbacks.py
+++ b/panel/io/callbacks.py
@@ -197,12 +197,12 @@ def stop(self):
         with param.discard_events(self):
             self.counter = 0
         self._timeout = None
-        if state._is_pyodide:
+        if state._is_pyodide and self._cb:
             self._cb.cancel()
-        elif self._doc:
+        elif self._doc and self._cb:
             if self._doc._session_context:
                 self._doc.callbacks.remove_session_callback(self._cb)
-            else:
+            elif self._cb in self._doc.callbacks.session_callbacks:
                 self._doc.callbacks._session_callbacks.remove(self._cb)
         elif self._cb:
             self._cb.stop()