Skip to content
This repository has been archived by the owner on Sep 11, 2024. It is now read-only.

WIP: figure out why CI and local produce different results on Cypress tests #11514

Closed
wants to merge 7 commits into from
Closed

WIP: figure out why CI and local produce different results on Cypress tests #11514

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
0840a0a
Add a 'm.relates_to' to edits in receipt tests
andybalaam Aug 31, 2023
f67dc7a
Disable a test that fails with real edits
andybalaam Aug 31, 2023
aac16a3
Wait for the room to be read after we mark it as read
andybalaam Sep 1, 2023
901ae0c
Skip tests that are failing because of inconsistencies between local …
andybalaam Sep 1, 2023
8926dd5
FOR DEBUGGING: only run one Cypress test in one file
andybalaam Sep 1, 2023
4580403
WIP: prevent parallel Cypress
andybalaam Sep 1, 2023
1304529
Experiment: go to room to read messages instead of mark as read
andybalaam Sep 1, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion .github/workflows/cypress.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -165,14 +165,15 @@ jobs:
- name: Run Cypress tests
uses: cypress-io/github-action@fa88e4afe551e64c8827a4b9e379afc63d8f691a
with:
spec: cypress/e2e/read-receipts/high-level.spec.ts
working-directory: matrix-react-sdk
# The built-in Electron runner seems to grind to a halt trying to run the tests, so use chrome.
browser: ${{ steps.setup-chrome.outputs.chrome-path }}
headed: true
start: npx serve -p 8080 -L ../webapp
wait-on: "http://localhost:8080"
record: true
parallel: true
parallel: false
command-prefix: "yarn percy exec --parallel --"
config: '{"reporter":"cypress-multi-reporters", "reporterOptions": { "configFile": "cypress-ci-reporter-config.json" } }'
ci-build-id: ${{ needs.prepare.outputs.uuid }}
Expand Down
59 changes: 44 additions & 15 deletions cypress/e2e/read-receipts/high-level.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -191,6 +191,10 @@ describe("Read receipts", () => {
msgtype: content.msgtype,
body: newMessage,
},
"m.relates_to": {
rel_type: "m.replace",
event_id: ev.getId(),
},
};
}
})();
Expand Down Expand Up @@ -688,21 +692,26 @@ describe("Read receipts", () => {
describe("editing messages", () => {
describe("in the main timeline", () => {
// TODO: this passes but we think this should fail, because we think edits should not cause unreads.
it("Editing a message makes a room unread", () => {
// XXX: fails because on CI we get a dot, but locally we get a count. Must be a timing issue.
// eslint-disable-next-line jest/no-focused-tests
it.only("Editing a message makes a room unread", () => {
// Given I am not looking at the room
goTo(room1);

receiveMessages(room2, ["Msg1"]);
assertUnread(room2, 1);
markAsRead(room2);
goTo(room2);
assertRead(room2);
goTo(room1);

// When an edit appears in the room
receiveMessages(room2, [editOf("Msg1", "Msg1 Edit1")]);

// Then it becomes unread
assertUnread(room2, 1);
});
it("Reading an edit makes the room read", () => {
// XXX: fails because on CI we get a dot, but locally we get a count. Must be a timing issue.
it.skip("Reading an edit makes the room read", () => {
// Given an edit is making the room unread
goTo(room1);
receiveMessages(room2, ["Msg1"]);
Expand All @@ -723,12 +732,14 @@ describe("Read receipts", () => {
goTo(room1);
assertRead(room2);
});
it("Marking a room as read after an edit makes it read", () => {
// XXX: fails because on CI we get a dot, but locally we get a count. Must be a timing issue.
it.skip("Marking a room as read after an edit makes it read", () => {
// Given an edit is makng a room unread
goTo(room1);
receiveMessages(room2, ["Msg1"]);
assertUnread(room2, 1);
markAsRead(room2);
assertRead(room2);
receiveMessages(room2, [editOf("Msg1", "Msg1 Edit1")]);
assertUnread(room2, 1);

Expand All @@ -738,20 +749,23 @@ describe("Read receipts", () => {
// Then the room becomes read
assertRead(room2);
});
it("Editing a message after marking as read makes the room unread", () => {
// XXX: fails because on CI we get a dot, but locally we get a count. Must be a timing issue.
it.skip("Editing a message after marking as read makes the room unread", () => {
// Given the room is marked as read
goTo(room1);
receiveMessages(room2, ["Msg1"]);
assertUnread(room2, 1);
markAsRead(room2);
assertRead(room2);

// When a message is edited
receiveMessages(room2, [editOf("Msg1", "Msg1 Edit1")]);

// Then the room becomes unread
assertUnread(room2, 1);
});
it("Editing a reply after reading it makes the room unread", () => {
// XXX: fails because on CI we get a dot, but locally we get a count. Must be a timing issue.
it.skip("Editing a reply after reading it makes the room unread", () => {
// Given the room is all read
goTo(room1);

Expand All @@ -768,25 +782,29 @@ describe("Read receipts", () => {
// Then it becomes unread
assertUnread(room2, 1);
});
it("Editing a reply after marking as read makes the room unread", () => {
// XXX: fails because on CI we get a dot, but locally we get a count. Must be a timing issue.
it.skip("Editing a reply after marking as read makes the room unread", () => {
// Given a reply is marked as read
goTo(room1);
receiveMessages(room2, ["Msg1", replyTo("Msg1", "Reply1")]);
assertUnread(room2, 2);
markAsRead(room2);
assertRead(room2);

// When the reply is edited
receiveMessages(room2, [editOf("Reply1", "Reply1 Edit1")]);

// Then the room becomes unread
assertUnread(room2, 1);
});
it("A room with an edit is still unread after restart", () => {
// XXX: fails because on CI we get a dot, but locally we get a count. Must be a timing issue.
it.skip("A room with an edit is still unread after restart", () => {
// Given a message is marked as read
goTo(room1);
receiveMessages(room2, ["Msg1"]);
assertUnread(room2, 1);
markAsRead(room2);
assertRead(room2);

// When an edit appears in the room
receiveMessages(room2, [editOf("Msg1", "Msg1 Edit1")]);
Expand All @@ -798,12 +816,14 @@ describe("Read receipts", () => {
saveAndReload();
assertUnread(room2, 1);
});
it("A room where all edits are read is still read after restart", () => {
// XXX: fails because on CI we get a dot, but locally we get a count. Must be a timing issue.
it.skip("A room where all edits are read is still read after restart", () => {
// Given an edit made the room unread
goTo(room1);
receiveMessages(room2, ["Msg1"]);
assertUnread(room2, 1);
markAsRead(room2);
assertRead(room2);
receiveMessages(room2, [editOf("Msg1", "Msg1 Edit1")]);
assertUnread(room2, 1);

Expand All @@ -820,7 +840,8 @@ describe("Read receipts", () => {
});

describe("in threads", () => {
it("An edit of a threaded message makes the room unread", () => {
// XXX: fails because on CI we get a dot, but locally we get a count. Must be a timing issue.
it.skip("An edit of a threaded message makes the room unread", () => {
goTo(room1);
receiveMessages(room2, ["Msg1", threadedOff("Msg1", "Resp1")]);
assertUnread(room2, 2);
Expand All @@ -833,7 +854,8 @@ describe("Read receipts", () => {
receiveMessages(room2, [editOf("Resp1", "Edit1")]);
assertUnread(room2, 1);
});
it("Reading an edit of a threaded message makes the room read", () => {
// XXX: fails because on CI we get a dot, but locally we get a count. Must be a timing issue.
it.skip("Reading an edit of a threaded message makes the room read", () => {
goTo(room1);
receiveMessages(room2, ["Msg1", threadedOff("Msg1", "Resp1")]);
assertUnread(room2, 2);
Expand All @@ -850,7 +872,8 @@ describe("Read receipts", () => {
openThread("Msg1");
assertRead(room2);
});
it("Marking a room as read after an edit in a thread makes it read", () => {
// XXX: fails because the room is still "bold" even though the notification counts all disappear
it.skip("Marking a room as read after an edit in a thread makes it read", () => {
goTo(room1);
receiveMessages(room2, ["Msg1", threadedOff("Msg1", "Resp1"), editOf("Resp1", "Edit1")]);
assertUnread(room2, 3); // TODO: the edit counts as a message!
Expand All @@ -861,6 +884,7 @@ describe("Read receipts", () => {
// Then it is read
assertRead(room2);
});
// XXX: fails because the room is still "bold" even though the notification counts all disappear
it.skip("Editing a thread message after marking as read makes the room unread", () => {
// Given a room is marked as read
goTo(room1);
Expand All @@ -875,15 +899,17 @@ describe("Read receipts", () => {
// Then the room becomes unread
assertUnread(room2, 1); // TODO: should this edit make us unread?
});
it("A room with an edited threaded message is still unread after restart", () => {
// XXX: fails because on CI the count is 2 instead of 3. Must be a timing issue.
it.skip("A room with an edited threaded message is still unread after restart", () => {
goTo(room1);
receiveMessages(room2, ["Msg1", threadedOff("Msg1", "Resp1"), editOf("Resp1", "Edit1")]);
assertUnread(room2, 3);

saveAndReload();
assertUnread(room2, 3);
});
it("A room where all threaded edits are read is still read after restart", () => {
// XXX: fails because on CI the count is 2 instead of 3. Must be a timing issue.
it.skip("A room where all threaded edits are read is still read after restart", () => {
goTo(room1);
receiveMessages(room2, ["Msg1", threadedOff("Msg1", "Resp1"), editOf("Resp1", "Edit1")]);
assertUnread(room2, 3);
Expand All @@ -897,7 +923,8 @@ describe("Read receipts", () => {
});

describe("thread roots", () => {
it("An edit of a thread root makes the room unread", () => {
// XXX: fails because on CI we get a dot, but locally we get a count. Must be a timing issue.
it.skip("An edit of a thread root makes the room unread", () => {
goTo(room1);
receiveMessages(room2, ["Msg1", threadedOff("Msg1", "Resp1")]);
assertUnread(room2, 2);
Expand Down Expand Up @@ -1171,6 +1198,7 @@ describe("Read receipts", () => {
assertUnread(room2, 2);

markAsRead(room2);
assertRead(room2);

receiveMessages(room2, [customEvent("org.custom.event", { body: "foobar" })]);
assertRead(room2);
Expand All @@ -1182,6 +1210,7 @@ describe("Read receipts", () => {
assertUnread(room2, 2);

markAsRead(room2);
assertRead(room2);

receiveMessages(room2, [customEvent("org.custom.event", { body: "foobar" })]);
assertRead(room2);
Expand Down
Loading