From e1d0aadde8a394282ce511defacd7dd96f3ff2d4 Mon Sep 17 00:00:00 2001 From: juligasa <11684004+juligasa@users.noreply.github.com> Date: Mon, 1 Apr 2024 13:50:49 +0200 Subject: [PATCH 1/4] mentions proto definition --- .../genproto/activity/v1alpha/activity.pb.go | 23 +++++++++++++++++-- .../activity/v1alpha/activity_pb.ts | 16 ++++++++++++- proto/activity/v1alpha/activity.proto | 11 ++++++++- proto/activity/v1alpha/go.gensum | 4 ++-- proto/activity/v1alpha/js.gensum | 4 ++-- 5 files changed, 50 insertions(+), 8 deletions(-) diff --git a/backend/genproto/activity/v1alpha/activity.pb.go b/backend/genproto/activity/v1alpha/activity.pb.go index d397518d2..8c188d9cd 100644 --- a/backend/genproto/activity/v1alpha/activity.pb.go +++ b/backend/genproto/activity/v1alpha/activity.pb.go @@ -47,6 +47,15 @@ type ListEventsRequest struct { FilterEventType []string `protobuf:"bytes,5,rep,name=filter_event_type,json=filterEventType,proto3" json:"filter_event_type,omitempty"` // Optional. If we want events only from specific resource IDs FilterResource []string `protobuf:"bytes,6,rep,name=filter_resource,json=filterResource,proto3" json:"filter_resource,omitempty"` + // Optional. If we want to include link events. These blobs (usually documents + // or comments), link (mention) to another resource (currently only account + // mentions supported). We can add these blobs to the feed result by providing a + // list of resources iris we want links to aggregated as a logical OR. + // These link events are also treated as ogical OR when grouped with other filters. + // Unlike filters (users, event_types) that are grouped under a logic AND. + // filter_users(u1 OR u2 ...) AND filter_event_type(et1 OR et2 ...) OR + // add_linked_resource(lr1 OR lr2 ...) + AddLinkedResource []string `protobuf:"bytes,7,rep,name=add_linked_resource,json=addLinkedResource,proto3" json:"add_linked_resource,omitempty"` } func (x *ListEventsRequest) Reset() { @@ -123,6 +132,13 @@ func (x *ListEventsRequest) GetFilterResource() []string { return nil } +func (x *ListEventsRequest) GetAddLinkedResource() []string { + if x != nil { + return x.AddLinkedResource + } + return nil +} + // The response with the list of events. type ListEventsResponse struct { state protoimpl.MessageState @@ -372,7 +388,7 @@ var file_activity_v1alpha_activity_proto_rawDesc = []byte{ 0x63, 0x74, 0x69, 0x76, 0x69, 0x74, 0x79, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x1a, 0x1f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x22, 0xea, 0x01, 0x0a, 0x11, 0x4c, 0x69, 0x73, 0x74, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x52, + 0x22, 0x9a, 0x02, 0x0a, 0x11, 0x4c, 0x69, 0x73, 0x74, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x70, 0x61, 0x67, 0x65, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x08, 0x70, 0x61, 0x67, 0x65, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x70, 0x61, 0x67, 0x65, 0x5f, 0x74, 0x6f, 0x6b, 0x65, @@ -386,7 +402,10 @@ var file_activity_v1alpha_activity_proto_rawDesc = []byte{ 0x03, 0x28, 0x09, 0x52, 0x0f, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x27, 0x0a, 0x0f, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x5f, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x06, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0e, 0x66, - 0x69, 0x6c, 0x74, 0x65, 0x72, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x22, 0x79, 0x0a, + 0x69, 0x6c, 0x74, 0x65, 0x72, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12, 0x2e, 0x0a, + 0x13, 0x61, 0x64, 0x64, 0x5f, 0x6c, 0x69, 0x6e, 0x6b, 0x65, 0x64, 0x5f, 0x72, 0x65, 0x73, 0x6f, + 0x75, 0x72, 0x63, 0x65, 0x18, 0x07, 0x20, 0x03, 0x28, 0x09, 0x52, 0x11, 0x61, 0x64, 0x64, 0x4c, + 0x69, 0x6e, 0x6b, 0x65, 0x64, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x22, 0x79, 0x0a, 0x12, 0x4c, 0x69, 0x73, 0x74, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3b, 0x0a, 0x06, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x63, 0x6f, 0x6d, 0x2e, 0x6d, 0x69, 0x6e, 0x74, 0x74, 0x65, diff --git a/frontend/packages/shared/src/client/.generated/activity/v1alpha/activity_pb.ts b/frontend/packages/shared/src/client/.generated/activity/v1alpha/activity_pb.ts index 354c234be..c06ff841a 100644 --- a/frontend/packages/shared/src/client/.generated/activity/v1alpha/activity_pb.ts +++ b/frontend/packages/shared/src/client/.generated/activity/v1alpha/activity_pb.ts @@ -57,11 +57,24 @@ export class ListEventsRequest extends Message { /** * Optional. If we want events only from specific resource IDs * - * * @generated from field: repeated string filter_resource = 6; */ filterResource: string[] = []; + /** + * Optional. If we want to include link events. These blobs (usually documents + * or comments), link (mention) to another resource (currently only account + * mentions supported). We can add these blobs to the feed result by providing a + * list of resources iris we want links to aggregated as a logical OR. + * These link events are also treated as ogical OR when grouped with other filters. + * Unlike filters (users, event_types) that are grouped under a logic AND. + * filter_users(u1 OR u2 ...) AND filter_event_type(et1 OR et2 ...) OR + * add_linked_resource(lr1 OR lr2 ...) + * + * @generated from field: repeated string add_linked_resource = 7; + */ + addLinkedResource: string[] = []; + constructor(data?: PartialMessage) { super(); proto3.util.initPartial(data, this); @@ -76,6 +89,7 @@ export class ListEventsRequest extends Message { { no: 4, name: "filter_users", kind: "scalar", T: 9 /* ScalarType.STRING */, repeated: true }, { no: 5, name: "filter_event_type", kind: "scalar", T: 9 /* ScalarType.STRING */, repeated: true }, { no: 6, name: "filter_resource", kind: "scalar", T: 9 /* ScalarType.STRING */, repeated: true }, + { no: 7, name: "add_linked_resource", kind: "scalar", T: 9 /* ScalarType.STRING */, repeated: true }, ]); static fromBinary(bytes: Uint8Array, options?: Partial): ListEventsRequest { diff --git a/proto/activity/v1alpha/activity.proto b/proto/activity/v1alpha/activity.proto index 520545042..86aa22c49 100644 --- a/proto/activity/v1alpha/activity.proto +++ b/proto/activity/v1alpha/activity.proto @@ -38,9 +38,18 @@ message ListEventsRequest { repeated string filter_event_type = 5; // Optional. If we want events only from specific resource IDs - // repeated string filter_resource = 6; + // Optional. If we want to include link events. These blobs (usually documents + // or comments), link (mention) to another resource (currently only account + // mentions supported). We can add these blobs to the feed result by providing a + // list of resources iris we want links to aggregated as a logical OR. + // These link events are also treated as ogical OR when grouped with other filters. + // Unlike filters (users, event_types) that are grouped under a logic AND. + // filter_users(u1 OR u2 ...) AND filter_event_type(et1 OR et2 ...) OR + // add_linked_resource(lr1 OR lr2 ...) + repeated string add_linked_resource = 7; + } // The response with the list of events. diff --git a/proto/activity/v1alpha/go.gensum b/proto/activity/v1alpha/go.gensum index cc1b6d02f..3924f28d9 100644 --- a/proto/activity/v1alpha/go.gensum +++ b/proto/activity/v1alpha/go.gensum @@ -1,2 +1,2 @@ -srcs: a89cea1221fc65cb12dbf7fc5e55b92a -outs: b5ff909757a801be9931f17d4669248f +srcs: fc15899e11013c57b0c36297c617270d +outs: 591d305d0ec2a1e29ca02bd073d2ed61 diff --git a/proto/activity/v1alpha/js.gensum b/proto/activity/v1alpha/js.gensum index fb53b44f7..312d0e9fa 100644 --- a/proto/activity/v1alpha/js.gensum +++ b/proto/activity/v1alpha/js.gensum @@ -1,2 +1,2 @@ -srcs: a89cea1221fc65cb12dbf7fc5e55b92a -outs: e30b2acb41c8f20995c6af6df944094c +srcs: fc15899e11013c57b0c36297c617270d +outs: cc2ba6e98262896aef8c423d8efcaed2 From 3cb89d9c30a1e58fab3b0b4d005ceb1e1651078b Mon Sep 17 00:00:00 2001 From: juligasa <11684004+juligasa@users.noreply.github.com> Date: Mon, 1 Apr 2024 14:20:40 +0200 Subject: [PATCH 2/4] wip(daemon): initial mentions filtering --- .../daemon/api/activity/v1alpha/activity.go | 21 +++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/backend/daemon/api/activity/v1alpha/activity.go b/backend/daemon/api/activity/v1alpha/activity.go index 05b55b0cb..93b15b4fa 100644 --- a/backend/daemon/api/activity/v1alpha/activity.go +++ b/backend/daemon/api/activity/v1alpha/activity.go @@ -127,6 +127,23 @@ func (srv *Server) ListEvents(ctx context.Context, req *activity.ListEventsReque } filtersStr += ") AND " } + var linksStr string + if len(req.AddLinkedResource) > 0 { + if len(req.FilterResource) > 0 || len(req.FilterEventType) > 0 { + linksStr += " OR " + } + linksStr += "(" + storage.StructuralBlobsID.String() + " in ('Change', 'Comment') AND " + storage.ResourcesIRI.String() + " IN (" + for i, resource := range req.AddLinkedResource { + if !resourcePattern.MatchString(resource) { + return nil, fmt.Errorf("Invalid link resource format [%s]", resource) + } + if i > 0 { + linksStr += ", " + } + linksStr += "'" + resource + "'" + } + linksStr += ")) AND " + } var ( selectStr = "SELECT " + storage.BlobsID + ", " + storage.StructuralBlobsType + ", " + storage.PublicKeysPrincipal + ", " + storage.ResourcesIRI + ", " + storage.StructuralBlobsTs + ", " + storage.BlobsInsertTime + ", " + storage.BlobsMultihash + ", " + storage.BlobsCodec tableStr = "FROM " + storage.T_StructuralBlobs @@ -143,8 +160,8 @@ func (srv *Server) ListEvents(ctx context.Context, req *activity.ListEventsReque %s %s %s - WHERE %s %s; - `, selectStr, tableStr, joinIDStr, joinpkStr, leftjoinStr, trustedStr, filtersStr, pageTokenStr) + WHERE %s %s %s; + `, selectStr, tableStr, joinIDStr, joinpkStr, leftjoinStr, trustedStr, filtersStr, linksStr, pageTokenStr) var lastBlobID int64 err = sqlitex.Exec(conn, dqb.Str(getEventsStr)(), func(stmt *sqlite.Stmt) error { lastBlobID = stmt.ColumnInt64(0) From 292c278a12bb56bd4aa065372bfdf0df408b8d3a Mon Sep 17 00:00:00 2001 From: juligasa <11684004+juligasa@users.noreply.github.com> Date: Mon, 1 Apr 2024 18:13:41 +0200 Subject: [PATCH 3/4] wip(daemon): include full mentions --- .../daemon/api/activity/v1alpha/activity.go | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/backend/daemon/api/activity/v1alpha/activity.go b/backend/daemon/api/activity/v1alpha/activity.go index 93b15b4fa..1975487af 100644 --- a/backend/daemon/api/activity/v1alpha/activity.go +++ b/backend/daemon/api/activity/v1alpha/activity.go @@ -132,7 +132,8 @@ func (srv *Server) ListEvents(ctx context.Context, req *activity.ListEventsReque if len(req.FilterResource) > 0 || len(req.FilterEventType) > 0 { linksStr += " OR " } - linksStr += "(" + storage.StructuralBlobsID.String() + " in ('Change', 'Comment') AND " + storage.ResourcesIRI.String() + " IN (" + linksStr += "(" + storage.StructuralBlobsType.String() + " in ('Change', 'Comment') AND " + storage.ResourceLinksTarget.String() + " IN (" + + "select " + storage.ResourcesID.String() + " FROM " + storage.T_Resources + " where " + storage.ResourcesIRI.String() + " in (" for i, resource := range req.AddLinkedResource { if !resourcePattern.MatchString(resource) { return nil, fmt.Errorf("Invalid link resource format [%s]", resource) @@ -142,15 +143,16 @@ func (srv *Server) ListEvents(ctx context.Context, req *activity.ListEventsReque } linksStr += "'" + resource + "'" } - linksStr += ")) AND " + linksStr += "))) AND " } var ( - selectStr = "SELECT " + storage.BlobsID + ", " + storage.StructuralBlobsType + ", " + storage.PublicKeysPrincipal + ", " + storage.ResourcesIRI + ", " + storage.StructuralBlobsTs + ", " + storage.BlobsInsertTime + ", " + storage.BlobsMultihash + ", " + storage.BlobsCodec - tableStr = "FROM " + storage.T_StructuralBlobs - joinIDStr = "JOIN " + storage.Blobs.String() + " ON " + storage.BlobsID.String() + "=" + storage.StructuralBlobsID.String() - joinpkStr = "JOIN " + storage.PublicKeys.String() + " ON " + storage.StructuralBlobsAuthor.String() + "=" + storage.PublicKeysID.String() - leftjoinStr = "LEFT JOIN " + storage.Resources.String() + " ON " + storage.StructuralBlobsResource.String() + "=" + storage.ResourcesID.String() - pageTokenStr = storage.BlobsID.String() + " <= :idx AND (" + storage.ResourcesIRI.String() + " NOT IN (SELECT " + storage.DraftsViewResource.String() + " from " + storage.DraftsView.String() + ") OR " + storage.ResourcesIRI.String() + " IS NULL) ORDER BY " + storage.BlobsID.String() + " desc limit :page_size" + selectStr = "SELECT " + storage.BlobsID + ", " + storage.StructuralBlobsType + ", " + storage.PublicKeysPrincipal + ", " + storage.ResourcesIRI + ", " + storage.StructuralBlobsTs + ", " + storage.BlobsInsertTime + ", " + storage.BlobsMultihash + ", " + storage.BlobsCodec + tableStr = "FROM " + storage.T_StructuralBlobs + joinIDStr = "JOIN " + storage.Blobs.String() + " ON " + storage.BlobsID.String() + "=" + storage.StructuralBlobsID.String() + joinpkStr = "JOIN " + storage.PublicKeys.String() + " ON " + storage.StructuralBlobsAuthor.String() + "=" + storage.PublicKeysID.String() + leftjoinResourcesStr = "LEFT JOIN " + storage.Resources.String() + " ON " + storage.StructuralBlobsResource.String() + "=" + storage.ResourcesID.String() + leftjoinLinksStr = "LEFT JOIN " + storage.ResourceLinks.String() + " ON " + storage.StructuralBlobsID.String() + "=" + storage.ResourceLinksSource.String() + pageTokenStr = storage.BlobsID.String() + " <= :idx AND (" + storage.ResourcesIRI.String() + " NOT IN (SELECT " + storage.DraftsViewResource.String() + " from " + storage.DraftsView.String() + ") OR " + storage.ResourcesIRI.String() + " IS NULL) ORDER BY " + storage.BlobsID.String() + " desc limit :page_size" ) var getEventsStr = fmt.Sprintf(` @@ -160,8 +162,9 @@ func (srv *Server) ListEvents(ctx context.Context, req *activity.ListEventsReque %s %s %s + %s WHERE %s %s %s; - `, selectStr, tableStr, joinIDStr, joinpkStr, leftjoinStr, trustedStr, filtersStr, linksStr, pageTokenStr) + `, selectStr, tableStr, joinIDStr, joinpkStr, leftjoinResourcesStr, leftjoinLinksStr, trustedStr, filtersStr, linksStr, pageTokenStr) var lastBlobID int64 err = sqlitex.Exec(conn, dqb.Str(getEventsStr)(), func(stmt *sqlite.Stmt) error { lastBlobID = stmt.ColumnInt64(0) From df4fd327f3b85cf30883ad9b048d385b6ecf85c7 Mon Sep 17 00:00:00 2001 From: juligasa <11684004+juligasa@users.noreply.github.com> Date: Tue, 2 Apr 2024 11:44:29 +0200 Subject: [PATCH 4/4] fix(daemon): remove duplicated mentions. --- backend/daemon/api/activity/v1alpha/activity.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/backend/daemon/api/activity/v1alpha/activity.go b/backend/daemon/api/activity/v1alpha/activity.go index 1975487af..7fea733fd 100644 --- a/backend/daemon/api/activity/v1alpha/activity.go +++ b/backend/daemon/api/activity/v1alpha/activity.go @@ -146,13 +146,14 @@ func (srv *Server) ListEvents(ctx context.Context, req *activity.ListEventsReque linksStr += "))) AND " } var ( - selectStr = "SELECT " + storage.BlobsID + ", " + storage.StructuralBlobsType + ", " + storage.PublicKeysPrincipal + ", " + storage.ResourcesIRI + ", " + storage.StructuralBlobsTs + ", " + storage.BlobsInsertTime + ", " + storage.BlobsMultihash + ", " + storage.BlobsCodec + selectStr = "SELECT distinct " + storage.BlobsID + ", " + storage.StructuralBlobsType + ", " + storage.PublicKeysPrincipal + ", " + storage.ResourcesIRI + ", " + storage.StructuralBlobsTs + ", " + storage.BlobsInsertTime + ", " + storage.BlobsMultihash + ", " + storage.BlobsCodec tableStr = "FROM " + storage.T_StructuralBlobs joinIDStr = "JOIN " + storage.Blobs.String() + " ON " + storage.BlobsID.String() + "=" + storage.StructuralBlobsID.String() joinpkStr = "JOIN " + storage.PublicKeys.String() + " ON " + storage.StructuralBlobsAuthor.String() + "=" + storage.PublicKeysID.String() + joinLinksStr = "JOIN " + storage.ResourceLinks.String() + " ON " + storage.StructuralBlobsID.String() + "=" + storage.ResourceLinksSource.String() leftjoinResourcesStr = "LEFT JOIN " + storage.Resources.String() + " ON " + storage.StructuralBlobsResource.String() + "=" + storage.ResourcesID.String() - leftjoinLinksStr = "LEFT JOIN " + storage.ResourceLinks.String() + " ON " + storage.StructuralBlobsID.String() + "=" + storage.ResourceLinksSource.String() - pageTokenStr = storage.BlobsID.String() + " <= :idx AND (" + storage.ResourcesIRI.String() + " NOT IN (SELECT " + storage.DraftsViewResource.String() + " from " + storage.DraftsView.String() + ") OR " + storage.ResourcesIRI.String() + " IS NULL) ORDER BY " + storage.BlobsID.String() + " desc limit :page_size" + + pageTokenStr = storage.BlobsID.String() + " <= :idx AND (" + storage.ResourcesIRI.String() + " NOT IN (SELECT " + storage.DraftsViewResource.String() + " from " + storage.DraftsView.String() + ") OR " + storage.ResourcesIRI.String() + " IS NULL) ORDER BY " + storage.BlobsID.String() + " desc limit :page_size" ) var getEventsStr = fmt.Sprintf(` @@ -164,7 +165,7 @@ func (srv *Server) ListEvents(ctx context.Context, req *activity.ListEventsReque %s %s WHERE %s %s %s; - `, selectStr, tableStr, joinIDStr, joinpkStr, leftjoinResourcesStr, leftjoinLinksStr, trustedStr, filtersStr, linksStr, pageTokenStr) + `, selectStr, tableStr, joinIDStr, joinpkStr, joinLinksStr, leftjoinResourcesStr, trustedStr, filtersStr, linksStr, pageTokenStr) var lastBlobID int64 err = sqlitex.Exec(conn, dqb.Str(getEventsStr)(), func(stmt *sqlite.Stmt) error { lastBlobID = stmt.ColumnInt64(0)