Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Peer to peer call with Kurento server #5

Merged
merged 53 commits into from
Dec 30, 2024
Merged
Changes from 1 commit
Commits
Show all changes
53 commits
Select commit Hold shift + click to select a range
0716b13
[Backend] Expose user field
May 27, 2024
12d07c5
[Backend] Add CI
May 27, 2024
04d9eac
[Backend] Add CI
May 27, 2024
c0cd6da
[Backend] Add CI
May 27, 2024
b785f41
Rename .java to .kt
May 27, 2024
96ec88c
[Backend] Add CI
May 27, 2024
ad7b90a
Merge remote-tracking branch 'origin/develop' into develop
physcom May 27, 2024
22631e4
[Backend] Fix message save
May 27, 2024
3148a22
Merge remote-tracking branch 'origin/develop' into develop
physcom May 27, 2024
21680b6
gemini response fix
physcom May 27, 2024
149c571
[Backend] Fix message save
May 27, 2024
30ac2d2
[Backend] Add endpoint for controller
May 27, 2024
9df44a6
[Backend] Fix message save
Jun 3, 2024
0579435
[Backend] Add ai entities
Jun 4, 2024
d1bbba5
[Backend]
Jun 6, 2024
5a4d50c
[Backend]
Jun 6, 2024
2b3c4ff
attachment fix
physcom Jun 6, 2024
45a9942
Merge remote-tracking branch 'origin/AI-modeling' into AI-modeling
physcom Jun 6, 2024
a5ac30f
permit attachment download url
physcom Jun 12, 2024
4b9ddf2
fixed message attachment save
physcom Jun 12, 2024
536ef75
[Backend]
Jun 13, 2024
05e5163
[Backend]
Jun 20, 2024
9b97c09
[Backend]
Jun 20, 2024
8687c01
[Backend]
Jun 24, 2024
790491c
[Backend]
Aug 7, 2024
8ac0a9b
Ws server
borbuevbeksultan Nov 4, 2024
03c7647
Add Kurento module
borbuevbeksultan Nov 20, 2024
e890c51
Remove old build script
borbuevbeksultan Nov 20, 2024
8714926
Add new workflow
borbuevbeksultan Nov 20, 2024
535de51
Add new workflow
borbuevbeksultan Nov 20, 2024
d1f018c
Add new workflow
borbuevbeksultan Nov 20, 2024
9ccb23d
Add new workflow
borbuevbeksultan Nov 20, 2024
37e22b1
Add new workflow
borbuevbeksultan Nov 20, 2024
47d7fe8
Add new workflow
borbuevbeksultan Nov 20, 2024
0ff605d
Add new workflow
borbuevbeksultan Nov 20, 2024
ac704a1
Add new workflow
borbuevbeksultan Nov 20, 2024
b606a21
Add new workflow
borbuevbeksultan Nov 20, 2024
7cf3af8
Add new workflow
borbuevbeksultan Nov 20, 2024
f9dbfbb
Add new workflow
borbuevbeksultan Nov 20, 2024
dced6ec
Add new workflow
borbuevbeksultan Nov 20, 2024
deba1af
Add new workflow
borbuevbeksultan Nov 21, 2024
9f8117c
Add new workflow
borbuevbeksultan Nov 21, 2024
f2ac5ff
Add new workflow
borbuevbeksultan Nov 21, 2024
f8d3f3e
Add new workflow
borbuevbeksultan Nov 21, 2024
ee9f8eb
Add new workflow
borbuevbeksultan Nov 21, 2024
6bda1c4
Add new workflow
borbuevbeksultan Nov 21, 2024
715d502
Add new workflow
borbuevbeksultan Nov 21, 2024
f1e80c5
Add group call
borbuevbeksultan Nov 25, 2024
3535008
Add group call
borbuevbeksultan Nov 25, 2024
5d21906
One to one call
borbuevbeksultan Dec 5, 2024
4a336c1
Recording mixed file
borbuevbeksultan Dec 9, 2024
87233cf
Recording mixed file
borbuevbeksultan Dec 30, 2024
b4acb51
Recording mixed file
borbuevbeksultan Dec 30, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
[Backend]
* Assistant
Beksultan committed Jun 24, 2024
commit 8687c017dec30c2a2fc9561ba8950df6a9c17b45
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
package io.openfuture.openmessenger.repository

import io.openfuture.openmessenger.repository.entity.AssistantReminderEntity
import org.springframework.data.jpa.repository.JpaRepository

interface ReminderRepository : JpaRepository<AssistantReminderEntity, Long> {
fun findAllByAuthorAndChatId(author: String, chatId: Int): List<AssistantReminderEntity>
fun findAllByAuthorAndGroupChatId(author: String, chatId: Int): List<AssistantReminderEntity>
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
package io.openfuture.openmessenger.repository

import io.openfuture.openmessenger.repository.entity.AssistantTodoEntity
import org.springframework.data.jpa.repository.JpaRepository

interface TodoRepository : JpaRepository<AssistantTodoEntity, Long> {
fun findAllByAuthorAndChatId(author: String, chatId: Int): List<AssistantTodoEntity>
fun findAllByAuthorAndGroupChatId(author: String, chatId: Int): List<AssistantTodoEntity>
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
package io.openfuture.openmessenger.repository.entity

import jakarta.persistence.*
import java.time.LocalDateTime
import java.time.LocalDateTime.now

@Entity
@Table(name = "assistant_reminders")
class AssistantReminderEntity() {
constructor(
author: String?,
chatId: Int?,
groupChatId: Int?,
members: String?,
recipient: String?,
generatedAt: LocalDateTime = now(),
version: Int = 1,
startTime: LocalDateTime?,
endTime: LocalDateTime?,
reminders: String?
): this() {
this.author = author
this.chatId = chatId
this.groupChatId = groupChatId
this.members = members
this.recipient = recipient
this.generatedAt = generatedAt
this.version = version
this.startTime = startTime
this.endTime = endTime
this.reminders = reminders
}

@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
var id: Long? = null
var author: String? = null
var chatId: Int? = null
var groupChatId: Int? = null
var members: String? = null
var recipient: String? = null
var generatedAt: LocalDateTime = now()
var version: Int = 1
var startTime: LocalDateTime? = null
var endTime: LocalDateTime? = null
var reminders: String? = null
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
package io.openfuture.openmessenger.repository.entity

import jakarta.persistence.*
import java.time.LocalDateTime
import java.time.LocalDateTime.now

@Entity
@Table(name = "assistant_todos")
class AssistantTodoEntity() {
constructor(
author: String?,
chatId: Int?,
groupChatId: Int?,
members: String?,
recipient: String?,
generatedAt: LocalDateTime = now(),
version: Int = 1,
startTime: LocalDateTime?,
endTime: LocalDateTime?,
todos: String?
): this() {
this.author = author
this.chatId = chatId
this.groupChatId = groupChatId
this.members = members
this.recipient = recipient
this.generatedAt = generatedAt
this.version = version
this.startTime = startTime
this.endTime = endTime
this.todos = todos
}

@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
var id: Long? = null
var author: String? = null
var chatId: Int? = null
var groupChatId: Int? = null
var members: String? = null
var recipient: String? = null
var generatedAt: LocalDateTime = now()
var version: Int = 1
var startTime: LocalDateTime? = null
var endTime: LocalDateTime? = null
var todos: String? = null
}
Original file line number Diff line number Diff line change
@@ -5,11 +5,15 @@ import io.openfuture.openmessenger.assistant.model.Reminder
import io.openfuture.openmessenger.assistant.model.Todos
import io.openfuture.openmessenger.service.dto.AssistantRequest
import io.openfuture.openmessenger.service.dto.GetAllNotesRequest
import io.openfuture.openmessenger.service.dto.GetAllRemindersRequest
import io.openfuture.openmessenger.service.dto.GetAllTodosRequest

interface AssistantService {
fun generateNotes(assistantRequest: AssistantRequest): ConversationNotes?
fun generateReminder(assistantRequest: AssistantRequest): Reminder
fun generateTodos(assistantRequest: AssistantRequest): Todos

fun getAllNotes(getAllNotesRequest: GetAllNotesRequest): List<ConversationNotes>
fun getAllTodos(getAllTodosRequest: GetAllTodosRequest): List<Todos>
fun getAllReminders(getAllRemindersRequest: GetAllRemindersRequest): List<Reminder>
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
package io.openfuture.openmessenger.service.dto

data class GetAllRemindersRequest(
val chatId: Int,
val isGroup: Boolean,
)
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
package io.openfuture.openmessenger.service.dto

data class GetAllTodosRequest(
val chatId: Int,
val isGroup: Boolean,
)
Original file line number Diff line number Diff line change
@@ -1,20 +1,25 @@
package io.openfuture.openmessenger.service.impl

import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule
import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
import com.fasterxml.jackson.module.kotlin.readValue
import io.openfuture.openmessenger.assistant.gemini.GeminiService
import io.openfuture.openmessenger.assistant.model.*
import io.openfuture.openmessenger.repository.MessageJdbcRepository
import io.openfuture.openmessenger.repository.NoteRepository
import io.openfuture.openmessenger.repository.ReminderRepository
import io.openfuture.openmessenger.repository.TodoRepository
import io.openfuture.openmessenger.repository.entity.AssistantNoteEntity
import io.openfuture.openmessenger.repository.entity.AssistantReminderEntity
import io.openfuture.openmessenger.repository.entity.AssistantTodoEntity
import io.openfuture.openmessenger.repository.entity.Message
import io.openfuture.openmessenger.service.AssistantService
import io.openfuture.openmessenger.service.GroupChatService
import io.openfuture.openmessenger.service.PrivateChatService
import io.openfuture.openmessenger.service.UserAuthService
import io.openfuture.openmessenger.service.dto.AssistantRequest
import io.openfuture.openmessenger.service.dto.GetAllNotesRequest
import io.openfuture.openmessenger.service.dto.GetAllRemindersRequest
import io.openfuture.openmessenger.service.dto.GetAllTodosRequest
import io.openfuture.openmessenger.service.response.UserResponse
import org.springframework.data.domain.Sort
import org.springframework.stereotype.Service
@@ -28,6 +33,8 @@ class AssistantServiceImpl(
val messageJdbcRepository: MessageJdbcRepository,
val userAuthService: UserAuthService,
val noteRepository: NoteRepository,
val todoRepository: TodoRepository,
val reminderRepository: ReminderRepository
) : AssistantService {

override fun generateNotes(assistantRequest: AssistantRequest): ConversationNotes? {
@@ -81,15 +88,34 @@ class AssistantServiceImpl(
val participants: List<String>? = getParticipants(assistantRequest)

val objectMapper = jacksonObjectMapper()
objectMapper.registerModule(JavaTimeModule())

val conversation = getConversation(assistantRequest)

val result = geminiService.chat("${PROMPT_FOR_REMINDER.format("alice@gmail.com")}. Conversation starts here. $conversation")
var result = geminiService.chat("${PROMPT_FOR_REMINDER.format(current.email)}. Conversation starts here. $conversation")

println("Result [$result]")

val reminderItemList = objectMapper.readValue<List<ReminderItem>>(result!!)
result = result
?.replace("[", "")
?.replace("]", "")
?.replace("```json", "")
?.replace("```", "")

val reminderItemList = objectMapper.readValue<List<ReminderItem>>("[$result]")

val assistantReminderEntity = AssistantReminderEntity(
current.email,
if (assistantRequest.isGroup) null else assistantRequest.chatId,
if (assistantRequest.isGroup) assistantRequest.chatId else null,
objectMapper.writeValueAsString(participants),
getRecipient(current, assistantRequest),
LocalDateTime.now(),
1,
assistantRequest.startTime,
assistantRequest.endTime,
objectMapper.writeValueAsString(reminderItemList)
)
reminderRepository.save(assistantReminderEntity)

return Reminder(
if (assistantRequest.isGroup) null else assistantRequest.chatId,
@@ -114,11 +140,30 @@ class AssistantServiceImpl(

println(conversation)

val result = geminiService.chat("${PROMPT_TODOS.format("alice@gmail.com")}. Conversation starts here. $conversation")
var result = geminiService.chat("${PROMPT_TODOS.format(current.email)}. Conversation starts here. $conversation")

println("Result [$result]")

val todos = objectMapper.readValue<List<Todo>>(result!!)
result = result
?.replace("[", "")
?.replace("]", "")
?.replace("```json", "")
?.replace("```", "")
val todos = objectMapper.readValue<List<Todo>>("[$result]")

val assistantTodoEntity = AssistantTodoEntity(
current.email,
if (assistantRequest.isGroup) null else assistantRequest.chatId,
if (assistantRequest.isGroup) assistantRequest.chatId else null,
objectMapper.writeValueAsString(participants),
getRecipient(current, assistantRequest),
LocalDateTime.now(),
1,
assistantRequest.startTime,
assistantRequest.endTime,
objectMapper.writeValueAsString(todos)
)
todoRepository.save(assistantTodoEntity)

return Todos(
if (assistantRequest.isGroup) null else assistantRequest.chatId,
@@ -141,7 +186,29 @@ class AssistantServiceImpl(
noteRepository.findAllByAuthorAndChatId(current.email!!, getAllNotesRequest.chatId)
}

return notes.map { convertFromEntity(it) }
return notes.map { convertFromEntity(it) }
}

override fun getAllTodos(getAllTodosRequest: GetAllTodosRequest): List<Todos> {
val current = userAuthService.current()
val todos: List<AssistantTodoEntity> = if (getAllTodosRequest.isGroup) {
todoRepository.findAllByAuthorAndGroupChatId(current.email!!, getAllTodosRequest.chatId)
} else {
todoRepository.findAllByAuthorAndChatId(current.email!!, getAllTodosRequest.chatId)
}

return todos.map { convertFromEntity(it) }
}

override fun getAllReminders(getAllRemindersRequest: GetAllRemindersRequest): List<Reminder> {
val current = userAuthService.current()
val reminders: List<AssistantReminderEntity> = if (getAllRemindersRequest.isGroup) {
reminderRepository.findAllByAuthorAndGroupChatId(current.email!!, getAllRemindersRequest.chatId)
} else {
reminderRepository.findAllByAuthorAndChatId(current.email!!, getAllRemindersRequest.chatId)
}

return reminders.map { convertFromEntity(it) }
}

private fun getConversation(assistantRequest: AssistantRequest): String {
@@ -187,9 +254,9 @@ class AssistantServiceImpl(
"give me only and json output, in case multiple items, result is array, no other text with following format " +
"{\\\"dueDate\\\": \\\"due date time for task in ISO 8601\\\", " +
"\\\"description\\\": \\\"description about the task\\\"" +
"\\\"executor\\\": \\\"who is assignee\\\"" +
"\\\"executor\\\": \\\"if participant is executor, put him here otherwise skip this task\\\"" +
"\\\"context\\\": \\\"in which context task was raised\\\"" +
"}"
"}. In case if dueDate is unknown and can't be extracted, put there null."
}

private fun convertFromEntity(entity: AssistantNoteEntity): ConversationNotes {
@@ -210,4 +277,40 @@ class AssistantServiceImpl(
)
}

private fun convertFromEntity(entity: AssistantTodoEntity): Todos {
val objectMapper = jacksonObjectMapper()
val membersList: List<String> = entity.members?.let { objectMapper.readValue(it) } ?: emptyList()
val todoList: List<Todo> = entity.todos?.let { objectMapper.readValue(it) } ?: emptyList()

return Todos(
chatId = entity.chatId,
groupChatId = entity.groupChatId,
members = membersList,
recipient = entity.recipient,
generatedAt = entity.generatedAt,
version = entity.version,
startTime = entity.startTime ?: LocalDateTime.now(),
endTime = entity.endTime ?: LocalDateTime.now(),
todos = todoList
)
}

private fun convertFromEntity(entity: AssistantReminderEntity): Reminder {
val objectMapper = jacksonObjectMapper()
val membersList: List<String> = entity.members?.let { objectMapper.readValue(it) } ?: emptyList()
val reminderList: List<ReminderItem> = entity.reminders?.let { objectMapper.readValue(it) } ?: emptyList()

return Reminder(
chatId = entity.chatId,
groupChatId = entity.groupChatId,
members = membersList,
recipient = entity.recipient,
generatedAt = entity.generatedAt,
version = entity.version,
startTime = entity.startTime ?: LocalDateTime.now(),
endTime = entity.endTime ?: LocalDateTime.now(),
reminders = reminderList
)
}

}
Original file line number Diff line number Diff line change
@@ -6,7 +6,12 @@ import io.openfuture.openmessenger.assistant.model.Todos
import io.openfuture.openmessenger.service.AssistantService
import io.openfuture.openmessenger.service.dto.AssistantRequest
import io.openfuture.openmessenger.service.dto.GetAllNotesRequest
import org.springframework.web.bind.annotation.*
import io.openfuture.openmessenger.service.dto.GetAllRemindersRequest
import io.openfuture.openmessenger.service.dto.GetAllTodosRequest
import org.springframework.web.bind.annotation.PostMapping
import org.springframework.web.bind.annotation.RequestBody
import org.springframework.web.bind.annotation.RequestMapping
import org.springframework.web.bind.annotation.RestController

@RequestMapping("/api/v1/ai")
@RestController
@@ -40,4 +45,14 @@ class PromptController(
return assistantService.getAllNotes(request)
}

@PostMapping("/todos")
fun getAllTodos(@RequestBody request: GetAllTodosRequest): List<Todos> {
return assistantService.getAllTodos(request)
}

@PostMapping("/reminders")
fun getAllReminders(@RequestBody request: GetAllRemindersRequest): List<Reminder> {
return assistantService.getAllReminders(request)
}

}
8 changes: 4 additions & 4 deletions src/main/resources/db/migration/V3__create-tables-3.sql
Original file line number Diff line number Diff line change
@@ -19,13 +19,13 @@ create table assistant_todos
author varchar(255),
chat_id int null,
group_chat_id int null,
members json,
members text,
recipient varchar(255),
generated_at timestamp,
version int,
start_time timestamp,
end_time timestamp,
todos json
todos text
);

create table assistant_reminders
@@ -34,11 +34,11 @@ create table assistant_reminders
author varchar(255),
chat_id int null,
group_chat_id int null,
members json,
members text,
recipient varchar(255),
generated_at timestamp,
version int,
start_time timestamp,
end_time timestamp,
reminders json
reminders text
);