Skip to content

Commit

Permalink
Memory management (#6)
Browse files Browse the repository at this point in the history
  • Loading branch information
@MMagueta
MMagueta authored Apr 7, 2024
1 parent dd6a039 commit 7a92fa3
Show file tree
Hide file tree
Showing 13 changed files with 285 additions and 180 deletions.
5 changes: 4 additions & 1 deletion .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,4 +8,7 @@
.idea
**/Parser.fs
**/Parser.fsi
**/Lexer.fs
**/Lexer.fs
/.vs
/Tree/x64/Debug
/x64/Debug
4 changes: 4 additions & 0 deletions sources/client.el
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,10 @@
UPDATE Account SET AccountNumber (PROJECT SUM(Value INTEGER) Credit SELECT AccountNumber = 1 - PROJECT SUM(Value INTEGER) Debit SELECT AccountNumber = 1) SELECT AccountNumber = 1
END")

(setq default-search "LOCK READ PROJECT (AccountNumber INTEGER Balance INTEGER) Account") ; SELECT Account@AccountNumber = 1")

(perplex-client default-search)

(perplex-client transact)

(perplex-client account-create-relation)
Expand Down
5 changes: 3 additions & 2 deletions src/Configuration/Configuration.fsproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,9 @@
</ItemGroup>
<ItemGroup>
<PackageReference Include="FsConfig" Version="4.1.0" />
<PackageReference Include="Serilog" Version="3.1.2-dev-02097" />
<PackageReference Include="Serilog" Version="4.0.0-dev-02149" />
<PackageReference Include="Serilog.Sinks.ColoredConsole" Version="3.0.1" />
<PackageReference Include="Serilog.Sinks.File" Version="5.0.1-dev-00968" />
<PackageReference Include="Serilog.Sinks.File" Version="5.0.1-dev-00972" />
<PackageReference Update="FSharp.Core" Version="8.0.300-beta.24154.4" />
</ItemGroup>
</Project>
189 changes: 129 additions & 60 deletions src/Disk/Conversion.fs
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,9 @@
namespace IO
// #nowarn "9"

open System
open System.Buffers.Text
open Microsoft.FSharp.NativeInterop

module Utilities = begin
[<RequireQualifiedAccess>]
Expand Down Expand Up @@ -113,16 +118,10 @@ module Read = begin

[<RequireQualifiedAccessAttribute>]
module Tree =
[<DllImport(__SOURCE_DIRECTORY__ + "/libbplustree.so", CallingConvention = CallingConvention.StdCall, ExactSpelling = true)>]
extern void print_leaves(void* _tree);
[<DllImport(__SOURCE_DIRECTORY__ + "/libbplustree.so", CallingConvention = CallingConvention.StdCall, ExactSpelling = true)>]
extern void* insert(void* _tree, int _key, int _chunkNumber, int _pageNumber, int _slotNumber)
[<DllImport(__SOURCE_DIRECTORY__ + "/libbplustree.so", CallingConvention = CallingConvention.StdCall, ExactSpelling = true)>]
extern void find_and_print(void * _tree, int _key, bool _verbose)
[<DllImport(__SOURCE_DIRECTORY__ + "/libbplustree.so", CallingConvention = CallingConvention.StdCall, ExactSpelling = true)>]
extern void* find_and_get_value(void * _tree, int _key, bool _verbose)
[<DllImport(__SOURCE_DIRECTORY__ + "/libbplustree.so", CallingConvention = CallingConvention.StdCall, ExactSpelling = true)>]
extern void* find_and_get_node(void * _tree, int _key)
extern IntPtr find_and_get_node(void * _tree, int _key, int* size)

type ChunkNumber = int
type PageNumber = int
Expand All @@ -134,7 +133,7 @@ module Read = begin
slotNumber: InstanceNumber
entity: ColumnMap
offset: int
key: int }
key: Value.t option }

type Page = Index array array

Expand Down Expand Up @@ -185,10 +184,17 @@ module Read = begin
chunkNumber = chunkNumber
pageNumber = pageNumber
slotNumber = slotNumber} ->
Tree.insert(tree, key, chunkNumber, pageNumber, slotNumber)) tree page) tree chunk) IntPtr.Zero pages
match key with
| Some (Value.VInteger32 key) -> Tree.insert(tree, key, chunkNumber, pageNumber, slotNumber)
| Some (Value.VVariableString key) ->
// TODO: For now uses the hash. This is incorrect given the set sizes differ. To fix this the C tree needs to support string comparisons
Tree.insert(tree, key.GetHashCode(), chunkNumber, pageNumber, slotNumber)
// 0 is the default key to get all via the tree
| None -> Tree.insert(tree, 0, chunkNumber, pageNumber, slotNumber)) tree page) tree chunk) IntPtr.Zero pages
{ pages = pages; tree = tree; amountAlreadyRead = amountAlreadyRead }

[<Struct>]
[<StructLayout(LayoutKind.Sequential)>]
type CRecord =
{ chunkNumber: int
pageNumber: int
Expand All @@ -215,70 +221,133 @@ module Read = begin
x
|> fun x -> (x, offset))
|> Array.map (fun (x, offset) -> Write.Disk.writeFact stream schema relationName offset x)

let private makeArray<'T> (value: IntPtr) size =
[| for i in 0..size-1 do
yield Marshal.PtrToStructure<'T>(Marshal.ReadIntPtr(IntPtr.Add (value, i*IntPtr.Size))) |]

let search stream (schema: Schema.t) (entityName: string) (projectionParam: Expression.ProjectionParameter) (refinement: Expression.Operators option) (indexBuilder: IndexBuilder) =
let search stream (schema: Schema.t) (entityName: string) (projectionParam: Expression.ProjectionParameter) (refinement: Expression.Operators option): (Map<string,Value.t> * OffsetNumber option) array option =
match Map.tryFind entityName schema with
| Some (Entity.Relation (relationAttributes, _physicalCount)) ->

let initialPageNumber = 0
let mutable amountAlreadyRead = 0

// printfn "Leaves: %A" lastReadChunk.pages
// Tree.print_leaves lastReadChunk.tree


match projectionParam, refinement with
| Expression.ProjectionParameter.Restrict attributes, None ->
None
| Expression.ProjectionParameter.Restrict _, Some (Expression.Operators.Equal (attr, key)) ->
let lastReadChunk = buildPagination stream schema entityName relationAttributes indexBuilder initialPageNumber amountAlreadyRead
amountAlreadyRead <- lastReadChunk.amountAlreadyRead
let value = Tree.find_and_get_value (lastReadChunk.tree, key, false)
let crecord =
Marshal.PtrToStructure<CRecord> value
printfn "%A" <| crecord
printfn "%A" <| lastReadChunk.pages.[crecord.chunkNumber].[crecord.pageNumber].[crecord.slotNumber]
None
| Expression.ProjectionParameter.Sum attr, Some (Expression.Operators.Equal (_, key)) ->
let lastReadChunk = buildPagination stream schema entityName relationAttributes indexBuilder initialPageNumber amountAlreadyRead
amountAlreadyRead <- lastReadChunk.amountAlreadyRead
lastReadChunk.pages
|> Array.concat
|> Array.concat
|> Array.choose (function
| {entity = entity; key = key_val; offset = offset} when key_val = key ->
let value = entity.[attr]
Some (value, offset)
| _ -> None)
|> Array.sumBy (fun (Value.VInteger32 x, _) -> x)
|> fun x -> ([|(Map.empty |> Map.add "SUM" (Value.VInteger32 x), None)|])
|> Some
| Expression.ProjectionParameter.All, Some (Expression.Operators.Equal (_, key)) ->
let indexBuilder: IndexBuilder =
fun offset chunkNumber pageNumber instanceNumber columns ->
{ entity = columns
key = None
chunkNumber = chunkNumber
offset = offset
pageNumber = pageNumber
slotNumber = instanceNumber }

let mutable size: int = 0
let nativeIntSize = NativePtr.toNativeInt<int> &&size
let sizePtr: nativeptr<int> = NativePtr.ofNativeInt<int> nativeIntSize
let lastReadChunk = buildPagination stream schema entityName relationAttributes indexBuilder initialPageNumber amountAlreadyRead
amountAlreadyRead <- lastReadChunk.amountAlreadyRead
lastReadChunk.pages
|> Array.concat
|> Array.concat
|> Array.choose (function
| {entity = entity; key = key_val; offset = offset} when key_val = key ->
//Some (entity |> Map.map (fun _ v -> v.Serialize()), Some offset)
Some (entity, Some offset)
| _ -> None)
|> Some
// The 0 searches for all given the lack of refinement
// Check the index builder population on the pagination
let record = Tree.find_and_get_node(lastReadChunk.tree, 0, sizePtr)
let size = NativePtr.read<int> sizePtr
let lookups = makeArray<CRecord> record size

//NativeLibrary.Free lastReadChunk.tree
//NativeLibrary.Free record

| Expression.ProjectionParameter.Taking(limit, attributes), Some (Expression.Operators.Equal (_, key)) ->
Some <| Array.map (fun record ->
let value = lastReadChunk.pages.[record.chunkNumber].[record.pageNumber].[record.slotNumber]
let projectedAttributes =
(value.entity: Map<string,Value.t>)
|> Map.filter (fun key _ -> List.contains key attributes)
(projectedAttributes, Some value.offset)) lookups
| Expression.ProjectionParameter.Restrict attributes, Some (Expression.Operators.Equal (attr, key)) ->
let indexBuilder: IndexBuilder =
fun offset chunkNumber pageNumber instanceNumber (columns: Map<string, Value.t>) ->
match attr with
| Expression.LocalizedIdentifier (relation, name) when relation = entityName ->
match Map.tryFind name columns with
| Some key ->
{ entity = columns
key = Some key
chunkNumber = chunkNumber
offset = offset
pageNumber = pageNumber
slotNumber = instanceNumber }
| None ->
failwith ""
| Expression.LocalizedIdentifier _ ->
failwith "Only allowed to refine over the relation attributes."
| _ -> failwith ""

let mutable size: int = 0
let nativeIntSize = NativePtr.toNativeInt<int> &&size
let sizePtr: nativeptr<int> = NativePtr.ofNativeInt<int> nativeIntSize
let lastReadChunk = buildPagination stream schema entityName relationAttributes indexBuilder initialPageNumber amountAlreadyRead
amountAlreadyRead <- lastReadChunk.amountAlreadyRead
lastReadChunk.pages
|> Array.concat
|> Array.concat
|> Array.choose (function
| {entity = entity; key = key_val; offset = offset} when key_val = key ->
Some (Map.filter (fun k _ -> List.contains k attributes) entity, Some offset)
| _ -> None)
|> fun elems -> if elems.Length < limit then Array.take elems.Length elems else Array.take limit elems
|> Some
| otherwise -> failwithf "NOT EXPECTING: %A" otherwise
let record =
match key with
| Expression.Literal (Value.VInteger32 key) ->
Tree.find_and_get_node(lastReadChunk.tree, key, sizePtr)
| Expression.Literal (Value.VVariableString key) ->
Tree.find_and_get_node(lastReadChunk.tree, key.GetHashCode(), sizePtr)
| _ -> failwith ""

let size = NativePtr.read<int> sizePtr
let lookups = makeArray<CRecord> record size

//NativeLibrary.Free lastReadChunk.tree
//NativeLibrary.Free record

Some <| Array.map (fun record ->
let value = lastReadChunk.pages.[record.chunkNumber].[record.pageNumber].[record.slotNumber]
let projectedAttributes =
(value.entity: Map<string,Value.t>)
|> Map.filter (fun key _ -> List.contains key attributes)
(projectedAttributes, Some value.offset)) lookups

// | Expression.ProjectionParameter.Sum attr, Some (Expression.Operators.Equal (_, key)) ->
// let lastReadChunk = buildPagination stream schema entityName relationAttributes indexBuilder initialPageNumber amountAlreadyRead
// amountAlreadyRead <- lastReadChunk.amountAlreadyRead
// lastReadChunk.pages
// |> Array.concat
// |> Array.concat
// |> Array.choose (function
// | {entity = entity; key = key_val; offset = offset} when key_val = 1 -> //key ->
// let value = entity.[attr]
// Some (value, offset)
// | _ -> None)
// |> Array.sumBy (fun (Value.VInteger32 x, _) -> x)
// |> fun x -> ([|(Map.empty |> Map.add "SUM" (Value.VInteger32 x), None)|])
// |> Some
// | Expression.ProjectionParameter.All, Some (Expression.Operators.Equal (_, key)) ->
// let lastReadChunk = buildPagination stream schema entityName relationAttributes indexBuilder initialPageNumber amountAlreadyRead
// amountAlreadyRead <- lastReadChunk.amountAlreadyRead
// lastReadChunk.pages
// |> Array.concat
// |> Array.concat
// |> Array.choose (function
// | {entity = entity; key = key_val; offset = offset} when key_val = 1 -> //key ->
// //Some (entity |> Map.map (fun _ v -> v.Serialize()), Some offset)
// Some (entity, Some offset)
// | _ -> None)
// |> Some
//
// | Expression.ProjectionParameter.Taking(limit, attributes), Some (Expression.Operators.Equal (_, key)) ->
// let lastReadChunk = buildPagination stream schema entityName relationAttributes indexBuilder initialPageNumber amountAlreadyRead
// amountAlreadyRead <- lastReadChunk.amountAlreadyRead
// lastReadChunk.pages
// |> Array.concat
// |> Array.concat
// |> Array.choose (function
// | {entity = entity; key = key_val; offset = offset} when key_val = 1 -> //key ->
// Some (Map.filter (fun k _ -> List.contains k attributes) entity, Some offset)
// | _ -> None)
// |> fun elems -> if elems.Length < limit then Array.take elems.Length elems else Array.take limit elems
// |> Some
// | otherwise -> failwithf "NOT EXPECTING: %A" otherwise

| None -> failwithf "Entity '%s' could not be located in the working schema." entityName

Expand Down
3 changes: 2 additions & 1 deletion src/Disk/Disk.fsproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,13 @@
<!-- </Target> -->
<ItemGroup>
<Compile Include="Conversion.fs" />
<ClCompile Include="bplustree.c" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Language\Language.fsproj" />
<ProjectReference Include="..\Configuration\Configuration.fsproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Update="FSharp.Core" Version="8.0.100" />
<PackageReference Update="FSharp.Core" Version="8.0.300-beta.24154.4" />
</ItemGroup>
</Project>
Loading

0 comments on commit 7a92fa3

Please sign in to comment.