Skip to content

Commit

Permalink
fix bugs surrounding truncated ints
Browse files Browse the repository at this point in the history
  • Loading branch information
@chirst
chirst committed Jun 27, 2024
1 parent 03ae899 commit 6897000
Show file tree
Hide file tree
Showing 5 changed files with 73 additions and 73 deletions.
39 changes: 19 additions & 20 deletions kv/kv.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ func (kv *KV) GetCatalog() *catalog {
// Get returns a byte array corresponding to the key and a bool indicating if
// the key was found. The pageNumber has to do with the root page of the
// corresponding table. The system catalog uses the page number 1.
func (kv *KV) Get(pageNumber uint16, key []byte) ([]byte, bool, error) {
func (kv *KV) Get(pageNumber int, key []byte) ([]byte, bool, error) {
if pageNumber == pager.EMPTY_PARENT_PAGE_NUMBER {
return nil, false, errorReservedPage
}
Expand All @@ -63,19 +63,15 @@ func (kv *KV) Get(pageNumber uint16, key []byte) ([]byte, bool, error) {
}
// Step 3. If the page is internal jump to the next page and go back to
// 2. This process guarantees that we are on a leaf page for step 4.
pageNumber = binary.LittleEndian.Uint16(v)
pageNumber = int(binary.LittleEndian.Uint16(v))
}
}

// Set inserts or updates the value for the given key. The pageNumber has to do
// with the root page of the corresponding table. The system catalog uses the
// page number 1.
func (kv *KV) Set(pageNumber uint16, key, value []byte) error {
// TODO set has some issues. One being set doesn't differentiate between
// insert and update which isn't very intentional. Two being set cannot
// perform multiple insertions in the span of one transaction. This is
// because page splits pull out stale pages on subsequent inserts which
// causes difficult to diagnose bugs.
func (kv *KV) Set(pageNumber int, key, value []byte) error {
// TODO set doesn't differentiate between insert and update
if pageNumber == pager.EMPTY_PARENT_PAGE_NUMBER {
return errorReservedPage
}
Expand Down Expand Up @@ -125,15 +121,15 @@ func insertIntoOne(key, value []byte, lp, rp *pager.Page) {
rp.SetEntries(append(rp.GetEntries(), pager.PageTuple{Key: key, Value: value}))
}

func (kv *KV) getLeafPage(nextPageNumber uint16, key []byte) *pager.Page {
func (kv *KV) getLeafPage(nextPageNumber int, key []byte) *pager.Page {
p := kv.pager.GetPage(nextPageNumber)
for p.GetType() != pager.PAGE_TYPE_LEAF {
nextPage, found := p.GetValue(key)
if !found {
return nil
}
nextPageNumber = binary.LittleEndian.Uint16(nextPage)
p = kv.pager.GetPage(nextPageNumber)
nextPageNumber16 := binary.LittleEndian.Uint16(nextPage)
p = kv.pager.GetPage(int(nextPageNumber16))
}
return p
}
Expand Down Expand Up @@ -215,7 +211,7 @@ func (kv *KV) parentInsert(p, l, r *pager.Page) {
// NewBTree creates an empty BTree and returns the new tree's root page number.
func (kv *KV) NewBTree() int {
np := kv.pager.NewPage()
return int(np.GetNumber())
return np.GetNumber()
}

func (kv *KV) BeginReadTransaction() {
Expand All @@ -238,14 +234,15 @@ func (kv *KV) EndWriteTransaction() error {
// For a integer key it is the largest integer key plus one.
func (kv *KV) NewRowID(rootPageNumber int) (int, error) {
// TODO could possibly cache this in the catalog or something
candidate := kv.pager.GetPage(uint16(rootPageNumber))
candidate := kv.pager.GetPage(rootPageNumber)
if len(candidate.GetEntries()) == 0 {
return 1, nil
}
for candidate.GetType() != pager.PAGE_TYPE_LEAF {
pagePointers := candidate.GetEntries()
descendingPageNum := pagePointers[len(pagePointers)-1].Value
candidate = kv.pager.GetPage(binary.LittleEndian.Uint16(descendingPageNum))
descendingPageNum16 := binary.LittleEndian.Uint16(descendingPageNum)
candidate = kv.pager.GetPage(int(descendingPageNum16))
}
k := candidate.GetEntries()[len(candidate.GetEntries())-1].Key
dk, err := DecodeKey(k)
Expand Down Expand Up @@ -292,8 +289,8 @@ type Cursor struct {
rootPageNumber int
currentPageEntries []pager.PageTuple
currentTupleIndex int
currentLeftPage uint16
currentRightPage uint16
currentLeftPage int
currentRightPage int
pager *pager.Pager
}

Expand All @@ -307,14 +304,15 @@ func (kv *KV) NewCursor(rootPageNumber int) *Cursor {
// GotoFirstRecord moves the cursor to the first tuple in ascending order. It
// returns true if the table has values. It returns false if the table is empty.
func (c *Cursor) GotoFirstRecord() bool {
candidatePage := c.pager.GetPage(uint16(c.rootPageNumber))
candidatePage := c.pager.GetPage(c.rootPageNumber)
if len(candidatePage.GetEntries()) == 0 {
return false
}
for candidatePage.GetType() != pager.PAGE_TYPE_LEAF {
pagePointers := candidatePage.GetEntries()
ascendingPageNum := pagePointers[0].Value
candidatePage = c.pager.GetPage(binary.LittleEndian.Uint16(ascendingPageNum))
ascendingPageNum16 := binary.LittleEndian.Uint16(ascendingPageNum)
candidatePage = c.pager.GetPage(int(ascendingPageNum16))
}
c.currentPageEntries = candidatePage.GetEntries()
c.currentTupleIndex = 0
Expand All @@ -324,14 +322,15 @@ func (c *Cursor) GotoFirstRecord() bool {
}

func (c *Cursor) GotoLastRecord() bool {
candidatePage := c.pager.GetPage(uint16(c.rootPageNumber))
candidatePage := c.pager.GetPage(c.rootPageNumber)
if len(candidatePage.GetEntries()) == 0 {
return false
}
for candidatePage.GetType() != pager.PAGE_TYPE_LEAF {
pagePointers := candidatePage.GetEntries()
descendingPageNum := pagePointers[len(pagePointers)-1].Value
candidatePage = c.pager.GetPage(binary.LittleEndian.Uint16(descendingPageNum))
descendingPageNum16 := binary.LittleEndian.Uint16(descendingPageNum)
candidatePage = c.pager.GetPage(int(descendingPageNum16))
}
c.currentPageEntries = candidatePage.GetEntries()
c.currentTupleIndex = len(c.currentPageEntries) - 1
Expand Down
2 changes: 1 addition & 1 deletion main_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ func TestBulk(t *testing.T) {
if r.Err != nil {
t.Fatal(r.Err.Error())
}
for i := 0; i < 700; i += 1 {
for i := 0; i < 1000; i += 1 {
r = d.Execute("INSERT INTO test (junk) VALUES ('asdf')")
if r.Err != nil {
t.Fatal(r.Err.Error())
Expand Down
93 changes: 47 additions & 46 deletions pager/pager.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ type Pager struct {
// store implements storage and is typically a file
store storage
// currentMaxPage is a counter that holds a free page number
currentMaxPage uint16
currentMaxPage int
// fileLock enables read and writes to be in isolation. The RWMutex allows
// either many readers or only one writer. RWMutex also prevents writer
// starvation by only allowing readers to acquire a lock if there is no
Expand Down Expand Up @@ -91,7 +91,7 @@ func New(useMemory bool) (*Pager, error) {
}
p := &Pager{
store: s,
currentMaxPage: cmpi,
currentMaxPage: int(cmpi),
fileLock: sync.RWMutex{},
dirtyPages: []*Page{},
pageCache: cache.NewLRU(PAGE_CACHE_SIZE),
Expand Down Expand Up @@ -126,7 +126,7 @@ func (p *Pager) EndWrite() error {
}
for _, fp := range p.dirtyPages {
p.WritePage(fp)
p.pageCache.Remove(int(fp.GetNumber()))
p.pageCache.Remove(fp.GetNumber())
}
p.dirtyPages = []*Page{}
p.writeMaxPageNumber()
Expand All @@ -138,7 +138,7 @@ func (p *Pager) EndWrite() error {
return nil
}

func (p *Pager) GetPage(pageNumber uint16) *Page {
func (p *Pager) GetPage(pageNumber int) *Page {
// During a write pages are collected in the dirtyPages buffer. These pages
// must be retrieved from the buffer as they are modified because the file
// is becoming outdated.
Expand All @@ -150,7 +150,7 @@ func (p *Pager) GetPage(pageNumber uint16) *Page {
return p.dirtyPages[dpn]
}
} else {
if v, hit := p.pageCache.Get(int(pageNumber)); hit {
if v, hit := p.pageCache.Get(pageNumber); hit {
return p.allocatePage(pageNumber, v)
}
}
Expand All @@ -161,19 +161,22 @@ func (p *Pager) GetPage(pageNumber uint16) *Page {
if p.isWriting {
p.dirtyPages = append(p.dirtyPages, ap)
}
p.pageCache.Add(int(pageNumber), page)
p.pageCache.Add(pageNumber, page)
return ap
}

func (p *Pager) WritePage(page *Page) error {
// Page number subtracted by one since 0 is reserved as a pointer to nothing
_, err := p.store.WriteAt(page.content, int64(ROOT_PAGE_START+(page.GetNumber()-1)*PAGE_SIZE))
pn := page.GetNumber() - 1
pns := pn * PAGE_SIZE
off := ROOT_PAGE_START + pns
_, err := p.store.WriteAt(page.content, int64(off))
return err
}

func (p *Pager) writeMaxPageNumber() {
cmpb := make([]byte, FREE_PAGE_COUNTER_SIZE)
binary.LittleEndian.PutUint16(cmpb, p.currentMaxPage)
binary.LittleEndian.PutUint16(cmpb, uint16(p.currentMaxPage))
p.store.WriteAt(cmpb, FREE_PAGE_COUNTER_OFFSET)
}

Expand All @@ -186,7 +189,7 @@ func (p *Pager) NewPage() *Page {
return np
}

func (p *Pager) allocatePage(pageNumber uint16, content []byte) *Page {
func (p *Pager) allocatePage(pageNumber int, content []byte) *Page {
np := &Page{
content: content,
number: pageNumber,
Expand Down Expand Up @@ -220,7 +223,7 @@ func (p *Pager) allocatePage(pageNumber uint16, content []byte) *Page {
// tuple the size of the Page.
type Page struct {
content []byte
number uint16
number int
}

// PageTuple is a variable length key value pair.
Expand All @@ -229,80 +232,78 @@ type PageTuple struct {
Value []byte
}

func (p *Page) GetParentPageNumber() (hasParent bool, pageNumber uint16) {
func (p *Page) GetParentPageNumber() (hasParent bool, pageNumber int) {
pn := binary.LittleEndian.Uint16(p.content[PARENT_POINTER_OFFSET : PARENT_POINTER_OFFSET+PAGE_POINTER_SIZE])
// An unsigned int page number has to be reserved to tell if the current
// page is a root.
if pn == EMPTY_PARENT_PAGE_NUMBER {
return false, EMPTY_PARENT_PAGE_NUMBER
}
return true, pn
return true, int(pn)
}

func (p *Page) SetParentPageNumber(pageNumber uint16) {
func (p *Page) SetParentPageNumber(pageNumber int) {
bpn := make([]byte, PAGE_POINTER_SIZE)
binary.LittleEndian.PutUint16(bpn, pageNumber)
binary.LittleEndian.PutUint16(bpn, uint16(pageNumber))
copy(p.content[PARENT_POINTER_OFFSET:PARENT_POINTER_OFFSET+PAGE_POINTER_SIZE], bpn)
}

func (p *Page) GetLeftPageNumber() (hasLeft bool, pageNumber uint16) {
func (p *Page) GetLeftPageNumber() (hasLeft bool, pageNumber int) {
pn := binary.LittleEndian.Uint16(p.content[LEFT_POINTER_OFFSET : LEFT_POINTER_OFFSET+PAGE_POINTER_SIZE])
if pn == EMPTY_PARENT_PAGE_NUMBER {
return false, EMPTY_PARENT_PAGE_NUMBER
}
return true, pn
return true, int(pn)
}

func (p *Page) SetLeftPageNumber(pageNumber uint16) {
func (p *Page) SetLeftPageNumber(pageNumber int) {
bpn := make([]byte, PAGE_POINTER_SIZE)
binary.LittleEndian.PutUint16(bpn, pageNumber)
binary.LittleEndian.PutUint16(bpn, uint16(pageNumber))
copy(p.content[LEFT_POINTER_OFFSET:LEFT_POINTER_OFFSET+PAGE_POINTER_SIZE], bpn)
}

func (p *Page) GetRightPageNumber() (hasRight bool, pageNumber uint16) {
func (p *Page) GetRightPageNumber() (hasRight bool, pageNumber int) {
pn := binary.LittleEndian.Uint16(p.content[RIGHT_POINTER_OFFSET : RIGHT_POINTER_OFFSET+PAGE_POINTER_SIZE])
if pn == EMPTY_PARENT_PAGE_NUMBER {
return false, EMPTY_PARENT_PAGE_NUMBER
}
return true, pn
return true, int(pn)
}

func (p *Page) SetRightPageNumber(pageNumber uint16) {
func (p *Page) SetRightPageNumber(pageNumber int) {
bpn := make([]byte, PAGE_POINTER_SIZE)
binary.LittleEndian.PutUint16(bpn, pageNumber)
binary.LittleEndian.PutUint16(bpn, uint16(pageNumber))
copy(p.content[RIGHT_POINTER_OFFSET:RIGHT_POINTER_OFFSET+PAGE_POINTER_SIZE], bpn)
}

func (p *Page) GetNumber() uint16 {
func (p *Page) GetNumber() int {
return p.number
}

func (p *Page) GetNumberAsBytes() []byte {
n := p.GetNumber()
bn := make([]byte, FREE_PAGE_COUNTER_SIZE)
binary.LittleEndian.PutUint16(bn, n)
binary.LittleEndian.PutUint16(bn, uint16(n))
return bn
}

func (p *Page) GetType() uint16 {
return binary.LittleEndian.Uint16(p.content[PAGE_TYPE_OFFSET:PAGE_TYPE_SIZE])
func (p *Page) GetType() int {
return int(binary.LittleEndian.Uint16(p.content[PAGE_TYPE_OFFSET:PAGE_TYPE_SIZE]))
}

func (p *Page) SetType(t uint16) {
func (p *Page) SetType(t int) {
bytePageType := make([]byte, PAGE_TYPE_SIZE)
binary.LittleEndian.PutUint16(bytePageType, t)
binary.LittleEndian.PutUint16(bytePageType, uint16(t))
copy(p.content[PAGE_TYPE_OFFSET:PAGE_TYPE_OFFSET+PAGE_TYPE_SIZE], bytePageType)
}

func (p *Page) getRecordCount() uint16 {
return binary.LittleEndian.Uint16(
p.content[PAGE_RECORD_COUNT_OFFSET : PAGE_RECORD_COUNT_OFFSET+PAGE_RECORD_COUNT_SIZE],
)
func (p *Page) getRecordCount() int {
return int(binary.LittleEndian.Uint16(p.content[PAGE_RECORD_COUNT_OFFSET : PAGE_RECORD_COUNT_OFFSET+PAGE_RECORD_COUNT_SIZE]))
}

func (p *Page) setRecordCount(newCount uint16) {
func (p *Page) setRecordCount(newCount int) {
byteRecordCount := make([]byte, PAGE_RECORD_COUNT_SIZE)
binary.LittleEndian.PutUint16(byteRecordCount, newCount)
binary.LittleEndian.PutUint16(byteRecordCount, uint16(newCount))
copy(
p.content[PAGE_RECORD_COUNT_OFFSET:PAGE_RECORD_COUNT_OFFSET+PAGE_RECORD_COUNT_SIZE],
byteRecordCount,
Expand Down Expand Up @@ -345,53 +346,53 @@ func (p *Page) SetEntries(entries []PageTuple) {
endValueOffset := shift + PAGE_ROW_OFFSET_SIZE + PAGE_ROW_OFFSET_SIZE

// set key offset
keyOffset := uint16(entryEnd - len(entry.Key) - len(entry.Value))
keyOffset := entryEnd - len(entry.Key) - len(entry.Value)
byteKeyOffset := make([]byte, PAGE_ROW_OFFSET_SIZE)
binary.LittleEndian.PutUint16(byteKeyOffset, keyOffset)
binary.LittleEndian.PutUint16(byteKeyOffset, uint16(keyOffset))
copy(p.content[startKeyOffset:endKeyOffset], byteKeyOffset)

// set value offset
valueOffset := uint16(entryEnd - len(entry.Value))
valueOffset := entryEnd - len(entry.Value)
byteValueOffset := make([]byte, PAGE_ROW_OFFSET_SIZE)
binary.LittleEndian.PutUint16(byteValueOffset, valueOffset)
binary.LittleEndian.PutUint16(byteValueOffset, uint16(valueOffset))
copy(p.content[endKeyOffset:endValueOffset], byteValueOffset)

// set key
copy(p.content[keyOffset:valueOffset], entry.Key)

// set value
copy(p.content[valueOffset:valueOffset+uint16(len(entry.Value))], entry.Value)
copy(p.content[valueOffset:valueOffset+len(entry.Value)], entry.Value)

// update for next iteration
shift = endValueOffset
entryEnd = int(keyOffset)
entryEnd = keyOffset
}
p.setRecordCount(uint16(len(entries)))
p.setRecordCount(len(entries))
}

func (p *Page) GetEntries() []PageTuple {
entries := []PageTuple{}
recordCount := p.getRecordCount()
entryEnd := PAGE_SIZE
for i := uint16(0); i < recordCount; i += 1 {
for i := 0; i < recordCount; i += 1 {
startKeyOffset := PAGE_ROW_OFFSETS_OFFSET + (i * (PAGE_ROW_OFFSET_SIZE + PAGE_ROW_OFFSET_SIZE))
endKeyOffset := PAGE_ROW_OFFSETS_OFFSET + (i * (PAGE_ROW_OFFSET_SIZE + PAGE_ROW_OFFSET_SIZE)) + PAGE_ROW_OFFSET_SIZE
endValueOffset := PAGE_ROW_OFFSETS_OFFSET + (i * (PAGE_ROW_OFFSET_SIZE + PAGE_ROW_OFFSET_SIZE)) + PAGE_ROW_OFFSET_SIZE + PAGE_ROW_OFFSET_SIZE

keyOffset := binary.LittleEndian.Uint16(p.content[startKeyOffset:endKeyOffset])
valueOffset := binary.LittleEndian.Uint16(p.content[endKeyOffset:endValueOffset])
keyOffset := int(binary.LittleEndian.Uint16(p.content[startKeyOffset:endKeyOffset]))
valueOffset := int(binary.LittleEndian.Uint16(p.content[endKeyOffset:endValueOffset]))

// These must be copied otherwise the underlying byte array is returned.
// This causes what seems a unique value to be treated as a reference.
byteKey := make([]byte, valueOffset-keyOffset)
copy(byteKey, p.content[keyOffset:valueOffset])
byteValue := make([]byte, entryEnd-int(valueOffset))
byteValue := make([]byte, entryEnd-valueOffset)
copy(byteValue, p.content[valueOffset:entryEnd])
entries = append(entries, PageTuple{
Key: byteKey,
Value: byteValue,
})
entryEnd = int(keyOffset)
entryEnd = keyOffset
}
return entries
}
Expand Down
Loading

0 comments on commit 6897000

Please sign in to comment.