Mercurial > games > semicongine
changeset 1465:7b2ec9f3d0f6 default tip
add: external serialization library
author | sam <sam@basx.dev> |
---|---|
date | Wed, 26 Mar 2025 00:37:50 +0700 |
parents | 3e3192241ea7 |
children | |
files | semicongine/platform/linux/rendering.nim semicongine/storage.nim semicongine/thirdparty/vsbf/LICENSE semicongine/thirdparty/vsbf/vsbf.nim semicongine/thirdparty/vsbf/vsbf/decoders.nim semicongine/thirdparty/vsbf/vsbf/encoders.nim semicongine/thirdparty/vsbf/vsbf/shared.nim tests/test_storage.nim |
diffstat | 8 files changed, 792 insertions(+), 14 deletions(-) [+] |
line wrap: on
line diff
--- a/semicongine/platform/linux/rendering.nim Mon Mar 24 22:57:47 2025 +0700 +++ b/semicongine/platform/linux/rendering.nim Wed Mar 26 00:37:50 2025 +0700 @@ -232,7 +232,7 @@ vec2i(attribs.width, attribs.height) # buffer to save utf8-data from keyboard events -var unicodeData: array[64, char] +var unicodeData = newStringOfCap(64) proc pendingEvents*(window: NativeWindow): seq[Event] = var event: XEvent @@ -253,9 +253,8 @@ key: KeyTypeMap.getOrDefault(int(event.xkey.keycode), Key.UNKNOWN), ) var status: Status - var ksym: KeySym = NoSymbol let len = window.ic.Xutf8LookupString( - addr(event.xkey), addr(unicodeData[0]), unicodeData.len.cint, nil, addr(status) + addr(event.xkey), unicodeData.cstring, unicodeData.len.cint, nil, addr(status) ) if len > 0 and status != XBufferOverflow: unicodeData[len] = '\0'
--- a/semicongine/storage.nim Mon Mar 24 22:57:47 2025 +0700 +++ b/semicongine/storage.nim Wed Mar 26 00:37:50 2025 +0700 @@ -10,6 +10,7 @@ import ./core import ./thirdparty/db_connector/db_sqlite +import ./thirdparty/vsbf/vsbf const STORAGE_NAME = Path("storage.db") const DEFAULT_KEY_VALUE_TABLE_NAME = "shelf" @@ -108,28 +109,40 @@ sql( &"""CREATE TABLE IF NOT EXISTS {table} ( key INT NOT NULL UNIQUE, - value TEXT NOT NULL + value BLOB NOT NULL )""" ) ) -proc storeWorld*[T]( +proc storeWorld*[T: object | tuple]( worldName: string, world: T, table = DEFAULT_WORLD_TABLE_NAME, deleteOld = false ) = let db = worldName.ensureExists(table) defer: db.close() let key = $(int(now().toTime().toUnixFloat() * 1000)) - db.exec(sql(&"""INSERT INTO {table} VALUES(?, ?)"""), key, $$world) + + var encoder = Encoder.init() + encoder.serializeRoot(world) + + let s = db.prepare(&"""INSERT INTO {table} VALUES(?, ?)""") + s.bindParam(1, key) + s.bindParam(2, encoder.data) + db.exec(s) + s.finalize() db.exec(sql(&"""DELETE FROM {table} WHERE key <> ?"""), key) -proc loadWorld*[T](worldName: string, table = DEFAULT_WORLD_TABLE_NAME): T = +proc loadWorld*[T: object | tuple]( + worldName: string, table = DEFAULT_WORLD_TABLE_NAME +): T = let db = worldName.ensureExists(table) defer: db.close() let dbResult = db.getValue(sql(&"""SELECT value FROM {table} ORDER BY key DESC LIMIT 1""")) - return to[T](dbResult) + + var decoder = Decoder.init(cast[seq[byte]](dbResult)) + decoder.deserialize(T) proc listWorlds*(): seq[string] = let dir = Path(getDataDir()) / Path(AppName()) / Path(WORLD_DIR)
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/semicongine/thirdparty/vsbf/LICENSE Wed Mar 26 00:37:50 2025 +0700 @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2024 Jason Beetham + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE.
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/semicongine/thirdparty/vsbf/vsbf.nim Wed Mar 26 00:37:50 2025 +0700 @@ -0,0 +1,6 @@ +## This is a very simple binary format, aimed at game save serialisation. +## By using type information encoded we can catch decode errors and optionally invoke converters +## Say data was saved as a `Float32` but now is an `Int32` we can know this and do `int32(floatVal)` + +import vsbf/[shared, decoders, encoders] +export skipSerialization, decoders, encoders
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/semicongine/thirdparty/vsbf/vsbf/decoders.nim Wed Mar 26 00:37:50 2025 +0700 @@ -0,0 +1,352 @@ +import std/[options, typetraits, tables, macros, strformat] +import shared + +type Decoder*[DataType: SeqOrArr[byte]] = object + strs*: seq[string] ## List of strings that are indexed by string indexes + when DataType is seq[byte]: + stream*: seq[byte] + else: + stream*: UnsafeView[byte] + pos*: int + +template debug(args: varargs[typed, `$`]): untyped = + when defined(vsbfDebug): + unpackVarargs(echo, args) + +proc len(dec: Decoder): int = + dec.stream.len + +proc atEnd*(dec: Decoder): bool = + dec.pos >= dec.len + +template data*[T](decoder: Decoder[T]): untyped = + if decoder.pos >= decoder.stream.len: + raise insufficientData("More data expect, but hit end of stream.", decoder.pos) + + decoder.stream.toOpenArray(decoder.pos, decoder.stream.len - 1) + +proc read*[T: SomeInteger](oa: openArray[byte], res: var T): bool = + if sizeof(T) <= oa.len: + res = T(0) + for i in T(0) ..< sizeof(T): + res = res or (T(oa[int i]) shl (i * 8)) + + true + else: + false + +proc read*(frm: openArray[byte], to: var openArray[byte | char]): int = + if to.len > frm.len: + -1 + else: + for i in 0 .. to.high: + to[i] = typeof(to[0])(frm[i]) + to.len + +proc read*[T](dec: var Decoder[T], data: typedesc): Option[data] = + var val = default data + if dec.data.read(val) > 0: + some(val) + else: + none(data) + +proc readString*(dec: var Decoder) = + var strLen = 0 + dec.pos += dec.data.readLeb128(strLen) + + var buffer = newString(strLen) + + for i in 0 ..< strLen: + buffer[i] = char dec.data[i] + debug "Stored string ", buffer, " at index ", dec.strs.len + dec.strs.add buffer + dec.pos += strLen + +proc typeNamePair*( + dec: var Decoder +): tuple[typ: SerialisationType, nameInd: options.Option[int]] = + ## reads the type and name's string index if it has it + var encodedType = 0u8 + if not dec.data.read(encodedType): + raise newException(VsbfError, fmt"Failed to read type info. Position: {dec.pos}") + dec.pos += 1 + + var hasName: bool + (result.typ, hasName) = encodedType.decodeType(dec.pos - 1) + + if hasName: + var ind = 0 + let indSize = dec.data.readLeb128(ind) + dec.pos += indSize + result.nameInd = some(ind) + if indSize > 0: + if ind notin 0 .. dec.strs.high: + dec.readString() + else: + raise + (ref VsbfError)(msg: fmt"No name following a declaration. Position {dec.pos}") + +proc peekTypeNamePair*(dec: var Decoder): tuple[typ: SerialisationType, name: string] = + ## peek the type and name's string index if it has it + let encodedType = dec.data[0] + var hasName: bool + (result.typ, hasName) = encodedType.decodeType(dec.pos - 1) + debug result.typ, " has name: ", hasName + if hasName: + var val = 0 + let indSize = dec.data.toOpenArray(1).readLeb128(val) + debug "String of index: ", val, " found at ", dec.pos + + if indSize > 0: + if val notin 0 .. dec.strs.high: + debug "String not loaded, reading it" + var strLen = 0 + let + strLenBytes = dec.data.toOpenArray(1 + indSize).readLeb128(strLen) + start = 1 + indSize + strLenBytes + result.name = newString(strLen) + + if start >= dec.len or start + strLen - 1 > dec.len: + raise insufficientData("Need more data for a field", dec.pos) + + for i, x in dec.data.toOpenArray(start, start + strLen - 1): + result.name[i] = char x + else: + result.name = dec.strs[val] + else: + raise incorrectData("No name following a declaration.", dec.pos) + +proc getStr*(dec: Decoder, ind: int): lent string = + dec.strs[ind] + +proc readHeader(dec: var Decoder) = + var ext = dec.read(array[4, char]) + if ext.isNone or ext.unsafeGet != "vsbf": + raise incorrectData("Not a VSBF stream, missing the header", 0) + dec.pos += 4 + + let ver = dec.read(array[2, byte]) + + if ver.isNone: + raise incorrectData("Cannot read, missing version", 4) + + dec.pos += 2 + +proc init*(_: typedesc[Decoder], data: sink seq[byte]): Decoder[seq[byte]] = + ## Heap allocated version, it manages it's own buffer you give it and reads from this. + ## Can recover the buffer using `close` after parsin + result = Decoder[seq[byte]](stream: data) + result.readHeader() + # We now should be sitting right on the root entry's typeId + +proc close*(decoder: sink Decoder[seq[byte]]): seq[byte] = + ensureMove(decoder.stream) + +proc init*(_: typedesc[Decoder], data: openArray[byte]): Decoder[openArray[byte]] = + ## Non heap allocating version of the decoder uses preallocated memory that must outlive the structure + result = Decoder[openArray[byte]](stream: toUnsafeView data) + result.readHeader() + +proc deserialize*(dec: var Decoder, i: var LebEncodedInt) = + let (typ, _) = dec.typeNamePair() + canConvertFrom(typ, i, dec.pos) + dec.pos += dec.data.readLeb128(i) + +proc deserialize*(dec: var Decoder, f: var SomeFloat) = + let (typ, _) = dec.typeNamePair() + canConvertFrom(typ, f, dec.pos) + var val = when f is float32: 0i32 else: 0i64 + if dec.data.read(val): + dec.pos += sizeof(val) + f = cast[typeof(f)](val) + else: + raise incorrectData("Could not read a float", dec.pos) + +proc deserialize*(dec: var Decoder, str: var string) = + let (typ, _) = dec.typeNamePair() + canConvertFrom(typ, str, dec.pos) + var ind = 0 + dec.pos += dec.data.readLeb128(ind) + + if ind notin 0 .. dec.strs.high: + # It has not been read into yet + dec.readString() + + str = dec.getStr(ind) + +proc deserialize*[Idx, T](dec: var Decoder, arr: var array[Idx, T]) = + let (typ, _) = dec.typeNamePair() + canConvertFrom(typ, arr, dec.pos) + var len = 0 + dec.pos += dec.data.readLeb128(len) + if len > arr.len: + raise incorrectData( + "Expected an array with a length less than or equal to '" & $arr.len & + "', but got length of '" & $len & "'.", + dec.pos, + ) + for i in 0 ..< len: + dec.deserialize(arr[Idx(Idx.low.ord + i)]) + +proc deserialize*[T](dec: var Decoder, arr: var seq[T]) = + let (typ, _) = dec.typeNamePair() + canConvertFrom(typ, arr, dec.pos) + var len = 0 + dec.pos += dec.data.readLeb128(len) + arr = newSeq[T](len) + for i in 0 ..< len: + dec.deserialize(arr[i]) + +proc skipToEndOfStructImpl(dec: var Decoder) = + var (typ, ind) = dec.typeNamePair() + + if ind.isSome: + debug "Skipping over field ", dec.strs[ind.get], " with type of ", typ + + case typ + of Bool: + inc dec.pos + of Int8 .. Int64: + var i = 0 + dec.pos += dec.data.readLeb128(i) + of Float32: + dec.pos += sizeof(float32) + of Float64: + dec.pos += sizeof(float64) + of String: + dec.readString() + of Array: + var len = 0 + dec.pos += dec.data.readLeb128(len) + debug "Skipping array of size ", len + for i in 0 ..< len: + dec.skipToEndOfStructImpl() + of Struct: + while not (dec.atEnd) and (var (typ, _) = dec.peekTypeNamePair(); typ) != EndStruct: + dec.skipToEndOfStructImpl() + of EndStruct: + discard + of Option: + let isOpt = dec.data[0].bool + inc dec.pos + if isOpt: + dec.skipToEndOfStructImpl() + +proc skipToEndOfStruct(dec: var Decoder) = + dec.skipToEndOfStructImpl() + + if (let (typ, _) = dec.peekTypeNamePair(); typ != EndStruct): + raise incorrectData("Cannot continue skipping over field.", dec.pos) + +proc deserialize*[T: object | tuple](dec: var Decoder, obj: var T) = + mixin deserialize + var (typ, nameInd) = dec.typeNamePair() + if nameInd.isSome: + debug "Deserialising struct: ", dec.strs[nameInd.get] + canConvertFrom(typ, obj, dec.pos) + + when compiles(obj = default(T)): + obj = default(T) + + while not (dec.atEnd) and (var (typ, name) = dec.peekTypeNamePair(); typ) != EndStruct: + if name == "": + raise incorrectData("Expected field name.", dec.pos) + + debug "Deserializing field: ", name + + var found = false + for fieldName, field in obj.fieldPairs: + const realName {.used.} = + when field.hasCustomPragma(vsbfName): + field.getCustomPragmaVal(vsbfName) + else: + fieldName + + when not field.hasCustomPragma(skipSerialization): + if realName == name: + found = true + debug "Deserializing ", astToStr(field), " for ", T + {.cast(uncheckedAssign).}: + when compiles(reset field): + reset field + dec.deserialize(field) + break + + if not found: + dec.skipToEndOfStruct() + + debug "End of struct ", T + if (let (typ, _) = dec.typeNamePair(); typ) != EndStruct: + # Pops the end and ensures it's correct' + raise incorrectData("Invalid struct expected EndStruct.", dec.pos) + +proc deserialize*(dec: var Decoder, data: var (distinct)) = + dec.deserialize(distinctBase(data)) + +proc deserialize*[T](dec: var Decoder, data: var set[T]) = + const setSize = sizeof(data) + when setSize == 1: + dec.deserialize(cast[ptr uint8](data.addr)[]) + elif setSize == 2: + dec.deserialize(cast[ptr uint16](data.addr)[]) + elif setSize == 4: + dec.deserialize(cast[ptr uint32](data.addr)[]) + elif setSize == 8: + dec.deserialize(cast[ptr uint64](data.addr)[]) + else: + dec.deserialize(cast[ptr array[setSize, byte]](data.addr)[]) + +proc deserialize*[T: bool | char | int8 | uint8 and not range]( + dec: var Decoder, data: var T +) = + let (typ, _) = dec.typeNamePair() + canConvertFrom(typ, data, dec.pos) + data = cast[T](dec.data[0]) + inc dec.pos + +proc deserialize*[T: enum](dec: var Decoder, data: var T) = + let (typ, _) = dec.typeNamePair() + canConvertFrom(typ, data, dec.pos) + + var base = 0i64 + let pos = dec.pos + dec.pos += dec.data.readLeb128(base) + if base notin T.low.ord .. T.high.ord: + raise typeMismatch(fmt"Cannot convert '{base}' to '{$T}'.", pos) + + data = T(base) + +proc deserialize*[T: range](dec: var Decoder, data: var T) = + var base = default T.rangeBase() + let pos = dec.pos + dec.deserialize(base) + if base notin T.low .. T.high: + raise typeMismatch( + fmt"Cannot convert to range got '{base}', but expected value in '{$T}'.", pos + ) + + data = T(base) + +proc deserialize*[T](dec: var Decoder, data: var Option[T]) = + let (typ, nameInd) = dec.typeNamePair() + canConvertFrom(typ, data, dec.pos) + let isOpt = dec.data[0].bool + dec.pos += 1 + if isOpt: + var val = default(T) + dec.deserialize(val) + data = some(val) + else: + data = none(T) + +proc deserialize*(dec: var Decoder, data: var ref) = + let (typ, _) = dec.typeNamePair() + canConvertFrom(typ, data, dec.pos) + let isRef = dec.data[0].bool + dec.pos += 1 + if isRef: + new data + dec.deserialize(data[]) + +proc deserialize*(dec: var Decoder, T: typedesc): T = + dec.deserialize(result)
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/semicongine/thirdparty/vsbf/vsbf/encoders.nim Wed Mar 26 00:37:50 2025 +0700 @@ -0,0 +1,150 @@ +import std/[options, typetraits, tables, macros] +import shared + +type + Encoder*[DataType: SeqOrArr[byte]] = object + strs: Table[string, int] + when DataType is seq[byte]: + dataBuffer*: seq[byte] + else: + dataBuffer*: UnsafeView[byte] + dataPos*: int + +template offsetDataBuffer*(encoder: Encoder[openArray[byte]]): untyped = + encoder.dataBuffer.toOpenArray(encoder.dataPos) + +template data*[T](encoder: Encoder[T]): untyped = + when T is seq: + encoder.databuffer.toOpenArray(0, encoder.dataBuffer.high) + else: + encoder.dataBuffer.toOpenArray(0, encoder.dataPos - 1) + +proc writeTo*[T](encoder: var Encoder[T], toWrite: SomeInteger) = + when T is seq: + discard encoder.dataBuffer.write(toWrite) + else: + encoder.dataPos += encoder.offsetDataBuffer().write toWrite + +proc writeTo*[T](encoder: var Encoder[T], toWrite: openArray[byte]) = + when T is seq: + encoder.dataBuffer.add toWrite + else: + for i, x in toWrite: + encoder.offsetDataBuffer()[i] = x + encoder.dataPos += toWrite.len + +proc close*(encoder: sink Encoder): seq[byte] = ensureMove encoder.databuffer + +proc init*( + _: typedesc[Encoder], dataBuffer: openArray[byte] +): Encoder[openArray[byte]] = + Encoder[openArray[byte]]( + dataBuffer: dataBuffer.toUnsafeView(), + ) + +proc init*(_: typedesc[Encoder]): Encoder[seq[byte]] = + result = + Encoder[seq[byte]]( + dataBuffer: newSeqOfCap[byte](256), + ) + result.dataBuffer.add cast[array[headerSize, byte]](header) + + +proc cacheStr*(encoder: var Encoder, str: sink string) = + ## Writes the string to the buffer + ## If the string has not been seen yet it'll print Index Len StringData to cache it + withValue encoder.strs, str, val: + let (data, len) = leb128 val[] + encoder.writeTo data.toOpenArray(0, len - 1) + do: + var (data, len) = leb128 encoder.strs.len + encoder.writeTo(data.toOpenArray(0, len - 1)) + (data, len) = leb128 str.len + encoder.writeTo(data.toOpenArray(0, len - 1)) + encoder.writeTo(str.toOpenArrayByte(0, str.high)) + encoder.strs[str] = encoder.strs.len + +proc serializeTypeInfo[T](encoder: var Encoder, val: T, name: sink string) = + ## Stores the typeID and name if it's required(0b1xxx_xxxx if there is a name) + encoder.writeTo T.vsbfId.encoded(name.len > 0) + if name.len > 0: + encoder.cacheStr(name) + +proc serialize*(encoder: var Encoder, i: LebEncodedInt, name: string) = + serializeTypeInfo(encoder, i, name) + let (data, len) = leb128 i + encoder.writeTo data.toOpenArray(0, len - 1) + +proc serialize*(encoder: var Encoder, val: bool | char | uint8 | int8, name: string) = + serializeTypeInfo(encoder, val, name) + encoder.writeTo cast[byte](val) + +proc serialize*(encoder: var Encoder, i: enum, name: string) = + encoder.serialize(int64(i), name) + +proc serialize*(encoder: var Encoder, f: SomeFloat, name: string) = + serializeTypeInfo(encoder, f, name) + when f is float32: + encoder.writeTo cast[int32](f) + else: + encoder.writeTo cast[int64](f) + +proc serialize*(encoder: var Encoder, str: string, name: string) = + serializeTypeInfo(encoder, str, name) + encoder.cacheStr(str) + +proc serialize*[T](encoder: var Encoder, arr: openArray[T], name: string) = + serializeTypeInfo(encoder, arr, name) + let (data, len) = leb128 arr.len + encoder.writeTo data.toOpenArray(0, len - 1) + for val in arr.items: + encoder.serialize(val, "") + +proc serialize*[T: object | tuple](encoder: var Encoder, obj: T, name: string) = + mixin serialize + serializeTypeInfo(encoder, obj, name) + for fieldName, field in obj.fieldPairs: + const realName {.used.} = + when field.hasCustomPragma(vsbfName): + field.getCustomPragmaVal(vsbfName) + else: + fieldName + when not field.hasCustomPragma(skipSerialization): + encoder.serialize(field, realName) + + encoder.writeTo EndStruct.encoded(false) + + +proc serialize*(encoder: var Encoder, data: ref, name: string) = + serializeTypeInfo(encoder, data, name) + encoder.writeTo byte(data != nil) + if data != nil: + encoder.serialize(data[], "") + +proc serialize*[T: Option](encoder: var Encoder, data: T, name: string) = + serializeTypeInfo(encoder, data, name) + encoder.writeTo byte(data.isSome) + if data.isSome: + encoder.serialize(data.unsafeGet, "") + +proc serialize*(encoder: var Encoder, data: distinct, name: string) = + encoder.serialize(distinctBase(data), name) + +proc serialize*[T: range](encoder: var Encoder, data: T, name: string) = + encoder.serialize((T.rangeBase) data, name) + +proc serialize*[T](encoder: var Encoder, data: set[T], name: string) = + const setSize = sizeof(data) + when setSize == 1: + encoder.serialize(cast[uint8](data), name) + elif setSize == 2: + encoder.serialize(cast[uint16](data), name) + elif setSize == 4: + encoder.serialize(cast[uint32](data), name) + elif setSize == 8: + encoder.serialize(cast[uint64](data), name) + else: + encoder.serialize(cast[array[setSize, byte]](data), name) + +proc serializeRoot*(encoder: var Encoder, val: object or tuple) = + encoder.serialize(val, "")
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/semicongine/thirdparty/vsbf/vsbf/shared.nim Wed Mar 26 00:37:50 2025 +0700 @@ -0,0 +1,229 @@ +import std/options + +const + version* = "\1\0" + header* = ['v', 's', 'b', 'f', version[0], version[1]] + headerSize* = header.len + +template skipSerialization*() {.pragma.} +template vsbfName*(s: string) {.pragma.} + +type + LebEncodedInt* = SomeInteger and not (int8 | uint8) + + SerialisationType* = enum + Bool + Int8 + Int16 + Int32 + Int64 + Float32 ## Floats do not use `Int` types to allow transitioning to and from floats + Float64 + String ## Strings are stored in a 'String section' + Array + Struct + EndStruct # Marks we've finished reading + Option ## If the next byte is 0x1 you parse the internal otherwise you skip + + SeqOrArr*[T] = seq[T] or openarray[T] + + UnsafeView*[T] = object + data*: ptr UncheckedArray[T] + len*: int + + VsbfErrorKind* = enum + None ## No error hit + InsufficientData ## Not enough data in the buffer + IncorrectData ## Expected something else in the buffer + ExpectedField ## Expected a field but got something else + TypeMismatch ## Expected a specific type but got another + + VsbfError* = object of ValueError + kind*: VsbfErrorKind + position*: int = -1 + +static: + assert sizeof(SerialisationType) == 1 # Types are always 1 + assert SerialisationType.high.ord <= 127 + +const + VsbfTypes* = {Bool .. Option} + LowerSeven = 0b0111_1111 + MSBit = 0b1000_0000u8 + +proc insufficientData*(msg: string, position: int = -1): ref VsbfError = + (ref VsbfError)(kind: InsufficientData, msg: msg, position: position) + +proc incorrectData*(msg: string, position: int = -1): ref VsbfError = + (ref VsbfError)(kind: IncorrectData, msg: msg, position: position) + +proc expectedField*(msg: string, position: int = -1): ref VsbfError = + (ref VsbfError)(kind: ExpectedField, msg: msg, position: position) + +proc typeMismatch*(msg: string, position: int = -1): ref VsbfError = + (ref VsbfError)(kind: TypeMismatch, msg: msg, position: position) + +proc encoded*(serType: SerialisationType, storeName: bool): byte = + if storeName: # las bit reserved for 'hasName' + 0b1000_0000u8 or byte(serType) + else: + byte(serType) + +{.warning[HoleEnumConv]: off.} +proc decodeType*(data: byte, pos: int): tuple[typ: SerialisationType, hasName: bool] = + result.hasName = (MSBit and data) > 0 # Extract whether the lastbit is set + let val = (data and LowerSeven) + if val notin 0u8 .. SerialisationType.high.uint8: + raise incorrectData("Cannot decode value " & $val & " into vsbf type tag", pos) + + result.typ = SerialisationType((data and LowerSeven)) + +{.warning[HoleEnumConv]: on.} + +proc vsbfId*(_: typedesc[bool]): SerialisationType = + Bool + +proc vsbfId*(_: typedesc[int8 or uint8 or char]): SerialisationType = + Int8 + +proc vsbfId*(_: typedesc[int16 or uint16]): SerialisationType = + Int16 + +proc vsbfId*(_: typedesc[int32 or uint32]): SerialisationType = + Int32 + +proc vsbfId*(_: typedesc[int64 or uint64]): SerialisationType = + Int64 + +proc vsbfId*(_: typedesc[int or uint]): SerialisationType = + Int64 + # Always 64 bits to ensure compatibillity + +proc vsbfId*(_: typedesc[float32]): SerialisationType = + Float32 + +proc vsbfId*(_: typedesc[float64]): SerialisationType = + Float64 + +proc vsbfId*(_: typedesc[string]): SerialisationType = + String + +proc vsbfId*(_: typedesc[openArray]): SerialisationType = + Array + +proc vsbfId*(_: typedesc[object or tuple]): SerialisationType = + Struct + +proc vsbfId*(_: typedesc[ref]): SerialisationType = + Option + +proc vsbfId*(_: typedesc[enum]): SerialisationType = + Int64 + +proc vsbfId*(_: typedesc[Option]): SerialisationType = + Option + +proc canConvertFrom*(typ: SerialisationType, val: auto, pos: int) = + var expected = typeof(val).vsbfId() + if typ != expected: + raise typeMismatch("Expected: " & $expected & " but got " & $typ, pos) + +proc toUnsafeView*[T](oa: openArray[T]): UnsafeView[T] = + UnsafeView[T](data: cast[ptr UncheckedArray[T]](oa[0].addr), len: oa.len) + +template toOa*[T](view: UnsafeView[T]): auto = + view.data.toOpenArray(0, view.len - 1) + +template toOpenArray*[T](view: UnsafeView[T], low, high: int): auto = + view.data.toOpenArray(low, high) + +template toOpenArray*[T](view: UnsafeView[T], low: int): auto = + view.data.toOpenArray(low, view.len - 1) + +template toOpenArray*[T](oa: openArray[T], low: int): auto = + oa.toOpenArray(low, oa.len - 1) + +proc write*(oa: var openArray[byte], toWrite: SomeInteger): int = + if oa.len > sizeof(toWrite): + result = sizeof(toWrite) + for offset in 0 ..< sizeof(toWrite): + oa[offset] = byte(toWrite shr (offset * 8) and 0xff) + +proc write*(sq: var seq[byte], toWrite: SomeInteger): int = + result = sizeof(toWrite) + for offset in 0 ..< sizeof(toWrite): + sq.add byte((toWrite shr (offset * 8)) and 0xff) + +template doWhile(cond: bool, body: untyped) = + body + while cond: + body + +proc writeLeb128*(buffer: var openArray[byte], i: SomeUnsignedInt): int = + var val = uint64(i) + doWhile(val != 0): + var data = byte(val and LowerSeven) + val = val shr 7 + if val != 0: + data = MSBit or data + buffer[result] = data + inc result + if result > buffer.len: + raise insufficientData("Not enough space to encode an unsigned leb128 integer.") + +proc writeLeb128*[T: SomeSignedInt](buffer: var openArray[byte], i: T): int = + var + val = i + more = true + + while more: + var data = byte(val and T(LowerSeven)) + val = val shr 7 + + let isSignSet = (0x40 and data) == 0x40 + if (val == 0 and not isSignSet) or (val == -1 and isSignSet): + more = false + else: + data = MSBit or data + + buffer[result] = data + + inc result + if result > buffer.len and more: + raise insufficientData("Not enough space to encode a signed leb128 integer") + +proc readLeb128*[T: SomeUnsignedInt](data: openArray[byte], val: var T): int = + var shift = T(0) + + while true: + if result > data.len: + raise incorrectData("Attempting to read a too large integer") + let theByte = data[result] + val = val or (T(theByte and LowerSeven) shl shift) + inc result + + if (MSBit and theByte) != MSBit: + break + shift += 7 + +proc readLeb128*[T: SomeSignedInt](data: openArray[byte], val: var T): int = + var + shift = T(0) + theByte = 0u8 + + while (MSBit and theByte) == MSBit or result == 0: + if result > data.len: + raise incorrectData("Attempting to read a too large integer") + + theByte = data[result] + val = val or (T(theByte and LowerSeven) shl shift) + shift += 7 + inc result + + if (shift < T(sizeof(T) * 8)) and (theByte and 0x40) == 0x40: + val = val or (not (T(0)) shl shift) + +proc leb128*(i: SomeInteger): (array[16, byte], int) = + var data = default array[16, byte] + let len = data.writeLeb128(i) + (data, len)
--- a/tests/test_storage.nim Mon Mar 24 22:57:47 2025 +0700 +++ b/tests/test_storage.nim Wed Mar 26 00:37:50 2025 +0700 @@ -15,20 +15,28 @@ assert storage.load(KEY, 0) == TEST_VALUE proc testWorldAPI() = + type Obj1 = object + value: int + + type Obj2 = object + value: string + assert listWorlds().len == 0 - "testWorld".storeWorld(42) + const obj1 = Obj1(value: 42) + "testWorld".storeWorld(obj1) assert listWorlds() == @["testWorld"] - assert loadWorld[int]("testWorld") == 42 + assert loadWorld[Obj1]("testWorld") == obj1 - "testWorld".storeWorld("hello") + const obj2 = Obj2(value: "Hello world") + "testWorld".storeWorld(obj2) assert listWorlds() == @["testWorld"] - assert loadWorld[string]("testWorld") == "hello" + assert loadWorld[Obj2]("testWorld") == obj2 - "earth".storeWorld("hello") + "earth".storeWorld(obj2) assert "earth" in listWorlds() assert "testWorld" in listWorlds() - assert loadWorld[string]("earth") == "hello" + assert loadWorld[Obj2]("earth") == obj2 "earth".purgeWorld() assert listWorlds() == @["testWorld"]