changeset 1465:7b2ec9f3d0f6 default tip

add: external serialization library
author sam <sam@basx.dev>
date Wed, 26 Mar 2025 00:37:50 +0700
parents 3e3192241ea7
children
files semicongine/platform/linux/rendering.nim semicongine/storage.nim semicongine/thirdparty/vsbf/LICENSE semicongine/thirdparty/vsbf/vsbf.nim semicongine/thirdparty/vsbf/vsbf/decoders.nim semicongine/thirdparty/vsbf/vsbf/encoders.nim semicongine/thirdparty/vsbf/vsbf/shared.nim tests/test_storage.nim
diffstat 8 files changed, 792 insertions(+), 14 deletions(-) [+]
line wrap: on
line diff
--- a/semicongine/platform/linux/rendering.nim	Mon Mar 24 22:57:47 2025 +0700
+++ b/semicongine/platform/linux/rendering.nim	Wed Mar 26 00:37:50 2025 +0700
@@ -232,7 +232,7 @@
   vec2i(attribs.width, attribs.height)
 
 # buffer to save utf8-data from keyboard events
-var unicodeData: array[64, char]
+var unicodeData = newStringOfCap(64)
 
 proc pendingEvents*(window: NativeWindow): seq[Event] =
   var event: XEvent
@@ -253,9 +253,8 @@
         key: KeyTypeMap.getOrDefault(int(event.xkey.keycode), Key.UNKNOWN),
       )
       var status: Status
-      var ksym: KeySym = NoSymbol
       let len = window.ic.Xutf8LookupString(
-        addr(event.xkey), addr(unicodeData[0]), unicodeData.len.cint, nil, addr(status)
+        addr(event.xkey), unicodeData.cstring, unicodeData.len.cint, nil, addr(status)
       )
       if len > 0 and status != XBufferOverflow:
         unicodeData[len] = '\0'
--- a/semicongine/storage.nim	Mon Mar 24 22:57:47 2025 +0700
+++ b/semicongine/storage.nim	Wed Mar 26 00:37:50 2025 +0700
@@ -10,6 +10,7 @@
 import ./core
 
 import ./thirdparty/db_connector/db_sqlite
+import ./thirdparty/vsbf/vsbf
 
 const STORAGE_NAME = Path("storage.db")
 const DEFAULT_KEY_VALUE_TABLE_NAME = "shelf"
@@ -108,28 +109,40 @@
     sql(
       &"""CREATE TABLE IF NOT EXISTS {table} (
     key INT NOT NULL UNIQUE,
-    value TEXT NOT NULL
+    value BLOB NOT NULL
   )"""
     )
   )
 
-proc storeWorld*[T](
+proc storeWorld*[T: object | tuple](
     worldName: string, world: T, table = DEFAULT_WORLD_TABLE_NAME, deleteOld = false
 ) =
   let db = worldName.ensureExists(table)
   defer:
     db.close()
   let key = $(int(now().toTime().toUnixFloat() * 1000))
-  db.exec(sql(&"""INSERT INTO {table} VALUES(?, ?)"""), key, $$world)
+
+  var encoder = Encoder.init()
+  encoder.serializeRoot(world)
+
+  let s = db.prepare(&"""INSERT INTO {table} VALUES(?, ?)""")
+  s.bindParam(1, key)
+  s.bindParam(2, encoder.data)
+  db.exec(s)
+  s.finalize()
   db.exec(sql(&"""DELETE FROM {table} WHERE key <> ?"""), key)
 
-proc loadWorld*[T](worldName: string, table = DEFAULT_WORLD_TABLE_NAME): T =
+proc loadWorld*[T: object | tuple](
+    worldName: string, table = DEFAULT_WORLD_TABLE_NAME
+): T =
   let db = worldName.ensureExists(table)
   defer:
     db.close()
   let dbResult =
     db.getValue(sql(&"""SELECT value FROM {table} ORDER BY key DESC LIMIT 1"""))
-  return to[T](dbResult)
+
+  var decoder = Decoder.init(cast[seq[byte]](dbResult))
+  decoder.deserialize(T)
 
 proc listWorlds*(): seq[string] =
   let dir = Path(getDataDir()) / Path(AppName()) / Path(WORLD_DIR)
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/semicongine/thirdparty/vsbf/LICENSE	Wed Mar 26 00:37:50 2025 +0700
@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Jason Beetham
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/semicongine/thirdparty/vsbf/vsbf.nim	Wed Mar 26 00:37:50 2025 +0700
@@ -0,0 +1,6 @@
+## This is a very simple binary format, aimed at game save serialisation.
+## By using type information encoded we can catch decode errors and optionally invoke converters
+## Say data was saved as a `Float32` but now is an `Int32` we can know this and do `int32(floatVal)`
+
+import vsbf/[shared, decoders, encoders]
+export skipSerialization, decoders, encoders
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/semicongine/thirdparty/vsbf/vsbf/decoders.nim	Wed Mar 26 00:37:50 2025 +0700
@@ -0,0 +1,352 @@
+import std/[options, typetraits, tables, macros, strformat]
+import shared
+
+type Decoder*[DataType: SeqOrArr[byte]] = object
+  strs*: seq[string] ## List of strings that are indexed by string indexes
+  when DataType is seq[byte]:
+    stream*: seq[byte]
+  else:
+    stream*: UnsafeView[byte]
+  pos*: int
+
+template debug(args: varargs[typed, `$`]): untyped =
+  when defined(vsbfDebug):
+    unpackVarargs(echo, args)
+
+proc len(dec: Decoder): int =
+  dec.stream.len
+
+proc atEnd*(dec: Decoder): bool =
+  dec.pos >= dec.len
+
+template data*[T](decoder: Decoder[T]): untyped =
+  if decoder.pos >= decoder.stream.len:
+    raise insufficientData("More data expect, but hit end of stream.", decoder.pos)
+
+  decoder.stream.toOpenArray(decoder.pos, decoder.stream.len - 1)
+
+proc read*[T: SomeInteger](oa: openArray[byte], res: var T): bool =
+  if sizeof(T) <= oa.len:
+    res = T(0)
+    for i in T(0) ..< sizeof(T):
+      res = res or (T(oa[int i]) shl (i * 8))
+
+    true
+  else:
+    false
+
+proc read*(frm: openArray[byte], to: var openArray[byte | char]): int =
+  if to.len > frm.len:
+    -1
+  else:
+    for i in 0 .. to.high:
+      to[i] = typeof(to[0])(frm[i])
+    to.len
+
+proc read*[T](dec: var Decoder[T], data: typedesc): Option[data] =
+  var val = default data
+  if dec.data.read(val) > 0:
+    some(val)
+  else:
+    none(data)
+
+proc readString*(dec: var Decoder) =
+  var strLen = 0
+  dec.pos += dec.data.readLeb128(strLen)
+
+  var buffer = newString(strLen)
+
+  for i in 0 ..< strLen:
+    buffer[i] = char dec.data[i]
+  debug "Stored string ", buffer, " at index ", dec.strs.len
+  dec.strs.add buffer
+  dec.pos += strLen
+
+proc typeNamePair*(
+    dec: var Decoder
+): tuple[typ: SerialisationType, nameInd: options.Option[int]] =
+  ## reads the type and name's string index if it has it
+  var encodedType = 0u8
+  if not dec.data.read(encodedType):
+    raise newException(VsbfError, fmt"Failed to read type info. Position: {dec.pos}")
+  dec.pos += 1
+
+  var hasName: bool
+  (result.typ, hasName) = encodedType.decodeType(dec.pos - 1)
+
+  if hasName:
+    var ind = 0
+    let indSize = dec.data.readLeb128(ind)
+    dec.pos += indSize
+    result.nameInd = some(ind)
+    if indSize > 0:
+      if ind notin 0 .. dec.strs.high:
+        dec.readString()
+    else:
+      raise
+        (ref VsbfError)(msg: fmt"No name following a declaration. Position {dec.pos}")
+
+proc peekTypeNamePair*(dec: var Decoder): tuple[typ: SerialisationType, name: string] =
+  ## peek the type and name's string index if it has it
+  let encodedType = dec.data[0]
+  var hasName: bool
+  (result.typ, hasName) = encodedType.decodeType(dec.pos - 1)
+  debug result.typ, " has name: ", hasName
+  if hasName:
+    var val = 0
+    let indSize = dec.data.toOpenArray(1).readLeb128(val)
+    debug "String of index: ", val, " found at ", dec.pos
+
+    if indSize > 0:
+      if val notin 0 .. dec.strs.high:
+        debug "String not loaded, reading it"
+        var strLen = 0
+        let
+          strLenBytes = dec.data.toOpenArray(1 + indSize).readLeb128(strLen)
+          start = 1 + indSize + strLenBytes
+        result.name = newString(strLen)
+
+        if start >= dec.len or start + strLen - 1 > dec.len:
+          raise insufficientData("Need more data for a field", dec.pos)
+
+        for i, x in dec.data.toOpenArray(start, start + strLen - 1):
+          result.name[i] = char x
+      else:
+        result.name = dec.strs[val]
+    else:
+      raise incorrectData("No name following a declaration.", dec.pos)
+
+proc getStr*(dec: Decoder, ind: int): lent string =
+  dec.strs[ind]
+
+proc readHeader(dec: var Decoder) =
+  var ext = dec.read(array[4, char])
+  if ext.isNone or ext.unsafeGet != "vsbf":
+    raise incorrectData("Not a VSBF stream, missing the header", 0)
+  dec.pos += 4
+
+  let ver = dec.read(array[2, byte])
+
+  if ver.isNone:
+    raise incorrectData("Cannot read, missing version", 4)
+
+  dec.pos += 2
+
+proc init*(_: typedesc[Decoder], data: sink seq[byte]): Decoder[seq[byte]] =
+  ## Heap allocated version, it manages it's own buffer you give it and reads from this.
+  ## Can recover the buffer using `close` after parsin
+  result = Decoder[seq[byte]](stream: data)
+  result.readHeader()
+  # We now should be sitting right on the root entry's typeId
+
+proc close*(decoder: sink Decoder[seq[byte]]): seq[byte] =
+  ensureMove(decoder.stream)
+
+proc init*(_: typedesc[Decoder], data: openArray[byte]): Decoder[openArray[byte]] =
+  ## Non heap allocating version of the decoder uses preallocated memory that must outlive the structure
+  result = Decoder[openArray[byte]](stream: toUnsafeView data)
+  result.readHeader()
+
+proc deserialize*(dec: var Decoder, i: var LebEncodedInt) =
+  let (typ, _) = dec.typeNamePair()
+  canConvertFrom(typ, i, dec.pos)
+  dec.pos += dec.data.readLeb128(i)
+
+proc deserialize*(dec: var Decoder, f: var SomeFloat) =
+  let (typ, _) = dec.typeNamePair()
+  canConvertFrom(typ, f, dec.pos)
+  var val = when f is float32: 0i32 else: 0i64
+  if dec.data.read(val):
+    dec.pos += sizeof(val)
+    f = cast[typeof(f)](val)
+  else:
+    raise incorrectData("Could not read a float", dec.pos)
+
+proc deserialize*(dec: var Decoder, str: var string) =
+  let (typ, _) = dec.typeNamePair()
+  canConvertFrom(typ, str, dec.pos)
+  var ind = 0
+  dec.pos += dec.data.readLeb128(ind)
+
+  if ind notin 0 .. dec.strs.high:
+    # It has not been read into yet
+    dec.readString()
+
+  str = dec.getStr(ind)
+
+proc deserialize*[Idx, T](dec: var Decoder, arr: var array[Idx, T]) =
+  let (typ, _) = dec.typeNamePair()
+  canConvertFrom(typ, arr, dec.pos)
+  var len = 0
+  dec.pos += dec.data.readLeb128(len)
+  if len > arr.len:
+    raise incorrectData(
+      "Expected an array with a length less than or equal to '" & $arr.len &
+        "', but got length of '" & $len & "'.",
+      dec.pos,
+    )
+  for i in 0 ..< len:
+    dec.deserialize(arr[Idx(Idx.low.ord + i)])
+
+proc deserialize*[T](dec: var Decoder, arr: var seq[T]) =
+  let (typ, _) = dec.typeNamePair()
+  canConvertFrom(typ, arr, dec.pos)
+  var len = 0
+  dec.pos += dec.data.readLeb128(len)
+  arr = newSeq[T](len)
+  for i in 0 ..< len:
+    dec.deserialize(arr[i])
+
+proc skipToEndOfStructImpl(dec: var Decoder) =
+  var (typ, ind) = dec.typeNamePair()
+
+  if ind.isSome:
+    debug "Skipping over field ", dec.strs[ind.get], " with type of ", typ
+
+  case typ
+  of Bool:
+    inc dec.pos
+  of Int8 .. Int64:
+    var i = 0
+    dec.pos += dec.data.readLeb128(i)
+  of Float32:
+    dec.pos += sizeof(float32)
+  of Float64:
+    dec.pos += sizeof(float64)
+  of String:
+    dec.readString()
+  of Array:
+    var len = 0
+    dec.pos += dec.data.readLeb128(len)
+    debug "Skipping array of size ", len
+    for i in 0 ..< len:
+      dec.skipToEndOfStructImpl()
+  of Struct:
+    while not (dec.atEnd) and (var (typ, _) = dec.peekTypeNamePair(); typ) != EndStruct:
+      dec.skipToEndOfStructImpl()
+  of EndStruct:
+    discard
+  of Option:
+    let isOpt = dec.data[0].bool
+    inc dec.pos
+    if isOpt:
+      dec.skipToEndOfStructImpl()
+
+proc skipToEndOfStruct(dec: var Decoder) =
+  dec.skipToEndOfStructImpl()
+
+  if (let (typ, _) = dec.peekTypeNamePair(); typ != EndStruct):
+    raise incorrectData("Cannot continue skipping over field.", dec.pos)
+
+proc deserialize*[T: object | tuple](dec: var Decoder, obj: var T) =
+  mixin deserialize
+  var (typ, nameInd) = dec.typeNamePair()
+  if nameInd.isSome:
+    debug "Deserialising struct: ", dec.strs[nameInd.get]
+  canConvertFrom(typ, obj, dec.pos)
+
+  when compiles(obj = default(T)):
+    obj = default(T)
+
+  while not (dec.atEnd) and (var (typ, name) = dec.peekTypeNamePair(); typ) != EndStruct:
+    if name == "":
+      raise incorrectData("Expected field name.", dec.pos)
+
+    debug "Deserializing field: ", name
+
+    var found = false
+    for fieldName, field in obj.fieldPairs:
+      const realName {.used.} =
+        when field.hasCustomPragma(vsbfName):
+          field.getCustomPragmaVal(vsbfName)
+        else:
+          fieldName
+
+      when not field.hasCustomPragma(skipSerialization):
+        if realName == name:
+          found = true
+          debug "Deserializing ", astToStr(field), " for ", T
+          {.cast(uncheckedAssign).}:
+            when compiles(reset field):
+              reset field
+            dec.deserialize(field)
+          break
+
+    if not found:
+      dec.skipToEndOfStruct()
+
+  debug "End of struct ", T
+  if (let (typ, _) = dec.typeNamePair(); typ) != EndStruct:
+    # Pops the end and ensures it's correct'
+    raise incorrectData("Invalid struct expected EndStruct.", dec.pos)
+
+proc deserialize*(dec: var Decoder, data: var (distinct)) =
+  dec.deserialize(distinctBase(data))
+
+proc deserialize*[T](dec: var Decoder, data: var set[T]) =
+  const setSize = sizeof(data)
+  when setSize == 1:
+    dec.deserialize(cast[ptr uint8](data.addr)[])
+  elif setSize == 2:
+    dec.deserialize(cast[ptr uint16](data.addr)[])
+  elif setSize == 4:
+    dec.deserialize(cast[ptr uint32](data.addr)[])
+  elif setSize == 8:
+    dec.deserialize(cast[ptr uint64](data.addr)[])
+  else:
+    dec.deserialize(cast[ptr array[setSize, byte]](data.addr)[])
+
+proc deserialize*[T: bool | char | int8 | uint8 and not range](
+    dec: var Decoder, data: var T
+) =
+  let (typ, _) = dec.typeNamePair()
+  canConvertFrom(typ, data, dec.pos)
+  data = cast[T](dec.data[0])
+  inc dec.pos
+
+proc deserialize*[T: enum](dec: var Decoder, data: var T) =
+  let (typ, _) = dec.typeNamePair()
+  canConvertFrom(typ, data, dec.pos)
+
+  var base = 0i64
+  let pos = dec.pos
+  dec.pos += dec.data.readLeb128(base)
+  if base notin T.low.ord .. T.high.ord:
+    raise typeMismatch(fmt"Cannot convert '{base}' to '{$T}'.", pos)
+
+  data = T(base)
+
+proc deserialize*[T: range](dec: var Decoder, data: var T) =
+  var base = default T.rangeBase()
+  let pos = dec.pos
+  dec.deserialize(base)
+  if base notin T.low .. T.high:
+    raise typeMismatch(
+      fmt"Cannot convert to range got '{base}', but expected value in '{$T}'.", pos
+    )
+
+  data = T(base)
+
+proc deserialize*[T](dec: var Decoder, data: var Option[T]) =
+  let (typ, nameInd) = dec.typeNamePair()
+  canConvertFrom(typ, data, dec.pos)
+  let isOpt = dec.data[0].bool
+  dec.pos += 1
+  if isOpt:
+    var val = default(T)
+    dec.deserialize(val)
+    data = some(val)
+  else:
+    data = none(T)
+
+proc deserialize*(dec: var Decoder, data: var ref) =
+  let (typ, _) = dec.typeNamePair()
+  canConvertFrom(typ, data, dec.pos)
+  let isRef = dec.data[0].bool
+  dec.pos += 1
+  if isRef:
+    new data
+    dec.deserialize(data[])
+
+proc deserialize*(dec: var Decoder, T: typedesc): T =
+  dec.deserialize(result)
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/semicongine/thirdparty/vsbf/vsbf/encoders.nim	Wed Mar 26 00:37:50 2025 +0700
@@ -0,0 +1,150 @@
+import std/[options, typetraits, tables, macros]
+import shared
+
+type
+  Encoder*[DataType: SeqOrArr[byte]] = object
+    strs: Table[string, int]
+    when DataType is seq[byte]:
+      dataBuffer*: seq[byte]
+    else:
+      dataBuffer*: UnsafeView[byte]
+      dataPos*: int
+
+template offsetDataBuffer*(encoder: Encoder[openArray[byte]]): untyped =
+  encoder.dataBuffer.toOpenArray(encoder.dataPos)
+
+template data*[T](encoder: Encoder[T]): untyped =
+  when T is seq:
+    encoder.databuffer.toOpenArray(0, encoder.dataBuffer.high)
+  else:
+    encoder.dataBuffer.toOpenArray(0, encoder.dataPos - 1)
+
+proc writeTo*[T](encoder: var Encoder[T], toWrite: SomeInteger) =
+  when T is seq:
+    discard encoder.dataBuffer.write(toWrite)
+  else:
+    encoder.dataPos += encoder.offsetDataBuffer().write toWrite
+
+proc writeTo*[T](encoder: var Encoder[T], toWrite: openArray[byte]) =
+  when T is seq:
+    encoder.dataBuffer.add toWrite
+  else:
+    for i, x in toWrite:
+      encoder.offsetDataBuffer()[i] = x
+    encoder.dataPos += toWrite.len
+
+proc close*(encoder: sink Encoder): seq[byte] = ensureMove encoder.databuffer
+
+proc init*(
+    _: typedesc[Encoder], dataBuffer: openArray[byte]
+): Encoder[openArray[byte]] =
+  Encoder[openArray[byte]](
+    dataBuffer: dataBuffer.toUnsafeView(),
+  )
+
+proc init*(_: typedesc[Encoder]): Encoder[seq[byte]] =
+  result =
+    Encoder[seq[byte]](
+      dataBuffer: newSeqOfCap[byte](256),
+    )
+  result.dataBuffer.add cast[array[headerSize, byte]](header)
+
+
+proc cacheStr*(encoder: var Encoder, str: sink string) =
+  ## Writes the string to the buffer
+  ## If the string has not been seen yet it'll print Index Len StringData to cache it
+  withValue encoder.strs, str, val:
+    let (data, len) = leb128 val[]
+    encoder.writeTo data.toOpenArray(0, len - 1)
+  do:
+    var (data, len) = leb128 encoder.strs.len
+    encoder.writeTo(data.toOpenArray(0, len - 1))
+    (data, len) = leb128 str.len
+    encoder.writeTo(data.toOpenArray(0, len - 1))
+    encoder.writeTo(str.toOpenArrayByte(0, str.high))
+    encoder.strs[str] = encoder.strs.len
+
+proc serializeTypeInfo[T](encoder: var Encoder, val: T, name: sink string) =
+  ## Stores the typeID and name if it's required(0b1xxx_xxxx if there is a name)
+  encoder.writeTo T.vsbfId.encoded(name.len > 0)
+  if name.len > 0:
+    encoder.cacheStr(name)
+
+proc serialize*(encoder: var Encoder, i: LebEncodedInt, name: string) =
+  serializeTypeInfo(encoder, i, name)
+  let (data, len) = leb128 i
+  encoder.writeTo data.toOpenArray(0, len - 1)
+
+proc serialize*(encoder: var Encoder, val: bool | char | uint8 | int8, name: string) =
+  serializeTypeInfo(encoder, val, name)
+  encoder.writeTo cast[byte](val)
+
+proc serialize*(encoder: var Encoder, i: enum, name: string) =
+ encoder.serialize(int64(i), name)
+
+proc serialize*(encoder: var Encoder, f: SomeFloat, name: string) =
+  serializeTypeInfo(encoder, f, name)
+  when f is float32:
+    encoder.writeTo cast[int32](f)
+  else:
+    encoder.writeTo cast[int64](f)
+
+proc serialize*(encoder: var Encoder, str: string, name: string) =
+  serializeTypeInfo(encoder, str, name)
+  encoder.cacheStr(str)
+
+proc serialize*[T](encoder: var Encoder, arr: openArray[T], name: string) =
+  serializeTypeInfo(encoder, arr, name)
+  let (data, len) = leb128 arr.len
+  encoder.writeTo data.toOpenArray(0, len - 1)
+  for val in arr.items:
+    encoder.serialize(val, "")
+
+proc serialize*[T: object | tuple](encoder: var Encoder, obj: T, name: string) =
+  mixin serialize
+  serializeTypeInfo(encoder, obj, name)
+  for fieldName, field in obj.fieldPairs:
+    const realName {.used.} =
+      when field.hasCustomPragma(vsbfName):
+        field.getCustomPragmaVal(vsbfName)
+      else:
+        fieldName
+    when not field.hasCustomPragma(skipSerialization):
+      encoder.serialize(field, realName)
+
+  encoder.writeTo EndStruct.encoded(false)
+
+
+proc serialize*(encoder: var Encoder, data: ref, name: string) =
+  serializeTypeInfo(encoder, data, name)
+  encoder.writeTo byte(data != nil)
+  if data != nil:
+    encoder.serialize(data[], "")
+
+proc serialize*[T: Option](encoder: var Encoder, data: T, name: string) =
+  serializeTypeInfo(encoder, data, name)
+  encoder.writeTo byte(data.isSome)
+  if data.isSome:
+    encoder.serialize(data.unsafeGet, "")
+
+proc serialize*(encoder: var Encoder, data: distinct, name: string) =
+  encoder.serialize(distinctBase(data), name)
+
+proc serialize*[T: range](encoder: var Encoder, data: T, name: string) =
+  encoder.serialize((T.rangeBase) data, name)
+
+proc serialize*[T](encoder: var Encoder, data: set[T], name: string) =
+  const setSize = sizeof(data)
+  when setSize == 1:
+    encoder.serialize(cast[uint8](data), name)
+  elif setSize == 2:
+    encoder.serialize(cast[uint16](data), name)
+  elif setSize == 4:
+    encoder.serialize(cast[uint32](data), name)
+  elif setSize == 8:
+    encoder.serialize(cast[uint64](data), name)
+  else:
+    encoder.serialize(cast[array[setSize, byte]](data), name)
+
+proc serializeRoot*(encoder: var Encoder, val: object or tuple) =
+  encoder.serialize(val, "")
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/semicongine/thirdparty/vsbf/vsbf/shared.nim	Wed Mar 26 00:37:50 2025 +0700
@@ -0,0 +1,229 @@
+import std/options
+
+const
+  version* = "\1\0"
+  header* = ['v', 's', 'b', 'f', version[0], version[1]]
+  headerSize* = header.len
+
+template skipSerialization*() {.pragma.}
+template vsbfName*(s: string) {.pragma.}
+
+type
+  LebEncodedInt* = SomeInteger and not (int8 | uint8)
+
+  SerialisationType* = enum
+    Bool
+    Int8
+    Int16
+    Int32
+    Int64
+    Float32 ## Floats do not use `Int` types to allow transitioning to and from floats
+    Float64
+    String ## Strings are stored in a 'String section'
+    Array
+    Struct
+    EndStruct # Marks we've finished reading
+    Option ## If the next byte is 0x1 you parse the internal otherwise you skip
+
+  SeqOrArr*[T] = seq[T] or openarray[T]
+
+  UnsafeView*[T] = object
+    data*: ptr UncheckedArray[T]
+    len*: int
+
+  VsbfErrorKind* = enum
+    None ## No error hit
+    InsufficientData ## Not enough data in the buffer
+    IncorrectData ## Expected something else in the buffer
+    ExpectedField ## Expected a field but got something else
+    TypeMismatch ## Expected a specific type but got another
+
+  VsbfError* = object of ValueError
+    kind*: VsbfErrorKind
+    position*: int = -1
+
+static:
+  assert sizeof(SerialisationType) == 1 # Types are always 1
+  assert SerialisationType.high.ord <= 127
+
+const
+  VsbfTypes* = {Bool .. Option}
+  LowerSeven = 0b0111_1111
+  MSBit = 0b1000_0000u8
+
+proc insufficientData*(msg: string, position: int = -1): ref VsbfError =
+  (ref VsbfError)(kind: InsufficientData, msg: msg, position: position)
+
+proc incorrectData*(msg: string, position: int = -1): ref VsbfError =
+  (ref VsbfError)(kind: IncorrectData, msg: msg, position: position)
+
+proc expectedField*(msg: string, position: int = -1): ref VsbfError =
+  (ref VsbfError)(kind: ExpectedField, msg: msg, position: position)
+
+proc typeMismatch*(msg: string, position: int = -1): ref VsbfError =
+  (ref VsbfError)(kind: TypeMismatch, msg: msg, position: position)
+
+proc encoded*(serType: SerialisationType, storeName: bool): byte =
+  if storeName: # las bit reserved for 'hasName'
+    0b1000_0000u8 or byte(serType)
+  else:
+    byte(serType)
+
+{.warning[HoleEnumConv]: off.}
+proc decodeType*(data: byte, pos: int): tuple[typ: SerialisationType, hasName: bool] =
+  result.hasName = (MSBit and data) > 0 # Extract whether the lastbit is set
+  let val = (data and LowerSeven)
+  if val notin 0u8 .. SerialisationType.high.uint8:
+    raise incorrectData("Cannot decode value " & $val & " into vsbf type tag", pos)
+
+  result.typ = SerialisationType((data and LowerSeven))
+
+{.warning[HoleEnumConv]: on.}
+
+proc vsbfId*(_: typedesc[bool]): SerialisationType =
+  Bool
+
+proc vsbfId*(_: typedesc[int8 or uint8 or char]): SerialisationType =
+  Int8
+
+proc vsbfId*(_: typedesc[int16 or uint16]): SerialisationType =
+  Int16
+
+proc vsbfId*(_: typedesc[int32 or uint32]): SerialisationType =
+  Int32
+
+proc vsbfId*(_: typedesc[int64 or uint64]): SerialisationType =
+  Int64
+
+proc vsbfId*(_: typedesc[int or uint]): SerialisationType =
+  Int64
+  # Always 64 bits to ensure compatibillity
+
+proc vsbfId*(_: typedesc[float32]): SerialisationType =
+  Float32
+
+proc vsbfId*(_: typedesc[float64]): SerialisationType =
+  Float64
+
+proc vsbfId*(_: typedesc[string]): SerialisationType =
+  String
+
+proc vsbfId*(_: typedesc[openArray]): SerialisationType =
+  Array
+
+proc vsbfId*(_: typedesc[object or tuple]): SerialisationType =
+  Struct
+
+proc vsbfId*(_: typedesc[ref]): SerialisationType =
+  Option
+
+proc vsbfId*(_: typedesc[enum]): SerialisationType =
+  Int64
+
+proc vsbfId*(_: typedesc[Option]): SerialisationType =
+  Option
+
+proc canConvertFrom*(typ: SerialisationType, val: auto, pos: int) =
+  var expected = typeof(val).vsbfId()
+  if typ != expected:
+    raise typeMismatch("Expected: " & $expected & " but got " & $typ, pos)
+
+proc toUnsafeView*[T](oa: openArray[T]): UnsafeView[T] =
+  UnsafeView[T](data: cast[ptr UncheckedArray[T]](oa[0].addr), len: oa.len)
+
+template toOa*[T](view: UnsafeView[T]): auto =
+  view.data.toOpenArray(0, view.len - 1)
+
+template toOpenArray*[T](view: UnsafeView[T], low, high: int): auto =
+  view.data.toOpenArray(low, high)
+
+template toOpenArray*[T](view: UnsafeView[T], low: int): auto =
+  view.data.toOpenArray(low, view.len - 1)
+
+template toOpenArray*[T](oa: openArray[T], low: int): auto =
+  oa.toOpenArray(low, oa.len - 1)
+
+proc write*(oa: var openArray[byte], toWrite: SomeInteger): int =
+  if oa.len > sizeof(toWrite):
+    result = sizeof(toWrite)
+    for offset in 0 ..< sizeof(toWrite):
+      oa[offset] = byte(toWrite shr (offset * 8) and 0xff)
+
+proc write*(sq: var seq[byte], toWrite: SomeInteger): int =
+  result = sizeof(toWrite)
+  for offset in 0 ..< sizeof(toWrite):
+    sq.add byte((toWrite shr (offset * 8)) and 0xff)
+
+template doWhile(cond: bool, body: untyped) =
+  body
+  while cond:
+    body
+
+proc writeLeb128*(buffer: var openArray[byte], i: SomeUnsignedInt): int =
+  var val = uint64(i)
+  doWhile(val != 0):
+    var data = byte(val and LowerSeven)
+    val = val shr 7
+    if val != 0:
+      data = MSBit or data
+    buffer[result] = data
+    inc result
+    if result > buffer.len:
+      raise insufficientData("Not enough space to encode an unsigned leb128 integer.")
+
+proc writeLeb128*[T: SomeSignedInt](buffer: var openArray[byte], i: T): int =
+  var
+    val = i
+    more = true
+
+  while more:
+    var data = byte(val and T(LowerSeven))
+    val = val shr 7
+
+    let isSignSet = (0x40 and data) == 0x40
+    if (val == 0 and not isSignSet) or (val == -1 and isSignSet):
+      more = false
+    else:
+      data = MSBit or data
+
+    buffer[result] = data
+
+    inc result
+    if result > buffer.len and more:
+      raise insufficientData("Not enough space to encode a signed leb128 integer")
+
+proc readLeb128*[T: SomeUnsignedInt](data: openArray[byte], val: var T): int =
+  var shift = T(0)
+
+  while true:
+    if result > data.len:
+      raise incorrectData("Attempting to read a too large integer")
+    let theByte = data[result]
+    val = val or (T(theByte and LowerSeven) shl shift)
+    inc result
+
+    if (MSBit and theByte) != MSBit:
+      break
+    shift += 7
+
+proc readLeb128*[T: SomeSignedInt](data: openArray[byte], val: var T): int =
+  var
+    shift = T(0)
+    theByte = 0u8
+
+  while (MSBit and theByte) == MSBit or result == 0:
+    if result > data.len:
+      raise incorrectData("Attempting to read a too large integer")
+
+    theByte = data[result]
+    val = val or (T(theByte and LowerSeven) shl shift)
+    shift += 7
+    inc result
+
+  if (shift < T(sizeof(T) * 8)) and (theByte and 0x40) == 0x40:
+    val = val or (not (T(0)) shl shift)
+
+proc leb128*(i: SomeInteger): (array[16, byte], int) =
+  var data = default array[16, byte]
+  let len = data.writeLeb128(i)
+  (data, len)
--- a/tests/test_storage.nim	Mon Mar 24 22:57:47 2025 +0700
+++ b/tests/test_storage.nim	Wed Mar 26 00:37:50 2025 +0700
@@ -15,20 +15,28 @@
   assert storage.load(KEY, 0) == TEST_VALUE
 
 proc testWorldAPI() =
+  type Obj1 = object
+    value: int
+
+  type Obj2 = object
+    value: string
+
   assert listWorlds().len == 0
 
-  "testWorld".storeWorld(42)
+  const obj1 = Obj1(value: 42)
+  "testWorld".storeWorld(obj1)
   assert listWorlds() == @["testWorld"]
-  assert loadWorld[int]("testWorld") == 42
+  assert loadWorld[Obj1]("testWorld") == obj1
 
-  "testWorld".storeWorld("hello")
+  const obj2 = Obj2(value: "Hello world")
+  "testWorld".storeWorld(obj2)
   assert listWorlds() == @["testWorld"]
-  assert loadWorld[string]("testWorld") == "hello"
+  assert loadWorld[Obj2]("testWorld") == obj2
 
-  "earth".storeWorld("hello")
+  "earth".storeWorld(obj2)
   assert "earth" in listWorlds()
   assert "testWorld" in listWorlds()
-  assert loadWorld[string]("earth") == "hello"
+  assert loadWorld[Obj2]("earth") == obj2
 
   "earth".purgeWorld()
   assert listWorlds() == @["testWorld"]