Mercurial > games > semicongine

changeset 438:1ab09f8cc68d

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
did: refactor and improve packaging API
author Sam <sam@basx.dev>
date Wed, 14 Feb 2024 21:58:43 +0700
parents e75c6da1d261
children 27c0c60fc792
files config.nims semicongine/build.nim semicongine/core/buildconfig.nim semicongine/material.nim semicongine/renderer.nim semicongine/resources.nim tests/resources/mod1/aSubdir/moreSubdir/superSubdir/secret.txt tests/test_resources.nim
diffstat 8 files changed, 148 insertions(+), 69 deletions(-) [+]
line wrap: on
line diff
--- a/config.nims	Sun Feb 11 19:26:07 2024 +0700
+++ b/config.nims	Wed Feb 14 21:58:43 2024 +0700
@@ -8,7 +8,7 @@
 const LINUX = "linux"
 const WINDOWS = "windows"
 
-const BUNDLETYPE* {.strdefine.}: string = "dir" # dir, zip, exe
+const PACKAGETYPE* {.strdefine.}: string = "dir" # dir, zip, exe
 const RESOURCEROOT* {.strdefine.}: string = "resources"
 
 switch("d", "nimPreviewHashRef")
@@ -17,7 +17,7 @@
 switch("nimblePath", "nimbledeps/pkgs2")
 
 task build, "build":
-  switch("d", "BUNDLETYPE=" & BUNDLETYPE)
+  switch("d", "PACKAGETYPE=" & PACKAGETYPE)
   switch("d", "RESOURCEROOT=" & RESOURCEROOT)
   var buildType = DEBUG
   var platformDir = ""
@@ -41,9 +41,9 @@
   let resourcedir = joinPath(projectDir(), RESOURCEROOT)
   if dirExists(resourcedir):
     let outdir_resources = joinPath(outdir, RESOURCEROOT)
-    if BUNDLETYPE == "dir":
+    if PACKAGETYPE == "dir":
       cpDir(resourcedir, outdir_resources)
-    elif BUNDLETYPE == "zip":
+    elif PACKAGETYPE == "zip":
       mkDir(outdir_resources)
       for resource in listDirs(resourcedir):
         let
@@ -70,9 +70,13 @@
 
 task test_all, "Run all test programs":
   for file in listFiles("tests"):
-    if file.endsWith(".nim"):
+    if file.endsWith(".nim") and not file.endsWith("test_resources.nim"):
       exec(&"nim build --run {file}")
 
+  exec("nim build -d:BUILD_RESOURCEROOT=tests/resources -d:PACKAGETYPE=dir --run tests/test_resources.nim")
+  exec("nim build -d:BUILD_RESOURCEROOT=tests/resources -d:PACKAGETYPE=zip --run tests/test_resources.nim")
+  exec("nim build -d:BUILD_RESOURCEROOT=tests/resources -d:PACKAGETYPE=exe --run tests/test_resources.nim")
+
 task clean, "remove all build files":
   exec(&"rm -rf {BUILDBASE}")
 
@@ -125,6 +129,6 @@
 
 if getCommand() in ["c", "compile", "r", "dump", "check", "idetools"]:
   if defined(linux):
-    --d:VK_USE_PLATFORM_XLIB_KHR
+    --d: VK_USE_PLATFORM_XLIB_KHR
   if defined(windows):
-    --d:VK_USE_PLATFORM_WIN32_KHR
+    --d: VK_USE_PLATFORM_WIN32_KHR
--- a/semicongine/build.nim	Sun Feb 11 19:26:07 2024 +0700
+++ b/semicongine/build.nim	Wed Feb 14 21:58:43 2024 +0700
@@ -30,7 +30,7 @@
   switch("outdir", semicongine_builddir(buildname, builddir = builddir))
 
 proc semicongine_pack*(outdir: string, bundleType: string, resourceRoot: string) =
-  switch("define", "BUNDLETYPE=" & bundleType)
+  switch("define", "PACKAGETYPE=" & bundleType)
 
   rmDir(outdir)
   mkDir(outdir)
--- a/semicongine/core/buildconfig.nim	Sun Feb 11 19:26:07 2024 +0700
+++ b/semicongine/core/buildconfig.nim	Wed Feb 14 21:58:43 2024 +0700
@@ -46,6 +46,6 @@
 const ENGINE_LOGLEVEL* = parseEnum[Level](LOGLEVEL)
 
 # resource bundleing settings, need to be configured per project
-const BUNDLETYPE* {.strdefine.}: string = "" # dir, zip, exe
+const PACKAGETYPE* {.strdefine.}: string = "" # dir, zip, exe
 static:
-  assert BUNDLETYPE in ["dir", "zip", "exe"], ENGINENAME & " requires one of -d:BUNDLETYPE=dir -d:BUNDLETYPE=zip -d:BUNDLETYPE=exe"
+  assert PACKAGETYPE in ["dir", "zip", "exe"], ENGINENAME & " requires one of -d:PACKAGETYPE=dir -d:PACKAGETYPE=zip -d:PACKAGETYPE=exe"
--- a/semicongine/material.nim	Sun Feb 11 19:26:07 2024 +0700
+++ b/semicongine/material.nim	Wed Feb 14 21:58:43 2024 +0700
@@ -131,7 +131,7 @@
   EMPTY_SHADER* = createShaderConfiguration(
     inputs = [
       attr[Mat4](TRANSFORM_ATTRIB, memoryPerformanceHint = PreferFastWrite, perInstance = true),
-      attr[Vec3f]("position", memoryPerformanceHint = PreferFastWrite),
+      attr[Vec3f]("position", memoryPerformanceHint = PreferFastRead),
     ],
     outputs = [attr[Vec4f]("color")],
     vertexCode = &"gl_Position = vec4(position, 1.0) * {TRANSFORM_ATTRIB};",
--- a/semicongine/renderer.nim	Sun Feb 11 19:26:07 2024 +0700
+++ b/semicongine/renderer.nim	Wed Feb 14 21:58:43 2024 +0700
@@ -153,8 +153,6 @@
 
   renderer.checkSceneIntegrity(scene)
 
-  let inputs = renderer.inputs(scene)
-
   # create index buffer if necessary
   var indicesBufferSize = 0
   for mesh in scene.meshes:
@@ -181,6 +179,9 @@
   var perLocationSizes: Table[MemoryPerformanceHint, int]
   for hint in MemoryPerformanceHint:
     perLocationSizes[hint] = 0
+
+  let inputs = renderer.inputs(scene)
+
   for attribute in inputs:
     scenedata.attributeLocation[attribute.name] = attribute.memoryPerformanceHint
     # setup one buffer per attribute-location-type
--- a/semicongine/resources.nim	Sun Feb 11 19:26:07 2024 +0700
+++ b/semicongine/resources.nim	Wed Feb 14 21:58:43 2024 +0700
@@ -1,8 +1,10 @@
 import std/streams
+import std/algorithm
 import std/strutils
 import std/sequtils
 import std/strformat
 import std/os
+import std/sets
 import std/unicode
 
 import ./core
@@ -24,24 +26,33 @@
     Exe # Embeded in executable
 
 const
-  thebundletype = parseEnum[ResourceBundlingType](BUNDLETYPE.toLowerAscii().capitalizeAscii())
+  thebundletype = parseEnum[ResourceBundlingType](PACKAGETYPE.toLowerAscii().capitalizeAscii())
   ASCII_CHARSET = PrintableChars.toSeq.toRunes
-
-var selectedMod* = "default"
+  DEFAULT_PACKAGE = "default"
 
 # resource loading
 
+func normalizeDir(dir: string): string =
+  result = dir
+  if result.startsWith("./"):
+    result = result[2 .. ^1]
+  if result.startsWith("/"):
+    result = result[1 .. ^1]
+  result = dir.replace('\\', '/')
+  if not result.endsWith("/") and result != "":
+    result = result & "/"
+
 when thebundletype == Dir:
 
   proc resourceRoot(): string =
-    joinPath(absolutePath(getAppDir()), RESOURCEROOT)
-  proc modRoot(): string =
-    joinPath(resourceRoot(), selectedMod)
+    getAppDir().absolutePath().joinPath(RESOURCEROOT)
+  proc packageRoot(package: string): string =
+    resourceRoot().joinPath(package)
 
-  proc loadResource_intern(path: string): Stream =
-    let realpath = joinPath(modRoot(), path)
+  proc loadResource_intern(path: string, package: string): Stream =
+    let realpath = package.packageRoot().joinPath(path)
     if not realpath.fileExists():
-      raise newException(Exception, &"Resource {path} not found")
+      raise newException(Exception, &"Resource {path} not found (checked {realpath})")
     newFileStream(realpath, fmRead)
 
   proc modList_intern(): seq[string] =
@@ -49,25 +60,29 @@
       if kind == pcDir:
         result.add file
 
-  iterator walkResources_intern(): string =
-    for file in walkDirRec(modRoot(), relative = true):
+  iterator walkResources_intern(dir: string, package = DEFAULT_PACKAGE): string =
+    for file in walkDirRec(package.packageRoot().joinPath(dir), relative = true):
       yield file
 
+  iterator ls_intern(dir: string, package: string): tuple[kind: PathComponent, path: string] =
+    for i in walkDir(package.packageRoot().joinPath(dir), relative = true):
+      yield i
+
 elif thebundletype == Zip:
 
   import zippy/ziparchives
 
   proc resourceRoot(): string =
-    joinPath(absolutePath(getAppDir()), RESOURCEROOT)
-  proc modRoot(): string =
-    joinPath(resourceRoot(), selectedMod)
+    absolutePath(getAppDir()).joinPath(RESOURCEROOT)
+  proc packageRoot(package: string): string =
+    resourceRoot().joinPath(package)
 
-  proc loadResource_intern(path: string): Stream =
-    if not path.fileExists():
+  proc loadResource_intern(path: string, package: string): Stream =
+    let archive = openZipArchive(package.packageRoot() & ".zip")
+    try:
+      result = newStringStream(archive.extractFile(path))
+    except ZippyError:
       raise newException(Exception, &"Resource {path} not found")
-    let archive = openZipArchive(modRoot() & ".zip")
-    # read all here so we can close the stream
-    result = newStringStream(archive.extractFile(path))
     archive.close()
 
   proc modList_intern(): seq[string] =
@@ -75,13 +90,30 @@
       if kind == pcFile and file.endsWith(".zip"):
         result.add file[0 ..< ^4]
 
-  iterator walkResources_intern(): string =
-    let archive = openZipArchive(modRoot() & ".zip")
+  iterator walkResources_intern(dir: string, package = DEFAULT_PACKAGE): string =
+    let archive = openZipArchive(package.packageRoot() & ".zip")
+    let normDir = dir.normalizeDir()
     for i in archive.walkFiles:
-      if i[^1] != '/':
+      if i.startsWith(normDir):
         yield i
     archive.close()
 
+  iterator ls_intern(dir: string, package: string): tuple[kind: PathComponent, path: string] =
+    let archive = openZipArchive(package.packageRoot() & ".zip")
+    let normDir = dir.normalizeDir()
+    var yielded: HashSet[string]
+
+    for i in archive.walkFiles:
+      if i.startsWith(normDir):
+        let components = i[normDir.len .. ^1].split('/', maxsplit = 1)
+        if components.len == 1:
+          if not (components[0] in yielded):
+            yield (kind: pcFile, path: components[0])
+        else:
+          if not (components[0] in yielded):
+            yield (kind: pcDir, path: components[0])
+    archive.close()
+
 elif thebundletype == Exe:
 
   import std/compilesettings
@@ -92,42 +124,56 @@
 
   proc loadResources(): Table[string, Table[string, string]] {.compileTime.} =
     assert BUILD_RESOURCEROOT != "", "define BUILD_RESOURCEROOT to build for bundle type 'exe'"
-    for kind, moddir in walkDir(BUILD_RESOURCEROOT):
+    for kind, packageDir in walkDir(BUILD_RESOURCEROOT):
       if kind == pcDir:
-        let modname = moddir.splitPath.tail
-        result[modname] = Table[string, string]()
-        for resourcefile in walkDirRec(moddir, relative = true):
-          result[modname][resourcefile] = staticRead(joinPath(moddir, resourcefile))
+        let package = packageDir.splitPath.tail
+        result[package] = Table[string, string]()
+        for resourcefile in walkDirRec(packageDir, relative = true):
+          result[package][resourcefile] = staticRead(packageDir.joinPath(resourcefile))
   const bundledResources = loadResources()
 
-  proc loadResource_intern(path: string): Stream =
-    if not (path in bundledResources[selectedMod]):
+  proc loadResource_intern(path: string, package: string): Stream =
+    if not (path in bundledResources[package]):
       raise newException(Exception, &"Resource {path} not found")
-    newStringStream(bundledResources[selectedMod][path])
+    newStringStream(bundledResources[package][path])
 
   proc modList_intern(): seq[string] =
     result = bundledResources.keys().toSeq()
 
-  iterator walkResources_intern(): string =
-    for i in bundledResources[selectedMod].keys:
+  iterator walkResources_intern(dir: string, package = DEFAULT_PACKAGE): string =
+    for i in bundledResources[package].keys:
       yield i
 
-proc loadResource*(path: string): Stream =
-  loadResource_intern(path)
+  iterator ls_intern(dir: string, package: string): tuple[kind: PathComponent, path: string] =
+    let normDir = dir.normalizeDir()
+    var yielded: HashSet[string]
 
-proc loadImage*[T](path: string): Image[RGBAPixel] =
+    for i in bundledResources[package].keys:
+      if i.startsWith(normDir):
+        let components = i[normDir.len .. ^1].split('/', maxsplit = 1)
+        if components.len == 1:
+          if not (components[0] in yielded):
+            yield (kind: pcFile, path: components[0])
+        else:
+          if not (components[0] in yielded):
+            yield (kind: pcDir, path: components[0])
+
+proc loadResource*(path: string, package = DEFAULT_PACKAGE): Stream =
+  loadResource_intern(path, package = package)
+
+proc loadImage*[T](path: string, package = DEFAULT_PACKAGE): Image[RGBAPixel] =
   if path.splitFile().ext.toLowerAscii == ".bmp":
-    loadResource_intern(path).readBMP()
+    loadResource_intern(path, package = package).readBMP()
   elif path.splitFile().ext.toLowerAscii == ".png":
-    loadResource_intern(path).readPNG()
+    loadResource_intern(path, package = package).readPNG()
   else:
     raise newException(Exception, "Unsupported image file type: " & path)
 
-proc loadAudio*(path: string): Sound =
+proc loadAudio*(path: string, package = DEFAULT_PACKAGE): Sound =
   if path.splitFile().ext.toLowerAscii == ".au":
-    loadResource_intern(path).readAU()
+    loadResource_intern(path, package = package).readAU()
   elif path.splitFile().ext.toLowerAscii == ".ogg":
-    loadResource_intern(path).readVorbis()
+    loadResource_intern(path, package = package).readVorbis()
   else:
     raise newException(Exception, "Unsupported audio file type: " & path)
 
@@ -136,23 +182,30 @@
   name = "",
   lineHeightPixels = 80'f32,
   additional_codepoints: openArray[Rune] = [],
-  charset = ASCII_CHARSET
+  charset = ASCII_CHARSET,
+  package = DEFAULT_PACKAGE
 ): Font =
   var thename = name
   if thename == "":
     thename = path.splitFile().name
-  loadResource_intern(path).readTrueType(name, charset & additional_codepoints.toSeq, lineHeightPixels)
+  loadResource_intern(path, package = package).readTrueType(name, charset & additional_codepoints.toSeq, lineHeightPixels)
 
-proc loadMeshes*(path: string, defaultMaterial: MaterialType): seq[MeshTree] =
-  loadResource_intern(path).readglTF(defaultMaterial)
+proc loadMeshes*(path: string, defaultMaterial: MaterialType, package = DEFAULT_PACKAGE): seq[MeshTree] =
+  loadResource_intern(path, package = package).readglTF(defaultMaterial)
 
-proc loadFirstMesh*(path: string, defaultMaterial: MaterialType): Mesh =
-  loadResource_intern(path).readglTF(defaultMaterial)[0].toSeq[0]
+proc loadFirstMesh*(path: string, defaultMaterial: MaterialType, package = DEFAULT_PACKAGE): Mesh =
+  loadResource_intern(path, package = package).readglTF(defaultMaterial)[0].toSeq[0]
 
-proc modList*(): seq[string] =
+proc packages*(): seq[string] =
   modList_intern()
 
-iterator walkResources*(dir = ""): string =
-  for i in walkResources_intern():
+proc walkResources*(dir = "", package = DEFAULT_PACKAGE): seq[string] =
+  for i in walkResources_intern(dir, package = package):
     if i.startsWith(dir):
-      yield i
+      result.add i
+  result.sort()
+
+proc ls*(dir: string, package = DEFAULT_PACKAGE): seq[tuple[kind: PathComponent, path: string]] =
+  for i in ls_intern(dir = dir, package = package):
+    result.add i
+  result.sort()
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/tests/resources/mod1/aSubdir/moreSubdir/superSubdir/secret.txt	Wed Feb 14 21:58:43 2024 +0700
@@ -0,0 +1,1 @@
+hello world!
--- a/tests/test_resources.nim	Sun Feb 11 19:26:07 2024 +0700
+++ b/tests/test_resources.nim	Wed Feb 14 21:58:43 2024 +0700
@@ -1,14 +1,34 @@
+import std/os
 import std/streams
+import std/strformat
+import std/strutils
 
 import semicongine
 
+proc list_all_mods_all_files() =
+  for package in packages():
+    echo &"Files in package {package}:"
+    for i in walkResources(package = package):
+      echo "  ", i, ": ", i.loadResource(package = package).readAll().len
+
+proc print_ls(dir, package: string, indent = 2) =
+  for i in dir.ls(package = package):
+    if i.kind == pcDir:
+      echo "".align(indent), i.path, "/"
+      print_ls(dir.joinPath(i.path), package = package, indent = indent + 2)
+    else:
+      echo "".align(indent), i.path, ": ", dir.joinPath(i.path).loadResource(package = package).readAll().len
+
+proc list_files() =
+  for package in packages():
+    echo &"Recursive walk of package {package}: "
+    print_ls("", package = package)
+
+
 proc main() =
-  echo "Mods available: ", modList()
-  for modName in modList():
-    echo modName, ":"
-    selectedMod = modName
-    for i in walkResources():
-      echo "  ", i, ": ", loadResource(i).readAll().len
+  echo "Packages available: ", packages()
+  list_all_mods_all_files()
+  list_files()
 
 when isMainModule:
   main()