# HG changeset patch # User Sam # Date 1707922723 -25200 # Node ID 1ab09f8cc68d1ece524e80830bba07974c4d4940 # Parent e75c6da1d2614058efd88cc2098a4369ae59bdd7 did: refactor and improve packaging API diff -r e75c6da1d261 -r 1ab09f8cc68d config.nims --- a/config.nims Sun Feb 11 19:26:07 2024 +0700 +++ b/config.nims Wed Feb 14 21:58:43 2024 +0700 @@ -8,7 +8,7 @@ const LINUX = "linux" const WINDOWS = "windows" -const BUNDLETYPE* {.strdefine.}: string = "dir" # dir, zip, exe +const PACKAGETYPE* {.strdefine.}: string = "dir" # dir, zip, exe const RESOURCEROOT* {.strdefine.}: string = "resources" switch("d", "nimPreviewHashRef") @@ -17,7 +17,7 @@ switch("nimblePath", "nimbledeps/pkgs2") task build, "build": - switch("d", "BUNDLETYPE=" & BUNDLETYPE) + switch("d", "PACKAGETYPE=" & PACKAGETYPE) switch("d", "RESOURCEROOT=" & RESOURCEROOT) var buildType = DEBUG var platformDir = "" @@ -41,9 +41,9 @@ let resourcedir = joinPath(projectDir(), RESOURCEROOT) if dirExists(resourcedir): let outdir_resources = joinPath(outdir, RESOURCEROOT) - if BUNDLETYPE == "dir": + if PACKAGETYPE == "dir": cpDir(resourcedir, outdir_resources) - elif BUNDLETYPE == "zip": + elif PACKAGETYPE == "zip": mkDir(outdir_resources) for resource in listDirs(resourcedir): let @@ -70,9 +70,13 @@ task test_all, "Run all test programs": for file in listFiles("tests"): - if file.endsWith(".nim"): + if file.endsWith(".nim") and not file.endsWith("test_resources.nim"): exec(&"nim build --run {file}") + exec("nim build -d:BUILD_RESOURCEROOT=tests/resources -d:PACKAGETYPE=dir --run tests/test_resources.nim") + exec("nim build -d:BUILD_RESOURCEROOT=tests/resources -d:PACKAGETYPE=zip --run tests/test_resources.nim") + exec("nim build -d:BUILD_RESOURCEROOT=tests/resources -d:PACKAGETYPE=exe --run tests/test_resources.nim") + task clean, "remove all build files": exec(&"rm -rf {BUILDBASE}") @@ -125,6 +129,6 @@ if getCommand() in ["c", "compile", "r", "dump", "check", "idetools"]: if defined(linux): - --d:VK_USE_PLATFORM_XLIB_KHR + --d: VK_USE_PLATFORM_XLIB_KHR if defined(windows): - --d:VK_USE_PLATFORM_WIN32_KHR + --d: VK_USE_PLATFORM_WIN32_KHR diff -r e75c6da1d261 -r 1ab09f8cc68d semicongine/build.nim --- a/semicongine/build.nim Sun Feb 11 19:26:07 2024 +0700 +++ b/semicongine/build.nim Wed Feb 14 21:58:43 2024 +0700 @@ -30,7 +30,7 @@ switch("outdir", semicongine_builddir(buildname, builddir = builddir)) proc semicongine_pack*(outdir: string, bundleType: string, resourceRoot: string) = - switch("define", "BUNDLETYPE=" & bundleType) + switch("define", "PACKAGETYPE=" & bundleType) rmDir(outdir) mkDir(outdir) diff -r e75c6da1d261 -r 1ab09f8cc68d semicongine/core/buildconfig.nim --- a/semicongine/core/buildconfig.nim Sun Feb 11 19:26:07 2024 +0700 +++ b/semicongine/core/buildconfig.nim Wed Feb 14 21:58:43 2024 +0700 @@ -46,6 +46,6 @@ const ENGINE_LOGLEVEL* = parseEnum[Level](LOGLEVEL) # resource bundleing settings, need to be configured per project -const BUNDLETYPE* {.strdefine.}: string = "" # dir, zip, exe +const PACKAGETYPE* {.strdefine.}: string = "" # dir, zip, exe static: - assert BUNDLETYPE in ["dir", "zip", "exe"], ENGINENAME & " requires one of -d:BUNDLETYPE=dir -d:BUNDLETYPE=zip -d:BUNDLETYPE=exe" + assert PACKAGETYPE in ["dir", "zip", "exe"], ENGINENAME & " requires one of -d:PACKAGETYPE=dir -d:PACKAGETYPE=zip -d:PACKAGETYPE=exe" diff -r e75c6da1d261 -r 1ab09f8cc68d semicongine/material.nim --- a/semicongine/material.nim Sun Feb 11 19:26:07 2024 +0700 +++ b/semicongine/material.nim Wed Feb 14 21:58:43 2024 +0700 @@ -131,7 +131,7 @@ EMPTY_SHADER* = createShaderConfiguration( inputs = [ attr[Mat4](TRANSFORM_ATTRIB, memoryPerformanceHint = PreferFastWrite, perInstance = true), - attr[Vec3f]("position", memoryPerformanceHint = PreferFastWrite), + attr[Vec3f]("position", memoryPerformanceHint = PreferFastRead), ], outputs = [attr[Vec4f]("color")], vertexCode = &"gl_Position = vec4(position, 1.0) * {TRANSFORM_ATTRIB};", diff -r e75c6da1d261 -r 1ab09f8cc68d semicongine/renderer.nim --- a/semicongine/renderer.nim Sun Feb 11 19:26:07 2024 +0700 +++ b/semicongine/renderer.nim Wed Feb 14 21:58:43 2024 +0700 @@ -153,8 +153,6 @@ renderer.checkSceneIntegrity(scene) - let inputs = renderer.inputs(scene) - # create index buffer if necessary var indicesBufferSize = 0 for mesh in scene.meshes: @@ -181,6 +179,9 @@ var perLocationSizes: Table[MemoryPerformanceHint, int] for hint in MemoryPerformanceHint: perLocationSizes[hint] = 0 + + let inputs = renderer.inputs(scene) + for attribute in inputs: scenedata.attributeLocation[attribute.name] = attribute.memoryPerformanceHint # setup one buffer per attribute-location-type diff -r e75c6da1d261 -r 1ab09f8cc68d semicongine/resources.nim --- a/semicongine/resources.nim Sun Feb 11 19:26:07 2024 +0700 +++ b/semicongine/resources.nim Wed Feb 14 21:58:43 2024 +0700 @@ -1,8 +1,10 @@ import std/streams +import std/algorithm import std/strutils import std/sequtils import std/strformat import std/os +import std/sets import std/unicode import ./core @@ -24,24 +26,33 @@ Exe # Embeded in executable const - thebundletype = parseEnum[ResourceBundlingType](BUNDLETYPE.toLowerAscii().capitalizeAscii()) + thebundletype = parseEnum[ResourceBundlingType](PACKAGETYPE.toLowerAscii().capitalizeAscii()) ASCII_CHARSET = PrintableChars.toSeq.toRunes - -var selectedMod* = "default" + DEFAULT_PACKAGE = "default" # resource loading +func normalizeDir(dir: string): string = + result = dir + if result.startsWith("./"): + result = result[2 .. ^1] + if result.startsWith("/"): + result = result[1 .. ^1] + result = dir.replace('\\', '/') + if not result.endsWith("/") and result != "": + result = result & "/" + when thebundletype == Dir: proc resourceRoot(): string = - joinPath(absolutePath(getAppDir()), RESOURCEROOT) - proc modRoot(): string = - joinPath(resourceRoot(), selectedMod) + getAppDir().absolutePath().joinPath(RESOURCEROOT) + proc packageRoot(package: string): string = + resourceRoot().joinPath(package) - proc loadResource_intern(path: string): Stream = - let realpath = joinPath(modRoot(), path) + proc loadResource_intern(path: string, package: string): Stream = + let realpath = package.packageRoot().joinPath(path) if not realpath.fileExists(): - raise newException(Exception, &"Resource {path} not found") + raise newException(Exception, &"Resource {path} not found (checked {realpath})") newFileStream(realpath, fmRead) proc modList_intern(): seq[string] = @@ -49,25 +60,29 @@ if kind == pcDir: result.add file - iterator walkResources_intern(): string = - for file in walkDirRec(modRoot(), relative = true): + iterator walkResources_intern(dir: string, package = DEFAULT_PACKAGE): string = + for file in walkDirRec(package.packageRoot().joinPath(dir), relative = true): yield file + iterator ls_intern(dir: string, package: string): tuple[kind: PathComponent, path: string] = + for i in walkDir(package.packageRoot().joinPath(dir), relative = true): + yield i + elif thebundletype == Zip: import zippy/ziparchives proc resourceRoot(): string = - joinPath(absolutePath(getAppDir()), RESOURCEROOT) - proc modRoot(): string = - joinPath(resourceRoot(), selectedMod) + absolutePath(getAppDir()).joinPath(RESOURCEROOT) + proc packageRoot(package: string): string = + resourceRoot().joinPath(package) - proc loadResource_intern(path: string): Stream = - if not path.fileExists(): + proc loadResource_intern(path: string, package: string): Stream = + let archive = openZipArchive(package.packageRoot() & ".zip") + try: + result = newStringStream(archive.extractFile(path)) + except ZippyError: raise newException(Exception, &"Resource {path} not found") - let archive = openZipArchive(modRoot() & ".zip") - # read all here so we can close the stream - result = newStringStream(archive.extractFile(path)) archive.close() proc modList_intern(): seq[string] = @@ -75,13 +90,30 @@ if kind == pcFile and file.endsWith(".zip"): result.add file[0 ..< ^4] - iterator walkResources_intern(): string = - let archive = openZipArchive(modRoot() & ".zip") + iterator walkResources_intern(dir: string, package = DEFAULT_PACKAGE): string = + let archive = openZipArchive(package.packageRoot() & ".zip") + let normDir = dir.normalizeDir() for i in archive.walkFiles: - if i[^1] != '/': + if i.startsWith(normDir): yield i archive.close() + iterator ls_intern(dir: string, package: string): tuple[kind: PathComponent, path: string] = + let archive = openZipArchive(package.packageRoot() & ".zip") + let normDir = dir.normalizeDir() + var yielded: HashSet[string] + + for i in archive.walkFiles: + if i.startsWith(normDir): + let components = i[normDir.len .. ^1].split('/', maxsplit = 1) + if components.len == 1: + if not (components[0] in yielded): + yield (kind: pcFile, path: components[0]) + else: + if not (components[0] in yielded): + yield (kind: pcDir, path: components[0]) + archive.close() + elif thebundletype == Exe: import std/compilesettings @@ -92,42 +124,56 @@ proc loadResources(): Table[string, Table[string, string]] {.compileTime.} = assert BUILD_RESOURCEROOT != "", "define BUILD_RESOURCEROOT to build for bundle type 'exe'" - for kind, moddir in walkDir(BUILD_RESOURCEROOT): + for kind, packageDir in walkDir(BUILD_RESOURCEROOT): if kind == pcDir: - let modname = moddir.splitPath.tail - result[modname] = Table[string, string]() - for resourcefile in walkDirRec(moddir, relative = true): - result[modname][resourcefile] = staticRead(joinPath(moddir, resourcefile)) + let package = packageDir.splitPath.tail + result[package] = Table[string, string]() + for resourcefile in walkDirRec(packageDir, relative = true): + result[package][resourcefile] = staticRead(packageDir.joinPath(resourcefile)) const bundledResources = loadResources() - proc loadResource_intern(path: string): Stream = - if not (path in bundledResources[selectedMod]): + proc loadResource_intern(path: string, package: string): Stream = + if not (path in bundledResources[package]): raise newException(Exception, &"Resource {path} not found") - newStringStream(bundledResources[selectedMod][path]) + newStringStream(bundledResources[package][path]) proc modList_intern(): seq[string] = result = bundledResources.keys().toSeq() - iterator walkResources_intern(): string = - for i in bundledResources[selectedMod].keys: + iterator walkResources_intern(dir: string, package = DEFAULT_PACKAGE): string = + for i in bundledResources[package].keys: yield i -proc loadResource*(path: string): Stream = - loadResource_intern(path) + iterator ls_intern(dir: string, package: string): tuple[kind: PathComponent, path: string] = + let normDir = dir.normalizeDir() + var yielded: HashSet[string] -proc loadImage*[T](path: string): Image[RGBAPixel] = + for i in bundledResources[package].keys: + if i.startsWith(normDir): + let components = i[normDir.len .. ^1].split('/', maxsplit = 1) + if components.len == 1: + if not (components[0] in yielded): + yield (kind: pcFile, path: components[0]) + else: + if not (components[0] in yielded): + yield (kind: pcDir, path: components[0]) + +proc loadResource*(path: string, package = DEFAULT_PACKAGE): Stream = + loadResource_intern(path, package = package) + +proc loadImage*[T](path: string, package = DEFAULT_PACKAGE): Image[RGBAPixel] = if path.splitFile().ext.toLowerAscii == ".bmp": - loadResource_intern(path).readBMP() + loadResource_intern(path, package = package).readBMP() elif path.splitFile().ext.toLowerAscii == ".png": - loadResource_intern(path).readPNG() + loadResource_intern(path, package = package).readPNG() else: raise newException(Exception, "Unsupported image file type: " & path) -proc loadAudio*(path: string): Sound = +proc loadAudio*(path: string, package = DEFAULT_PACKAGE): Sound = if path.splitFile().ext.toLowerAscii == ".au": - loadResource_intern(path).readAU() + loadResource_intern(path, package = package).readAU() elif path.splitFile().ext.toLowerAscii == ".ogg": - loadResource_intern(path).readVorbis() + loadResource_intern(path, package = package).readVorbis() else: raise newException(Exception, "Unsupported audio file type: " & path) @@ -136,23 +182,30 @@ name = "", lineHeightPixels = 80'f32, additional_codepoints: openArray[Rune] = [], - charset = ASCII_CHARSET + charset = ASCII_CHARSET, + package = DEFAULT_PACKAGE ): Font = var thename = name if thename == "": thename = path.splitFile().name - loadResource_intern(path).readTrueType(name, charset & additional_codepoints.toSeq, lineHeightPixels) + loadResource_intern(path, package = package).readTrueType(name, charset & additional_codepoints.toSeq, lineHeightPixels) -proc loadMeshes*(path: string, defaultMaterial: MaterialType): seq[MeshTree] = - loadResource_intern(path).readglTF(defaultMaterial) +proc loadMeshes*(path: string, defaultMaterial: MaterialType, package = DEFAULT_PACKAGE): seq[MeshTree] = + loadResource_intern(path, package = package).readglTF(defaultMaterial) -proc loadFirstMesh*(path: string, defaultMaterial: MaterialType): Mesh = - loadResource_intern(path).readglTF(defaultMaterial)[0].toSeq[0] +proc loadFirstMesh*(path: string, defaultMaterial: MaterialType, package = DEFAULT_PACKAGE): Mesh = + loadResource_intern(path, package = package).readglTF(defaultMaterial)[0].toSeq[0] -proc modList*(): seq[string] = +proc packages*(): seq[string] = modList_intern() -iterator walkResources*(dir = ""): string = - for i in walkResources_intern(): +proc walkResources*(dir = "", package = DEFAULT_PACKAGE): seq[string] = + for i in walkResources_intern(dir, package = package): if i.startsWith(dir): - yield i + result.add i + result.sort() + +proc ls*(dir: string, package = DEFAULT_PACKAGE): seq[tuple[kind: PathComponent, path: string]] = + for i in ls_intern(dir = dir, package = package): + result.add i + result.sort() diff -r e75c6da1d261 -r 1ab09f8cc68d tests/resources/mod1/aSubdir/moreSubdir/superSubdir/secret.txt --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tests/resources/mod1/aSubdir/moreSubdir/superSubdir/secret.txt Wed Feb 14 21:58:43 2024 +0700 @@ -0,0 +1,1 @@ +hello world! diff -r e75c6da1d261 -r 1ab09f8cc68d tests/test_resources.nim --- a/tests/test_resources.nim Sun Feb 11 19:26:07 2024 +0700 +++ b/tests/test_resources.nim Wed Feb 14 21:58:43 2024 +0700 @@ -1,14 +1,34 @@ +import std/os import std/streams +import std/strformat +import std/strutils import semicongine +proc list_all_mods_all_files() = + for package in packages(): + echo &"Files in package {package}:" + for i in walkResources(package = package): + echo " ", i, ": ", i.loadResource(package = package).readAll().len + +proc print_ls(dir, package: string, indent = 2) = + for i in dir.ls(package = package): + if i.kind == pcDir: + echo "".align(indent), i.path, "/" + print_ls(dir.joinPath(i.path), package = package, indent = indent + 2) + else: + echo "".align(indent), i.path, ": ", dir.joinPath(i.path).loadResource(package = package).readAll().len + +proc list_files() = + for package in packages(): + echo &"Recursive walk of package {package}: " + print_ls("", package = package) + + proc main() = - echo "Mods available: ", modList() - for modName in modList(): - echo modName, ":" - selectedMod = modName - for i in walkResources(): - echo " ", i, ": ", loadResource(i).readAll().len + echo "Packages available: ", packages() + list_all_mods_all_files() + list_files() when isMainModule: main()