osb/source/base/StarAssets.cpp

1291 lines
43 KiB
C++
Raw Normal View History

2023-06-20 04:33:09 +00:00
#include "StarAssets.hpp"
2023-06-23 15:30:55 +00:00
#include "StarAssetPath.hpp"
2023-06-20 04:33:09 +00:00
#include "StarFile.hpp"
#include "StarTime.hpp"
#include "StarDirectoryAssetSource.hpp"
#include "StarPackedAssetSource.hpp"
#include "StarMemoryAssetSource.hpp"
2023-06-20 04:33:09 +00:00
#include "StarJsonBuilder.hpp"
#include "StarJsonExtra.hpp"
#include "StarJsonPatch.hpp"
#include "StarIterator.hpp"
#include "StarImageProcessing.hpp"
#include "StarLogging.hpp"
#include "StarRandom.hpp"
#include "StarFont.hpp"
#include "StarAudio.hpp"
#include "StarCasting.hpp"
#include "StarLexicalCast.hpp"
#include "StarSha256.hpp"
#include "StarDataStreamDevices.hpp"
2024-03-15 10:28:11 +00:00
#include "StarLua.hpp"
#include "StarImageLuaBindings.hpp"
2024-03-15 10:28:11 +00:00
#include "StarUtilityLuaBindings.hpp"
2023-06-20 04:33:09 +00:00
namespace Star {
2024-03-15 10:28:11 +00:00
static void validateBasePath(std::string_view const& basePath) {
if (basePath.empty() || basePath[0] != '/')
throw AssetException(strf("Path '{}' must be absolute", basePath));
2023-06-20 04:33:09 +00:00
bool first = true;
bool slashed = true;
bool dotted = false;
2024-03-15 10:28:11 +00:00
for (auto c : basePath) {
2023-06-20 04:33:09 +00:00
if (c == '/') {
if (!first) {
if (slashed)
2024-03-15 10:28:11 +00:00
throw AssetException(strf("Path '{}' contains consecutive //, not allowed", basePath));
2023-06-20 04:33:09 +00:00
else if (dotted)
2024-03-15 10:28:11 +00:00
throw AssetException(strf("Path '{}' '.' and '..' not allowed", basePath));
2023-06-20 04:33:09 +00:00
}
slashed = true;
dotted = false;
} else if (c == ':') {
if (slashed)
2024-03-15 10:28:11 +00:00
throw AssetException(strf("Path '{}' has ':' after directory", basePath));
2023-06-20 04:33:09 +00:00
break;
} else if (c == '?') {
if (slashed)
2024-03-15 10:28:11 +00:00
throw AssetException(strf("Path '{}' has '?' after directory", basePath));
2023-06-20 04:33:09 +00:00
break;
} else {
slashed = false;
dotted = c == '.';
}
first = false;
}
if (slashed)
2024-03-15 10:28:11 +00:00
throw AssetException(strf("Path '{}' cannot be a file", basePath));
}
static void validatePath(AssetPath const& components, bool canContainSubPath, bool canContainDirectives) {
validateBasePath(components.basePath.utf8());
2023-06-20 04:33:09 +00:00
if (!canContainSubPath && components.subPath)
2023-06-27 10:23:44 +00:00
throw AssetException::format("Path '{}' cannot contain sub-path", components);
2023-06-20 04:33:09 +00:00
if (!canContainDirectives && !components.directives.empty())
2023-06-27 10:23:44 +00:00
throw AssetException::format("Path '{}' cannot contain directives", components);
2023-06-20 04:33:09 +00:00
}
2024-03-15 10:28:11 +00:00
static void validatePath(StringView const& path, bool canContainSubPath, bool canContainDirectives) {
std::string_view const& str = path.utf8();
size_t end = str.find_first_of(":?");
auto basePath = str.substr(0, end);
validateBasePath(basePath);
bool subPath = false;
if (str[end] == ':') {
size_t beg = end + 1;
if (beg != str.size()) {
end = str.find_first_of('?', beg);
if (end == NPos && beg + 1 != str.size())
subPath = true;
else if (size_t len = end - beg)
subPath = true;
}
}
if (subPath)
throw AssetException::format("Path '{}' cannot contain sub-path", path);
if (end != NPos && str[end] == '?' && !canContainDirectives)
throw AssetException::format("Path '{}' cannot contain directives", path);
}
2023-06-20 04:33:09 +00:00
Maybe<RectU> FramesSpecification::getRect(String const& frame) const {
if (auto alias = aliases.ptr(frame)) {
return frames.get(*alias);
} else {
return frames.maybe(frame);
}
}
Assets::Assets(Settings settings, StringList assetSources) {
const char* AssetsPatchSuffix = ".patch";
2024-06-25 09:56:44 +00:00
const char* AssetsPatchListSuffix = ".patchlist";
const char* AssetsLuaPatchSuffix = ".patch.lua";
2023-06-20 04:33:09 +00:00
m_settings = std::move(settings);
2023-06-20 04:33:09 +00:00
m_stopThreads = false;
m_assetSources = std::move(assetSources);
2023-06-20 04:33:09 +00:00
2024-03-15 10:28:11 +00:00
auto luaEngine = LuaEngine::create();
2024-03-20 13:57:49 +00:00
m_luaEngine = luaEngine;
2024-03-22 07:56:20 +00:00
auto pushGlobalContext = [&luaEngine](String const& name, LuaCallbacks && callbacks) {
2024-03-20 13:57:49 +00:00
auto table = luaEngine->createTable();
for (auto const& p : callbacks.callbacks())
table.set(p.first, luaEngine->createWrappedFunction(p.second));
luaEngine->setGlobal(name, table);
};
auto makeBaseAssetCallbacks = [this]() {
2024-03-15 10:28:11 +00:00
LuaCallbacks callbacks;
callbacks.registerCallbackWithSignature<StringSet, String>("byExtension", bind(&Assets::scanExtension, this, _1));
callbacks.registerCallbackWithSignature<Json, String>("json", bind(&Assets::json, this, _1));
2024-09-07 04:03:07 +00:00
callbacks.registerCallbackWithSignature<bool, String>("exists", bind(&Assets::assetExists, this, _1));
2024-03-15 10:28:11 +00:00
callbacks.registerCallback("bytes", [this](String const& path) -> String {
auto assetBytes = bytes(path);
return String(assetBytes->ptr(), assetBytes->size());
});
callbacks.registerCallback("image", [this](String const& path) -> Image {
auto assetImage = image(path);
if (assetImage->bytesPerPixel() == 3)
return assetImage->convert(PixelFormat::RGBA32);
else
return *assetImage;
});
2024-03-15 10:28:11 +00:00
callbacks.registerCallback("scan", [this](Maybe<String> const& a, Maybe<String> const& b) -> StringList {
return b ? scan(a.value(), *b) : scan(a.value());
});
return callbacks;
};
pushGlobalContext("sb", LuaBindings::makeUtilityCallbacks());
pushGlobalContext("assets", makeBaseAssetCallbacks());
auto decorateLuaContext = [&](LuaContext& context, MemoryAssetSourcePtr newFiles) {
2024-03-20 13:57:49 +00:00
if (newFiles) {
// re-add the assets callbacks with more functions
context.remove("assets");
auto callbacks = makeBaseAssetCallbacks();
callbacks.registerCallback("add", [newFiles](LuaEngine& engine, String const& path, LuaValue const& data) {
2024-03-20 13:57:49 +00:00
ByteArray bytes;
if (auto str = engine.luaMaybeTo<String>(data))
bytes = ByteArray(str->utf8Ptr(), str->utf8Size());
else if (auto image = engine.luaMaybeTo<Image>(data)) {
newFiles->set(path, std::move(*image));
return;
} else {
2024-03-20 13:57:49 +00:00
auto json = engine.luaTo<Json>(data).repr();
bytes = ByteArray(json.utf8Ptr(), json.utf8Size());
}
newFiles->set(path, bytes);
});
callbacks.registerCallback("patch", [this, newFiles](String const& path, String const& patchPath) -> bool {
2024-03-20 13:57:49 +00:00
if (auto file = m_files.ptr(path)) {
if (newFiles->contains(patchPath)) {
file->patchSources.append(make_pair(patchPath, newFiles));
return true;
2024-03-20 13:57:49 +00:00
} else {
if (auto asset = m_files.ptr(patchPath)) {
file->patchSources.append(make_pair(patchPath, asset->source));
return true;
}
}
}
2024-03-20 13:57:49 +00:00
return false;
});
callbacks.registerCallback("erase", [this](String const& path) -> bool {
bool erased = m_files.erase(path);
if (erased)
m_filesByExtension[AssetPath::extension(path).toLower()].erase(path);
return erased;
});
context.setCallbacks("assets", callbacks);
}
2024-03-15 10:28:11 +00:00
};
2024-03-15 13:02:51 +00:00
auto addSource = [&](String const& sourcePath, AssetSourcePtr source) {
m_assetSourcePaths.add(sourcePath, source);
2023-06-20 04:33:09 +00:00
2024-03-15 13:02:51 +00:00
for (auto const& filename : source->assetPaths()) {
if (filename.contains(AssetsPatchSuffix, String::CaseInsensitive)) {
if (filename.endsWith(AssetsPatchSuffix, String::CaseInsensitive)) {
auto targetPatchFile = filename.substr(0, filename.size() - strlen(AssetsPatchSuffix));
if (auto p = m_files.ptr(targetPatchFile))
p->patchSources.append({filename, source});
} else if (filename.endsWith(AssetsLuaPatchSuffix, String::CaseInsensitive)) {
auto targetPatchFile = filename.substr(0, filename.size() - strlen(AssetsLuaPatchSuffix));
if (auto p = m_files.ptr(targetPatchFile))
p->patchSources.append({filename, source});
2024-06-25 09:56:44 +00:00
} else if (filename.endsWith(AssetsPatchListSuffix, String::CaseInsensitive)) {
auto stream = source->read(filename);
size_t patchIndex = 0;
for (auto const& patchPair : inputUtf8Json(stream.begin(), stream.end(), JsonParseType::Top).iterateArray()) {
2024-06-25 10:03:35 +00:00
auto patches = patchPair.getArray("patches");
2024-06-25 09:56:44 +00:00
for (auto& path : patchPair.getArray("paths")) {
if (auto p = m_files.ptr(path.toString())) {
for (size_t i = 0; i != patches.size(); ++i) {
auto& patch = patches[i];
if (patch.isType(Json::Type::String))
p->patchSources.append({patch.toString(), source});
else
p->patchSources.append({strf("{}:[{}].patches[{}]", filename, patchIndex, i), source});
}
}
}
patchIndex++;
}
2024-03-15 13:02:51 +00:00
} else {
for (int i = 0; i < 10; i++) {
if (filename.endsWith(AssetsPatchSuffix + toString(i), String::CaseInsensitive)) {
auto targetPatchFile = filename.substr(0, filename.size() - strlen(AssetsPatchSuffix) + 1);
if (auto p = m_files.ptr(targetPatchFile))
p->patchSources.append({filename, source});
break;
2024-03-06 17:40:38 +00:00
}
}
}
2023-06-20 04:33:09 +00:00
}
2024-03-15 13:02:51 +00:00
auto& descriptor = m_files[filename];
descriptor.sourceName = filename;
descriptor.source = source;
m_filesByExtension[AssetPath::extension(filename).toLower()].insert(filename);
}
};
auto runLoadScripts = [&](String const& groupName, String const& sourcePath, AssetSourcePtr source) {
2024-03-15 10:28:11 +00:00
auto metadata = source->metadata();
if (auto scripts = metadata.ptr("scripts")) {
2024-03-15 13:02:51 +00:00
if (auto scriptGroup = scripts->optArray(groupName)) {
auto memoryName = strf("{}::{}", metadata.value("name", File::baseName(sourcePath)), groupName);
JsonObject memoryMetadata{ {"name", memoryName} };
auto memoryAssets = make_shared<MemoryAssetSource>(memoryName, memoryMetadata);
Logger::info("Running {} scripts {}", groupName, *scriptGroup);
2024-03-15 10:28:11 +00:00
try {
auto context = luaEngine->createContext();
decorateLuaContext(context, memoryAssets);
2024-03-15 13:02:51 +00:00
for (auto& jPath : *scriptGroup) {
2024-03-15 10:28:11 +00:00
auto path = jPath.toString();
auto script = source->read(path);
context.load(script, path);
}
2024-03-15 13:02:51 +00:00
} catch (LuaException const& e) {
Logger::error("Exception while running {} scripts from asset source '{}': {}", groupName, sourcePath, e.what());
2024-03-15 10:28:11 +00:00
}
if (!memoryAssets->empty())
2024-03-15 13:02:51 +00:00
addSource(strf("{}::{}", sourcePath, groupName), memoryAssets);
2024-03-15 10:28:11 +00:00
}
2023-06-20 04:33:09 +00:00
}
// clear any caching that may have been trigered by load scripts as they may no longer be valid
m_framesSpecifications.clear();
m_assetsCache.clear();
2024-03-15 13:02:51 +00:00
};
List<pair<String, AssetSourcePtr>> sources;
for (auto& sourcePath : m_assetSources) {
Logger::info("Loading assets from: '{}'", sourcePath);
AssetSourcePtr source;
if (File::isDirectory(sourcePath))
source = std::make_shared<DirectoryAssetSource>(sourcePath, m_settings.pathIgnore);
else
source = std::make_shared<PackedAssetSource>(sourcePath);
addSource(sourcePath, source);
sources.append(make_pair(sourcePath, source));
runLoadScripts("onLoad", sourcePath, source);
2023-06-20 04:33:09 +00:00
}
2024-03-15 13:02:51 +00:00
for (auto& pair : sources)
runLoadScripts("postLoad", pair.first, pair.second);
2023-06-20 04:33:09 +00:00
Sha256Hasher digest;
for (auto const& assetPath : m_files.keys().transformed([](String const& s) {
return s.toLower();
}).sorted()) {
bool digestFile = true;
for (auto const& pattern : m_settings.digestIgnore) {
if (assetPath.regexMatch(pattern, false, false)) {
digestFile = false;
break;
}
}
auto const& descriptor = m_files.get(assetPath);
if (digestFile) {
digest.push(assetPath);
digest.push(DataStreamBuffer::serialize(descriptor.source->open(descriptor.sourceName)->size()));
for (auto const& pair : descriptor.patchSources)
2024-06-25 09:56:44 +00:00
digest.push(DataStreamBuffer::serialize(pair.second->open(AssetPath::removeSubPath(pair.first))->size()));
2023-06-20 04:33:09 +00:00
}
}
m_digest = digest.compute();
int workerPoolSize = m_settings.workerPoolSize;
for (int i = 0; i < workerPoolSize; i++)
m_workerThreads.append(Thread::invoke("Assets::workerMain", mem_fn(&Assets::workerMain), this));
}
Assets::~Assets() {
m_stopThreads = true;
{
// Should lock associated mutex to prevent loss of wakeups,
MutexLocker locker(m_assetsMutex);
// Notify all worker threads to allow them to stop
m_assetsQueued.broadcast();
}
// Join them all
m_workerThreads.clear();
}
StringList Assets::assetSources() const {
MutexLocker assetsLocker(m_assetsMutex);
return m_assetSources;
}
JsonObject Assets::assetSourceMetadata(String const& sourceName) const {
MutexLocker assetsLocker(m_assetsMutex);
return m_assetSourcePaths.getRight(sourceName)->metadata();
}
ByteArray Assets::digest() const {
MutexLocker assetsLocker(m_assetsMutex);
return m_digest;
}
bool Assets::assetExists(String const& path) const {
MutexLocker assetsLocker(m_assetsMutex);
return m_files.contains(path);
}
2023-11-02 19:51:17 +00:00
Maybe<Assets::AssetFileDescriptor> Assets::assetDescriptor(String const& path) const {
MutexLocker assetsLocker(m_assetsMutex);
return m_files.maybe(path);
}
2023-06-20 04:33:09 +00:00
String Assets::assetSource(String const& path) const {
MutexLocker assetsLocker(m_assetsMutex);
if (auto p = m_files.ptr(path))
return m_assetSourcePaths.getLeft(p->source);
2023-06-27 10:23:44 +00:00
throw AssetException(strf("No such asset '{}'", path));
2023-06-20 04:33:09 +00:00
}
2023-11-02 19:51:17 +00:00
Maybe<String> Assets::assetSourcePath(AssetSourcePtr const& source) const {
MutexLocker assetsLocker(m_assetsMutex);
return m_assetSourcePaths.maybeLeft(source);
}
2023-06-20 04:33:09 +00:00
StringList Assets::scan(String const& suffix) const {
if (suffix.beginsWith(".") && !suffix.substr(1).hasChar('.')) {
2024-03-15 10:28:11 +00:00
return scanExtension(suffix).values();
} else if (suffix.empty()) {
return m_files.keys();
2023-06-20 04:33:09 +00:00
} else {
StringList result;
for (auto const& fileEntry : m_files) {
String const& file = fileEntry.first;
if (file.endsWith(suffix, String::CaseInsensitive))
result.append(file);
}
return result;
}
}
StringList Assets::scan(String const& prefix, String const& suffix) const {
StringList result;
if (suffix.beginsWith(".") && !suffix.substr(1).hasChar('.')) {
auto& filesWithExtension = scanExtension(suffix);
2023-06-20 04:33:09 +00:00
for (auto const& file : filesWithExtension) {
if (file.beginsWith(prefix, String::CaseInsensitive))
result.append(file);
}
} else {
for (auto const& fileEntry : m_files) {
String const& file = fileEntry.first;
if (file.beginsWith(prefix, String::CaseInsensitive) && file.endsWith(suffix, String::CaseInsensitive))
result.append(file);
}
}
return result;
}
const CaseInsensitiveStringSet NullExtensionScan;
2024-03-15 10:28:11 +00:00
CaseInsensitiveStringSet const& Assets::scanExtension(String const& extension) const {
2024-03-15 10:28:11 +00:00
auto find = m_filesByExtension.find(extension.beginsWith(".") ? extension.substr(1) : extension);
return find != m_filesByExtension.end() ? find->second : NullExtensionScan;
2023-06-20 04:33:09 +00:00
}
Json Assets::json(String const& path) const {
auto components = AssetPath::split(path);
validatePath(components, true, false);
return as<JsonData>(getAsset(AssetId{AssetType::Json, std::move(components)}))->json;
2023-06-20 04:33:09 +00:00
}
Json Assets::fetchJson(Json const& v, String const& dir) const {
if (v.isType(Json::Type::String))
return Assets::json(AssetPath::relativeTo(dir, v.toString()));
else
return v;
}
void Assets::queueJsons(StringList const& paths) const {
queueAssets(paths.transformed([](String const& path) {
auto components = AssetPath::split(path);
validatePath(components, true, false);
return AssetId{AssetType::Json, {components.basePath, {}, {}}};
}));
}
void Assets::queueJsons(CaseInsensitiveStringSet const& paths) const {
2024-03-15 10:28:11 +00:00
MutexLocker assetsLocker(m_assetsMutex);
for (String const& path : paths) {
auto components = AssetPath::split(path);
validatePath(components, true, false);
queueAsset(AssetId{AssetType::Json, {components.basePath, {}, {}}});
};
}
2023-06-24 12:49:47 +00:00
ImageConstPtr Assets::image(AssetPath const& path) const {
return as<ImageData>(getAsset(AssetId{AssetType::Image, path}))->image;
2023-06-20 04:33:09 +00:00
}
void Assets::queueImages(StringList const& paths) const {
queueAssets(paths.transformed([](String const& path) {
auto components = AssetPath::split(path);
2023-06-20 04:33:09 +00:00
validatePath(components, true, true);
return AssetId{AssetType::Image, std::move(components)};
2023-06-20 04:33:09 +00:00
}));
}
void Assets::queueImages(CaseInsensitiveStringSet const& paths) const {
2024-03-15 10:28:11 +00:00
MutexLocker assetsLocker(m_assetsMutex);
for (String const& path : paths) {
auto components = AssetPath::split(path);
validatePath(components, true, true);
queueAsset(AssetId{AssetType::Image, std::move(components)});
};
}
2023-06-24 12:49:47 +00:00
ImageConstPtr Assets::tryImage(AssetPath const& path) const {
validatePath(path, true, true);
2023-06-20 04:33:09 +00:00
2023-06-24 12:49:47 +00:00
if (auto imageData = as<ImageData>(tryAsset(AssetId{AssetType::Image, path})))
2023-06-20 04:33:09 +00:00
return imageData->image;
else
return {};
}
FramesSpecificationConstPtr Assets::imageFrames(String const& path) const {
auto components = AssetPath::split(path);
validatePath(components, false, false);
MutexLocker assetsLocker(m_assetsMutex);
return bestFramesSpecification(path);
}
AudioConstPtr Assets::audio(String const& path) const {
auto components = AssetPath::split(path);
validatePath(components, false, false);
return as<AudioData>(getAsset(AssetId{AssetType::Audio, std::move(components)}))->audio;
2023-06-20 04:33:09 +00:00
}
void Assets::queueAudios(StringList const& paths) const {
queueAssets(paths.transformed([](String const& path) {
2024-03-15 10:28:11 +00:00
auto components = AssetPath::split(path);
2023-06-20 04:33:09 +00:00
validatePath(components, false, false);
return AssetId{AssetType::Audio, std::move(components)};
2023-06-20 04:33:09 +00:00
}));
}
void Assets::queueAudios(CaseInsensitiveStringSet const& paths) const {
2024-03-15 10:28:11 +00:00
MutexLocker assetsLocker(m_assetsMutex);
for (String const& path : paths) {
auto components = AssetPath::split(path);
validatePath(components, false, true);
queueAsset(AssetId{AssetType::Audio, std::move(components)});
};
}
2023-06-20 04:33:09 +00:00
AudioConstPtr Assets::tryAudio(String const& path) const {
auto components = AssetPath::split(path);
validatePath(components, false, false);
if (auto audioData = as<AudioData>(tryAsset(AssetId{AssetType::Audio, std::move(components)})))
2023-06-20 04:33:09 +00:00
return audioData->audio;
else
return {};
}
FontConstPtr Assets::font(String const& path) const {
auto components = AssetPath::split(path);
validatePath(components, false, false);
return as<FontData>(getAsset(AssetId{AssetType::Font, std::move(components)}))->font;
2023-06-20 04:33:09 +00:00
}
ByteArrayConstPtr Assets::bytes(String const& path) const {
auto components = AssetPath::split(path);
validatePath(components, false, false);
return as<BytesData>(getAsset(AssetId{AssetType::Bytes, std::move(components)}))->bytes;
2023-06-20 04:33:09 +00:00
}
IODevicePtr Assets::openFile(String const& path) const {
return open(path);
}
void Assets::clearCache() {
MutexLocker assetsLocker(m_assetsMutex);
// Clear all assets that are not queued or broken.
auto it = makeSMutableMapIterator(m_assetsCache);
while (it.hasNext()) {
auto const& pair = it.next();
// Don't clean up queued, persistent, or broken assets.
if (pair.second && !pair.second->shouldPersist() && !m_queue.contains(pair.first))
it.remove();
}
}
void Assets::cleanup() {
MutexLocker assetsLocker(m_assetsMutex);
double time = Time::monotonicTime();
auto it = makeSMutableMapIterator(m_assetsCache);
while (it.hasNext()) {
auto pair = it.next();
// Don't clean up broken assets or queued assets.
if (pair.second && !m_queue.contains(pair.first)) {
double liveTime = time - pair.second->time;
if (liveTime > m_settings.assetTimeToLive) {
// If the asset should persist, just refresh the access time.
if (pair.second->shouldPersist())
pair.second->time = time;
else
it.remove();
}
}
}
}
bool Assets::AssetId::operator==(AssetId const& assetId) const {
return tie(type, path) == tie(assetId.type, assetId.path);
}
size_t Assets::AssetIdHash::operator()(AssetId const& id) const {
return hashOf(id.type, id.path.basePath, id.path.subPath, id.path.directives);
}
bool Assets::JsonData::shouldPersist() const {
return !json.unique();
}
bool Assets::ImageData::shouldPersist() const {
return !alias && !image.unique();
}
bool Assets::AudioData::shouldPersist() const {
return !audio.unique();
}
bool Assets::FontData::shouldPersist() const {
return !font.unique();
}
bool Assets::BytesData::shouldPersist() const {
return !bytes.unique();
}
FramesSpecification Assets::parseFramesSpecification(Json const& frameConfig, String path) {
FramesSpecification framesSpecification;
framesSpecification.framesFile = std::move(path);
2023-06-20 04:33:09 +00:00
if (frameConfig.contains("frameList")) {
for (auto const& pair : frameConfig.get("frameList").toObject()) {
String frameName = pair.first;
RectU rect = RectU(jsonToRectI(pair.second));
if (rect.isEmpty())
throw AssetException(
2023-06-27 10:23:44 +00:00
strf("Empty rect in frame specification in image {} frame {}", framesSpecification.framesFile, frameName));
2023-06-20 04:33:09 +00:00
else
framesSpecification.frames[frameName] = rect;
}
}
if (frameConfig.contains("frameGrid")) {
auto grid = frameConfig.get("frameGrid").toObject();
Vec2U begin(jsonToVec2I(grid.value("begin", jsonFromVec2I(Vec2I()))));
Vec2U size(jsonToVec2I(grid.get("size")));
Vec2U dimensions(jsonToVec2I(grid.get("dimensions")));
if (dimensions[0] == 0 || dimensions[1] == 0)
2023-06-27 10:23:44 +00:00
throw AssetException(strf("Image {} \"dimensions\" in frameGrid cannot be zero", framesSpecification.framesFile));
2023-06-20 04:33:09 +00:00
if (grid.contains("names")) {
auto nameList = grid.get("names");
for (size_t y = 0; y < nameList.size(); ++y) {
if (y >= dimensions[1])
2023-06-27 10:23:44 +00:00
throw AssetException(strf("Image {} row {} is out of bounds for y-dimension {}",
2023-06-20 04:33:09 +00:00
framesSpecification.framesFile,
y + 1,
dimensions[1]));
auto rowList = nameList.get(y);
if (rowList.isNull())
continue;
for (unsigned x = 0; x < rowList.size(); ++x) {
if (x >= dimensions[0])
2023-06-27 10:23:44 +00:00
throw AssetException(strf("Image {} column {} is out of bounds for x-dimension {}",
2023-06-20 04:33:09 +00:00
framesSpecification.framesFile,
x + 1,
dimensions[0]));
auto frame = rowList.get(x);
if (frame.isNull())
continue;
auto frameName = frame.toString();
if (!frameName.empty())
framesSpecification.frames[frameName] =
RectU::withSize(Vec2U(begin[0] + x * size[0], begin[1] + y * size[1]), size);
}
}
} else {
// If "names" not specified, use auto naming algorithm
for (size_t y = 0; y < dimensions[1]; ++y)
for (size_t x = 0; x < dimensions[0]; ++x)
framesSpecification.frames[toString(y * dimensions[0] + x)] =
2023-06-20 04:33:09 +00:00
RectU::withSize(Vec2U(begin[0] + x * size[0], begin[1] + y * size[1]), size);
}
}
if (auto aliasesConfig = frameConfig.opt("aliases")) {
auto aliases = aliasesConfig->objectPtr();
for (auto const& pair : *aliases) {
String const& key = pair.first;
String value = pair.second.toString();
// Resolve aliases to aliases by checking to see if the alias value in
// the alias map itself. Don't do this more than aliases.size() times to
// avoid infinite cycles.
for (size_t i = 0; i <= aliases->size(); ++i) {
auto it = aliases->find(value);
if (it != aliases->end()) {
if (i == aliases->size())
2023-06-27 10:23:44 +00:00
throw AssetException(strf("Infinite alias loop detected for alias '{}'", key));
2023-06-20 04:33:09 +00:00
value = it->second.toString();
} else {
break;
}
}
if (!framesSpecification.frames.contains(value))
2023-06-27 10:23:44 +00:00
throw AssetException(strf("No such frame '{}' found for alias '{}'", value, key));
framesSpecification.aliases[key] = std::move(value);
2023-06-20 04:33:09 +00:00
}
}
return framesSpecification;
}
void Assets::queueAssets(List<AssetId> const& assetIds) const {
MutexLocker assetsLocker(m_assetsMutex);
2024-03-15 10:28:11 +00:00
for (auto const& id : assetIds)
queueAsset(id);
}
void Assets::queueAsset(AssetId const& assetId) const {
auto i = m_assetsCache.find(assetId);
if (i != m_assetsCache.end()) {
if (i->second)
freshen(i->second);
} else {
auto j = m_queue.find(assetId);
if (j == m_queue.end()) {
m_queue[assetId] = QueuePriority::Load;
m_assetsQueued.signal();
2023-06-20 04:33:09 +00:00
}
}
}
shared_ptr<Assets::AssetData> Assets::tryAsset(AssetId const& id) const {
MutexLocker assetsLocker(m_assetsMutex);
auto i = m_assetsCache.find(id);
if (i != m_assetsCache.end()) {
if (i->second) {
freshen(i->second);
return i->second;
} else {
2023-06-27 10:23:44 +00:00
throw AssetException::format("Error loading asset {}", id.path);
2023-06-20 04:33:09 +00:00
}
} else {
auto j = m_queue.find(id);
if (j == m_queue.end()) {
m_queue[id] = QueuePriority::Load;
m_assetsQueued.signal();
}
return {};
}
}
shared_ptr<Assets::AssetData> Assets::getAsset(AssetId const& id) const {
MutexLocker assetsLocker(m_assetsMutex);
while (true) {
auto j = m_assetsCache.find(id);
if (j != m_assetsCache.end()) {
if (j->second) {
auto asset = j->second;
freshen(asset);
return asset;
} else {
2023-06-27 10:23:44 +00:00
throw AssetException::format("Error loading asset {}", id.path);
2023-06-20 04:33:09 +00:00
}
} else {
// Try to load the asset in-thread, if we cannot, then the asset has been
// queued so wait for a worker thread to finish it.
if (!doLoad(id))
m_assetsDone.wait(m_assetsMutex);
}
}
}
void Assets::workerMain() {
while (true) {
if (m_stopThreads)
break;
MutexLocker assetsLocker(m_assetsMutex);
AssetId assetId;
QueuePriority queuePriority = QueuePriority::None;
// Find the highest priority queue entry
for (auto const& pair : m_queue) {
if (pair.second == QueuePriority::Load || pair.second == QueuePriority::PostProcess) {
assetId = pair.first;
queuePriority = pair.second;
if (pair.second == QueuePriority::Load)
break;
}
}
if (queuePriority != QueuePriority::Load && queuePriority != QueuePriority::PostProcess) {
// Nothing in the queue that needs work
m_assetsQueued.wait(m_assetsMutex);
continue;
}
bool workIsBlocking;
if (queuePriority == QueuePriority::PostProcess)
workIsBlocking = !doPost(assetId);
else
workIsBlocking = !doLoad(assetId);
if (workIsBlocking) {
// We are blocking on some sort of busy asset, so need to wait on
// something to complete here, rather than spinning and burning cpu.
m_assetsDone.wait(m_assetsMutex);
continue;
}
// After processing an asset, unlock the main asset mutex and yield so we
// don't starve other threads.
assetsLocker.unlock();
Thread::yield();
}
}
template <typename Function>
decltype(auto) Assets::unlockDuring(Function f) const {
m_assetsMutex.unlock();
try {
auto r = f();
m_assetsMutex.lock();
return r;
} catch (...) {
m_assetsMutex.lock();
throw;
}
}
FramesSpecificationConstPtr Assets::bestFramesSpecification(String const& image) const {
if (auto framesSpecification = m_framesSpecifications.maybe(image)) {
return *framesSpecification;
}
String framesFile;
if (auto bestFramesFile = m_bestFramesFiles.maybe(image)) {
framesFile = *bestFramesFile;
} else {
String searchPath = AssetPath::directory(image);
String filePrefix = AssetPath::filename(image);
filePrefix = filePrefix.substr(0, filePrefix.findLast('.'));
auto subdir = [](String const& dir) -> String {
auto dirsplit = dir.substr(0, dir.size() - 1).rsplit("/", 1);
if (dirsplit.size() < 2)
return "";
else
return dirsplit[0] + "/";
};
Maybe<String> foundFramesFile;
// look for <full-path-minus-extension>.frames or default.frames up to root
while (!searchPath.empty()) {
String framesPath = searchPath + filePrefix + ".frames";
if (m_files.contains(framesPath)) {
foundFramesFile = framesPath;
break;
}
framesPath = searchPath + "default.frames";
if (m_files.contains(framesPath)) {
foundFramesFile = framesPath;
break;
}
searchPath = subdir(searchPath);
}
if (foundFramesFile) {
framesFile = foundFramesFile.take();
m_bestFramesFiles[image] = framesFile;
} else {
return {};
}
}
auto framesSpecification = unlockDuring([&]() {
return make_shared<FramesSpecification>(parseFramesSpecification(readJson(framesFile), framesFile));
});
m_framesSpecifications[image] = framesSpecification;
return framesSpecification;
}
IODevicePtr Assets::open(String const& path) const {
if (auto p = m_files.ptr(path))
return p->source->open(p->sourceName);
2023-06-27 10:23:44 +00:00
throw AssetException(strf("No such asset '{}'", path));
2023-06-20 04:33:09 +00:00
}
ByteArray Assets::read(String const& path) const {
if (auto p = m_files.ptr(path))
return p->source->read(p->sourceName);
2023-06-27 10:23:44 +00:00
throw AssetException(strf("No such asset '{}'", path));
2023-06-20 04:33:09 +00:00
}
ImageConstPtr Assets::readImage(String const& path) const {
if (auto p = m_files.ptr(path)) {
ImageConstPtr image;
if (auto memorySource = as<MemoryAssetSource>(p->source))
image = memorySource->image(p->sourceName);
if (!image)
image = make_shared<Image>(Image::readPng(p->source->open(p->sourceName)));
if (!p->patchSources.empty()) {
RecursiveMutexLocker luaLocker(m_luaMutex);
LuaEngine* luaEngine = as<LuaEngine>(m_luaEngine.get());
LuaValue result = luaEngine->createUserData(*image);
luaLocker.unlock();
for (auto const& pair : p->patchSources) {
auto& patchPath = pair.first;
auto& patchSource = pair.second;
auto patchStream = patchSource->read(patchPath);
if (patchPath.endsWith(".lua")) {
luaLocker.lock();
LuaContextPtr& context = m_patchContexts[patchPath];
if (!context) {
context = make_shared<LuaContext>(luaEngine->createContext());
context->load(patchStream, patchPath);
}
auto newResult = context->invokePath<LuaValue>("patch", result, path);
if (!newResult.is<LuaNilType>()) {
if (auto ud = newResult.ptr<LuaUserData>()) {
if (ud->is<Image>())
result = std::move(newResult);
else
Logger::warn("Patch '{}' for image '{}' returned a non-Image userdata value, ignoring");
} else {
Logger::warn("Patch '{}' for image '{}' returned a non-Image value, ignoring");
}
}
luaLocker.unlock();
} else {
Logger::warn("Patch '{}' for image '{}' isn't a Lua script, ignoring", patchPath, path);
}
}
image = make_shared<Image>(std::move(result.get<LuaUserData>().get<Image>()));
}
return image;
}
throw AssetException(strf("No such asset '{}'", path));
}
Json Assets::checkPatchArray(String const& path, AssetSourcePtr const& source, Json const result, JsonArray const patchData, Maybe<Json> const external) const {
auto externalRef = external.value();
auto newResult = result;
for (auto const& patch : patchData) {
2024-03-09 00:09:04 +00:00
switch(patch.type()) {
case Json::Type::Array: // if the patch is an array, go down recursively until we get objects
try {
newResult = checkPatchArray(path, source, newResult, patch.toArray(), externalRef);
} catch (JsonPatchTestFail const& e) {
2024-03-07 23:56:13 +00:00
Logger::debug("Patch test failure from file {} in source: '{}' at '{}'. Caused by: {}", path, source->metadata().value("name", ""), m_assetSourcePaths.getLeft(source), e.what());
} catch (JsonPatchException const& e) {
2024-03-07 23:56:13 +00:00
Logger::error("Could not apply patch from file {} in source: '{}' at '{}'. Caused by: {}", path, source->metadata().value("name", ""), m_assetSourcePaths.getLeft(source), e.what());
}
break;
case Json::Type::Object: // if its an object, check for operations, or for if an external file is needed for patches to reference
newResult = JsonPatching::applyOperation(newResult, patch, externalRef);
break;
case Json::Type::String:
try {
externalRef = json(patch.toString());
} catch (...) {
2024-03-07 23:56:13 +00:00
throw JsonPatchTestFail(strf("Unable to load reference asset: {}", patch.toString()));
}
break;
default:
2024-03-07 23:56:13 +00:00
throw JsonPatchException(strf("Patch data is wrong type: {}", Json::typeName(patch.type())));
break;
}
}
return newResult;
}
2023-06-20 04:33:09 +00:00
Json Assets::readJson(String const& path) const {
ByteArray streamData = read(path);
try {
Json result = inputUtf8Json(streamData.begin(), streamData.end(), JsonParseType::Top);
2023-06-20 04:33:09 +00:00
for (auto const& pair : m_files.get(path).patchSources) {
2024-06-25 09:56:44 +00:00
auto patchAssetPath = AssetPath::split(pair.first);
auto& patchBasePath = patchAssetPath.basePath;
2024-03-20 13:57:49 +00:00
auto& patchSource = pair.second;
2024-06-25 09:56:44 +00:00
auto patchStream = patchSource->read(patchBasePath);
if (patchBasePath.endsWith(".lua")) {
RecursiveMutexLocker luaLocker(m_luaMutex);
// Kae: i don't like that lock. perhaps have a LuaEngine and patch context cache per worker thread later on?
2024-06-25 09:56:44 +00:00
LuaContextPtr& context = m_patchContexts[patchBasePath];
if (!context) {
context = make_shared<LuaContext>(as<LuaEngine>(m_luaEngine.get())->createContext());
2024-06-25 09:56:44 +00:00
context->load(patchStream, patchBasePath);
2023-06-20 04:33:09 +00:00
}
auto newResult = context->invokePath<Json>("patch", result, path);
if (newResult)
result = std::move(newResult);
} else {
auto patchJson = inputUtf8Json(patchStream.begin(), patchStream.end(), JsonParseType::Top);
2024-06-25 09:56:44 +00:00
if (patchAssetPath.subPath)
patchJson = patchJson.query(*patchAssetPath.subPath);
if (patchJson.isType(Json::Type::Array)) {
2024-06-25 09:56:44 +00:00
auto patchData = patchJson.toArray();
try {
result = checkPatchArray(pair.first, patchSource, result, patchData, {});
} catch (JsonPatchTestFail const& e) {
Logger::debug("Patch test failure from file {} in source: '{}' at '{}'. Caused by: {}", pair.first, patchSource->metadata().value("name", ""), m_assetSourcePaths.getLeft(patchSource), e.what());
} catch (JsonPatchException const& e) {
Logger::error("Could not apply patch from file {} in source: '{}' at '{}'. Caused by: {}", pair.first, patchSource->metadata().value("name", ""), m_assetSourcePaths.getLeft(patchSource), e.what());
}
} else if (patchJson.isType(Json::Type::Object)) {
result = jsonMergeNulling(result, patchJson.toObject());
}
}
2023-06-20 04:33:09 +00:00
}
return result;
} catch (std::exception const& e) {
2023-06-27 10:23:44 +00:00
throw JsonParsingException(strf("Cannot parse json file: {}", path), e);
2023-06-20 04:33:09 +00:00
}
}
2023-06-20 04:33:09 +00:00
bool Assets::doLoad(AssetId const& id) const {
try {
// loadAsset automatically manages the queue and freshens the asset
// data.
return (bool)loadAsset(id);
} catch (std::exception const& e) {
2023-06-27 10:23:44 +00:00
Logger::error("Exception caught loading asset: {}, {}", id.path, outputException(e, true));
2023-06-20 04:33:09 +00:00
} catch (...) {
2023-06-27 10:23:44 +00:00
Logger::error("Unknown exception caught loading asset: {}", id.path);
2023-06-20 04:33:09 +00:00
}
// There was an exception, remove the asset from the queue and fill the cache
// with null so that getAsset will throw.
m_assetsCache[id] = {};
m_assetsDone.broadcast();
m_queue.remove(id);
return true;
}
bool Assets::doPost(AssetId const& id) const {
shared_ptr<AssetData> assetData;
try {
assetData = m_assetsCache.get(id);
if (id.type == AssetType::Audio)
assetData = postProcessAudio(assetData);
} catch (std::exception const& e) {
2023-06-27 10:23:44 +00:00
Logger::error("Exception caught post-processing asset: {}, {}", id.path, outputException(e, true));
2023-06-20 04:33:09 +00:00
} catch (...) {
2023-06-27 10:23:44 +00:00
Logger::error("Unknown exception caught post-processing asset: {}", id.path);
2023-06-20 04:33:09 +00:00
}
m_queue.remove(id);
if (assetData) {
assetData->needsPostProcessing = false;
m_assetsCache[id] = assetData;
freshen(assetData);
m_assetsDone.broadcast();
}
return true;
}
shared_ptr<Assets::AssetData> Assets::loadAsset(AssetId const& id) const {
if (auto asset = m_assetsCache.value(id))
return asset;
if (m_queue.value(id, QueuePriority::None) == QueuePriority::Working)
return {};
try {
m_queue[id] = QueuePriority::Working;
shared_ptr<AssetData> assetData;
try {
if (id.type == AssetType::Json) {
assetData = loadJson(id.path);
} else if (id.type == AssetType::Image) {
assetData = loadImage(id.path);
} else if (id.type == AssetType::Audio) {
assetData = loadAudio(id.path);
} else if (id.type == AssetType::Font) {
assetData = loadFont(id.path);
} else if (id.type == AssetType::Bytes) {
assetData = loadBytes(id.path);
}
} catch (StarException const& e) {
if (id.type == AssetType::Image && m_settings.missingImage) {
2023-06-27 10:23:44 +00:00
Logger::error("Could not load image asset '{}', using placeholder default.\n{}", id.path, outputException(e, false));
2023-06-20 04:33:09 +00:00
assetData = loadImage({*m_settings.missingImage, {}, {}});
} else if (id.type == AssetType::Audio && m_settings.missingAudio) {
2023-06-27 10:23:44 +00:00
Logger::error("Could not load audio asset '{}', using placeholder default.\n{}", id.path, outputException(e, false));
2023-06-20 04:33:09 +00:00
assetData = loadAudio({*m_settings.missingAudio, {}, {}});
} else {
throw;
}
}
if (assetData) {
if (assetData->needsPostProcessing)
m_queue[id] = QueuePriority::PostProcess;
else
m_queue.remove(id);
m_assetsCache[id] = assetData;
m_assetsDone.broadcast();
freshen(assetData);
} else {
// We have failed to load an asset because it depends on an asset
// currently being worked on. Mark it as needing loading and move it to
// the end of the queue.
m_queue[id] = QueuePriority::Load;
m_assetsQueued.signal();
m_queue.toBack(id);
}
return assetData;
} catch (...) {
m_queue.remove(id);
m_assetsCache[id] = {};
m_assetsDone.broadcast();
throw;
}
}
shared_ptr<Assets::AssetData> Assets::loadJson(AssetPath const& path) const {
Json json;
if (path.subPath) {
auto topJson =
as<JsonData>(loadAsset(AssetId{AssetType::Json, {path.basePath, {}, {}}}));
if (!topJson)
return {};
try {
auto newData = make_shared<JsonData>();
newData->json = topJson->json.query(*path.subPath);
return newData;
} catch (StarException const& e) {
2023-06-27 10:23:44 +00:00
throw AssetException(strf("Could not read JSON value {}", path), e);
2023-06-20 04:33:09 +00:00
}
} else {
return unlockDuring([&]() {
try {
auto newData = make_shared<JsonData>();
newData->json = readJson(path.basePath);
return newData;
} catch (StarException const& e) {
2023-06-27 10:23:44 +00:00
throw AssetException(strf("Could not read JSON asset {}", path), e);
2023-06-20 04:33:09 +00:00
}
});
}
}
shared_ptr<Assets::AssetData> Assets::loadImage(AssetPath const& path) const {
validatePath(path, true, true);
2023-06-20 04:33:09 +00:00
if (!path.directives.empty()) {
shared_ptr<ImageData> source =
as<ImageData>(loadAsset(AssetId{AssetType::Image, {path.basePath, path.subPath, {}}}));
if (!source)
return {};
StringMap<ImageConstPtr> references;
2023-06-24 09:41:52 +00:00
StringList referencePaths;
for (auto& directives : path.directives.list())
directives.loadOperations();
path.directives.forEach([&](auto const& entry, Directives const&) {
2023-06-24 09:41:52 +00:00
addImageOperationReferences(entry.operation, referencePaths);
}); // TODO: This can definitely be better, was changed quickly to support the new Directives.
for (auto const& ref : referencePaths) {
2023-06-20 04:33:09 +00:00
auto components = AssetPath::split(ref);
validatePath(components, true, false);
auto refImage = as<ImageData>(loadAsset(AssetId{AssetType::Image, std::move(components)}));
2023-06-20 04:33:09 +00:00
if (!refImage)
return {};
references[ref] = refImage->image;
}
return unlockDuring([&]() {
auto newData = make_shared<ImageData>();
2023-06-24 09:41:52 +00:00
Image newImage = *source->image;
2024-04-25 00:05:19 +00:00
path.directives.forEach([&](Directives::Entry const& entry, Directives const&) {
if (auto error = entry.operation.ptr<ErrorImageOperation>())
if (auto string = error->cause.ptr<std::string>())
throw DirectivesException::format("ImageOperation parse error: {}", *string);
else
std::rethrow_exception(error->cause.get<std::exception_ptr>());
else
processImageOperation(entry.operation, newImage, [&](String const& ref) { return references.get(ref).get(); });
2023-06-24 09:41:52 +00:00
});
newData->image = make_shared<Image>(std::move(newImage));
2023-06-20 04:33:09 +00:00
return newData;
});
} else if (path.subPath) {
auto imageData = as<ImageData>(loadAsset(AssetId{AssetType::Image, {path.basePath, {}, {}}}));
if (!imageData)
return {};
// Base image must have frames data associated with it.
if (!imageData->frames)
2023-06-27 10:23:44 +00:00
throw AssetException::format("No associated frames file found for image '{}' while resolving image frame '{}'", path.basePath, path);
2023-06-20 04:33:09 +00:00
if (auto alias = imageData->frames->aliases.ptr(*path.subPath)) {
imageData = as<ImageData>(loadAsset(AssetId{AssetType::Image, {path.basePath, *alias, path.directives}}));
if (!imageData)
return {};
auto newData = make_shared<ImageData>();
newData->image = imageData->image;
newData->alias = true;
return newData;
} else {
auto frameRect = imageData->frames->frames.ptr(*path.subPath);
if (!frameRect)
2023-06-27 10:23:44 +00:00
throw AssetException(strf("No such frame {} in frames spec {}", *path.subPath, imageData->frames->framesFile));
2023-06-20 04:33:09 +00:00
return unlockDuring([&]() {
// Need to flip frame coordinates because frame configs assume top
// down image coordinates
auto newData = make_shared<ImageData>();
newData->image = make_shared<Image>(imageData->image->subImage(
Vec2U(frameRect->xMin(), imageData->image->height() - frameRect->yMax()), frameRect->size()));
return newData;
});
}
} else {
auto imageData = make_shared<ImageData>();
imageData->image = unlockDuring([&]() {
return readImage(path.basePath);
2023-06-20 04:33:09 +00:00
});
imageData->frames = bestFramesSpecification(path.basePath);
return imageData;
}
}
shared_ptr<Assets::AssetData> Assets::loadAudio(AssetPath const& path) const {
return unlockDuring([&]() {
auto newData = make_shared<AudioData>();
newData->audio = make_shared<Audio>(open(path.basePath), path.basePath);
2023-06-20 04:33:09 +00:00
newData->needsPostProcessing = newData->audio->compressed();
return newData;
});
}
shared_ptr<Assets::AssetData> Assets::loadFont(AssetPath const& path) const {
return unlockDuring([&]() {
auto newData = make_shared<FontData>();
2024-03-25 01:49:18 +00:00
newData->font = Font::loadFont(make_shared<ByteArray>(read(path.basePath)));
2023-06-20 04:33:09 +00:00
return newData;
});
}
shared_ptr<Assets::AssetData> Assets::loadBytes(AssetPath const& path) const {
return unlockDuring([&]() {
auto newData = make_shared<BytesData>();
newData->bytes = make_shared<ByteArray>(read(path.basePath));
return newData;
});
}
shared_ptr<Assets::AssetData> Assets::postProcessAudio(shared_ptr<AssetData> const& original) const {
return unlockDuring([&]() -> shared_ptr<AssetData> {
if (auto audioData = as<AudioData>(original)) {
if (audioData->audio->totalTime() < m_settings.audioDecompressLimit) {
auto audio = make_shared<Audio>(*audioData->audio);
audio->uncompress();
auto newData = make_shared<AudioData>();
newData->audio = audio;
return newData;
} else {
return audioData;
}
} else {
return {};
}
});
}
void Assets::freshen(shared_ptr<AssetData> const& asset) const {
asset->time = Time::monotonicTime();
}
}