diff --git a/src/citra_qt/game_list.cpp b/src/citra_qt/game_list.cpp index 2d5af1b0cf..8a5df51721 100644 --- a/src/citra_qt/game_list.cpp +++ b/src/citra_qt/game_list.cpp @@ -468,6 +468,8 @@ void GameList::AddGamePopup(QMenu& context_menu, const QString& path, u64 progra QAction* open_texture_dump_location = context_menu.addAction(tr("Open Texture Dump Location")); QAction* open_texture_load_location = context_menu.addAction(tr("Open Custom Texture Location")); + QAction* open_mods_location = context_menu.addAction(tr("Open Mods Location")); + QAction* dump_romfs = context_menu.addAction(tr("Dump RomFS")); QAction* navigate_to_gamedb_entry = context_menu.addAction(tr("Navigate to GameDB entry")); const bool is_application = @@ -497,6 +499,8 @@ void GameList::AddGamePopup(QMenu& context_menu, const QString& path, u64 progra open_texture_dump_location->setVisible(is_application); open_texture_load_location->setVisible(is_application); + open_mods_location->setVisible(is_application); + dump_romfs->setVisible(is_application); navigate_to_gamedb_entry->setVisible(it != compatibility_list.end()); @@ -526,6 +530,15 @@ void GameList::AddGamePopup(QMenu& context_menu, const QString& path, u64 progra emit OpenFolderRequested(program_id, GameListOpenTarget::TEXTURE_LOAD); } }); + connect(open_mods_location, &QAction::triggered, [this, program_id] { + if (FileUtil::CreateFullPath(fmt::format("{}mods/{:016X}/", + FileUtil::GetUserPath(FileUtil::UserPath::LoadDir), + program_id))) { + emit OpenFolderRequested(program_id, GameListOpenTarget::MODS); + } + }); + connect(dump_romfs, &QAction::triggered, + [this, path, program_id] { emit DumpRomFSRequested(path, program_id); }); connect(navigate_to_gamedb_entry, &QAction::triggered, [this, program_id]() { emit NavigateToGamedbEntryRequested(program_id, compatibility_list); }); diff --git a/src/citra_qt/game_list.h b/src/citra_qt/game_list.h index ef280ef043..3340890375 100644 --- a/src/citra_qt/game_list.h +++ b/src/citra_qt/game_list.h @@ -35,7 +35,8 @@ enum class GameListOpenTarget { APPLICATION = 2, UPDATE_DATA = 3, TEXTURE_DUMP = 4, - TEXTURE_LOAD = 5 + TEXTURE_LOAD = 5, + MODS = 6, }; class GameList : public QWidget { @@ -81,6 +82,7 @@ signals: void OpenFolderRequested(u64 program_id, GameListOpenTarget target); void NavigateToGamedbEntryRequested(u64 program_id, const CompatibilityList& compatibility_list); + void DumpRomFSRequested(QString game_path, u64 program_id); void OpenDirectory(const QString& directory); void AddDirectory(); void ShowList(bool show); diff --git a/src/citra_qt/main.cpp b/src/citra_qt/main.cpp index daccbb083c..0c87e98e10 100644 --- a/src/citra_qt/main.cpp +++ b/src/citra_qt/main.cpp @@ -568,6 +568,7 @@ void GMainWindow::ConnectWidgetEvents() { connect(game_list, &GameList::OpenFolderRequested, this, &GMainWindow::OnGameListOpenFolder); connect(game_list, &GameList::NavigateToGamedbEntryRequested, this, &GMainWindow::OnGameListNavigateToGamedbEntry); + connect(game_list, &GameList::DumpRomFSRequested, this, &GMainWindow::OnGameListDumpRomFS); connect(game_list, &GameList::AddDirectory, this, &GMainWindow::OnGameListAddDirectory); connect(game_list_placeholder, &GameListPlaceholder::AddDirectory, this, &GMainWindow::OnGameListAddDirectory); @@ -1144,6 +1145,11 @@ void GMainWindow::OnGameListOpenFolder(u64 data_id, GameListOpenTarget target) { path = fmt::format("{}textures/{:016X}/", FileUtil::GetUserPath(FileUtil::UserPath::LoadDir), data_id); break; + case GameListOpenTarget::MODS: + open_target = "Mods"; + path = fmt::format("{}mods/{:016X}/", FileUtil::GetUserPath(FileUtil::UserPath::LoadDir), + data_id); + break; default: LOG_ERROR(Frontend, "Unexpected target {}", static_cast(target)); return; @@ -1175,6 +1181,46 @@ void GMainWindow::OnGameListNavigateToGamedbEntry(u64 program_id, QDesktopServices::openUrl(QUrl(QStringLiteral("https://citra-emu.org/game/") + directory)); } +void GMainWindow::OnGameListDumpRomFS(QString game_path, u64 program_id) { + auto* dialog = new QProgressDialog(tr("Dumping..."), tr("Cancel"), 0, 0, this); + dialog->setWindowModality(Qt::WindowModal); + dialog->setWindowFlags(dialog->windowFlags() & + ~(Qt::WindowCloseButtonHint | Qt::WindowContextHelpButtonHint)); + dialog->setCancelButton(nullptr); + dialog->setMinimumDuration(0); + dialog->setValue(0); + + const auto base_path = fmt::format( + "{}romfs/{:016X}", FileUtil::GetUserPath(FileUtil::UserPath::DumpDir), program_id); + const auto update_path = + fmt::format("{}romfs/{:016X}", FileUtil::GetUserPath(FileUtil::UserPath::DumpDir), + program_id | 0x0004000e00000000); + using FutureWatcher = QFutureWatcher>; + auto* future_watcher = new FutureWatcher(this); + connect(future_watcher, &FutureWatcher::finished, + [this, program_id, dialog, base_path, update_path, future_watcher] { + dialog->hide(); + const auto& [base, update] = future_watcher->result(); + if (base != Loader::ResultStatus::Success) { + QMessageBox::critical( + this, tr("Citra"), + tr("Could not dump base RomFS.\nRefer to the log for details.")); + return; + } + QDesktopServices::openUrl(QUrl::fromLocalFile(QString::fromStdString(base_path))); + if (update == Loader::ResultStatus::Success) { + QDesktopServices::openUrl( + QUrl::fromLocalFile(QString::fromStdString(update_path))); + } + }); + + auto future = QtConcurrent::run([game_path, base_path, update_path] { + std::unique_ptr loader = Loader::GetLoader(game_path.toStdString()); + return std::make_pair(loader->DumpRomFS(base_path), loader->DumpUpdateRomFS(update_path)); + }); + future_watcher->setFuture(future); +} + void GMainWindow::OnGameListOpenDirectory(const QString& directory) { QString path; if (directory == QStringLiteral("INSTALLED")) { diff --git a/src/citra_qt/main.h b/src/citra_qt/main.h index 91d7eed1b4..75a1bbb3d3 100644 --- a/src/citra_qt/main.h +++ b/src/citra_qt/main.h @@ -169,6 +169,7 @@ private slots: void OnGameListOpenFolder(u64 program_id, GameListOpenTarget target); void OnGameListNavigateToGamedbEntry(u64 program_id, const CompatibilityList& compatibility_list); + void OnGameListDumpRomFS(QString game_path, u64 program_id); void OnGameListOpenDirectory(const QString& directory); void OnGameListAddDirectory(); void OnGameListShowList(bool show); diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt index 423d13630a..ab9f52e325 100644 --- a/src/core/CMakeLists.txt +++ b/src/core/CMakeLists.txt @@ -72,6 +72,8 @@ add_library(core STATIC file_sys/delay_generator.h file_sys/ivfc_archive.cpp file_sys/ivfc_archive.h + file_sys/layered_fs.cpp + file_sys/layered_fs.h file_sys/ncch_container.cpp file_sys/ncch_container.h file_sys/patch.cpp diff --git a/src/core/core.cpp b/src/core/core.cpp index cd1799e425..01ab8481c4 100644 --- a/src/core/core.cpp +++ b/src/core/core.cpp @@ -433,6 +433,7 @@ void System::Shutdown() { perf_stats.reset(); rpc_server.reset(); cheat_engine.reset(); + archive_manager.reset(); service_manager.reset(); dsp_core.reset(); cpu_cores.clear(); diff --git a/src/core/file_sys/layered_fs.cpp b/src/core/file_sys/layered_fs.cpp new file mode 100644 index 0000000000..9d5fbf7c21 --- /dev/null +++ b/src/core/file_sys/layered_fs.cpp @@ -0,0 +1,604 @@ +// Copyright 2020 Citra Emulator Project +// Licensed under GPLv2 or any later version +// Refer to the license.txt file included. + +#include +#include +#include "common/alignment.h" +#include "common/assert.h" +#include "common/common_paths.h" +#include "common/file_util.h" +#include "common/string_util.h" +#include "common/swap.h" +#include "core/file_sys/layered_fs.h" +#include "core/file_sys/patch.h" + +namespace FileSys { + +struct FileRelocationInfo { + int type; // 0 - none, 1 - replaced / created, 2 - patched, 3 - removed + u64 original_offset; // Type 0. Offset is absolute + std::string replace_file_path; // Type 1 + std::vector patched_file; // Type 2 + u64 size; // Relocated file size +}; +struct LayeredFS::File { + std::string name; + std::string path; + FileRelocationInfo relocation{}; + Directory* parent; +}; + +struct DirectoryMetadata { + u32_le parent_directory_offset; + u32_le next_sibling_offset; + u32_le first_child_directory_offset; + u32_le first_file_offset; + u32_le hash_bucket_next; + u32_le name_length; + // Followed by a name of name length (aligned up to 4) +}; +static_assert(sizeof(DirectoryMetadata) == 0x18, "Size of DirectoryMetadata is not correct"); + +struct FileMetadata { + u32_le parent_directory_offset; + u32_le next_sibling_offset; + u64_le file_data_offset; + u64_le file_data_length; + u32_le hash_bucket_next; + u32_le name_length; + // Followed by a name of name length (aligned up to 4) +}; +static_assert(sizeof(FileMetadata) == 0x20, "Size of FileMetadata is not correct"); + +LayeredFS::LayeredFS(std::shared_ptr romfs_, std::string patch_path_, + std::string patch_ext_path_, bool load_relocations) + : romfs(std::move(romfs_)), patch_path(std::move(patch_path_)), + patch_ext_path(std::move(patch_ext_path_)) { + + romfs->ReadFile(0, sizeof(header), reinterpret_cast(&header)); + + ASSERT_MSG(header.header_length == sizeof(header), "Header size is incorrect"); + + // TODO: is root always the first directory in table? + root.parent = &root; + LoadDirectory(root, 0); + + if (load_relocations) { + LoadRelocations(); + LoadExtRelocations(); + } + + RebuildMetadata(); +} + +LayeredFS::~LayeredFS() = default; + +void LayeredFS::LoadDirectory(Directory& current, u32 offset) { + DirectoryMetadata metadata; + romfs->ReadFile(header.directory_metadata_table.offset + offset, sizeof(metadata), + reinterpret_cast(&metadata)); + + current.name = ReadName(header.directory_metadata_table.offset + offset + sizeof(metadata), + metadata.name_length); + current.path = current.parent->path + current.name + DIR_SEP; + directory_path_map.emplace(current.path, ¤t); + + if (metadata.first_file_offset != 0xFFFFFFFF) { + LoadFile(current, metadata.first_file_offset); + } + + if (metadata.first_child_directory_offset != 0xFFFFFFFF) { + auto child = std::make_unique(); + auto& directory = *child; + directory.parent = ¤t; + current.directories.emplace_back(std::move(child)); + LoadDirectory(directory, metadata.first_child_directory_offset); + } + + if (metadata.next_sibling_offset != 0xFFFFFFFF) { + auto sibling = std::make_unique(); + auto& directory = *sibling; + directory.parent = current.parent; + current.parent->directories.emplace_back(std::move(sibling)); + LoadDirectory(directory, metadata.next_sibling_offset); + } +} + +void LayeredFS::LoadFile(Directory& parent, u32 offset) { + FileMetadata metadata; + romfs->ReadFile(header.file_metadata_table.offset + offset, sizeof(metadata), + reinterpret_cast(&metadata)); + + auto file = std::make_unique(); + file->name = ReadName(header.file_metadata_table.offset + offset + sizeof(metadata), + metadata.name_length); + file->path = parent.path + file->name; + file->relocation.original_offset = header.file_data_offset + metadata.file_data_offset; + file->relocation.size = metadata.file_data_length; + file->parent = &parent; + + file_path_map.emplace(file->path, file.get()); + parent.files.emplace_back(std::move(file)); + + if (metadata.next_sibling_offset != 0xFFFFFFFF) { + LoadFile(parent, metadata.next_sibling_offset); + } +} + +std::string LayeredFS::ReadName(u32 offset, u32 name_length) { + std::vector buffer(name_length / sizeof(u16_le)); + romfs->ReadFile(offset, name_length, reinterpret_cast(buffer.data())); + + std::u16string name(buffer.size(), 0); + std::transform(buffer.begin(), buffer.end(), name.begin(), [](u16_le character) { + return static_cast(static_cast(character)); + }); + return Common::UTF16ToUTF8(name); +} + +void LayeredFS::LoadRelocations() { + if (!FileUtil::Exists(patch_path)) { + return; + } + + const FileUtil::DirectoryEntryCallable callback = [this, + &callback](u64* /*num_entries_out*/, + const std::string& directory, + const std::string& virtual_name) { + auto* parent = directory_path_map.at(directory.substr(patch_path.size() - 1)); + + if (FileUtil::IsDirectory(directory + virtual_name + DIR_SEP)) { + const auto path = (directory + virtual_name + DIR_SEP).substr(patch_path.size() - 1); + if (!directory_path_map.count(path)) { // Add this directory + auto directory = std::make_unique(); + directory->name = virtual_name; + directory->path = path; + directory->parent = parent; + directory_path_map.emplace(path, directory.get()); + parent->directories.emplace_back(std::move(directory)); + LOG_INFO(Service_FS, "LayeredFS created directory {}", path); + } + return FileUtil::ForeachDirectoryEntry(nullptr, directory + virtual_name + DIR_SEP, + callback); + } + + const auto path = (directory + virtual_name).substr(patch_path.size() - 1); + if (!file_path_map.count(path)) { // Newly created file + auto file = std::make_unique(); + file->name = virtual_name; + file->path = path; + file->parent = parent; + file_path_map.emplace(path, file.get()); + parent->files.emplace_back(std::move(file)); + LOG_INFO(Service_FS, "LayeredFS created file {}", path); + } + + auto* file = file_path_map.at(path); + file->relocation.type = 1; + file->relocation.replace_file_path = directory + virtual_name; + file->relocation.size = FileUtil::GetSize(directory + virtual_name); + LOG_INFO(Service_FS, "LayeredFS replacement file in use for {}", path); + return true; + }; + + FileUtil::ForeachDirectoryEntry(nullptr, patch_path, callback); +} + +void LayeredFS::LoadExtRelocations() { + if (!FileUtil::Exists(patch_ext_path)) { + return; + } + + if (patch_ext_path.back() == '/' || patch_ext_path.back() == '\\') { + // ScanDirectoryTree expects a path without trailing '/' + patch_ext_path.erase(patch_ext_path.size() - 1, 1); + } + + FileUtil::FSTEntry result; + FileUtil::ScanDirectoryTree(patch_ext_path, result, 256); + + for (const auto& entry : result.children) { + if (FileUtil::IsDirectory(entry.physicalName)) { + continue; + } + + const auto path = entry.physicalName.substr(patch_ext_path.size()); + if (path.size() >= 5 && path.substr(path.size() - 5) == ".stub") { + // Remove the corresponding file if exists + const auto file_path = path.substr(0, path.size() - 5); + if (file_path_map.count(file_path)) { + auto& file = *file_path_map[file_path]; + file.relocation.type = 3; + file.relocation.size = 0; + file_path_map.erase(file_path); + LOG_INFO(Service_FS, "LayeredFS removed file {}", file_path); + } else { + LOG_WARNING(Service_FS, "LayeredFS file for stub {} not found", path); + } + } else if (path.size() >= 4) { + const auto extension = path.substr(path.size() - 4); + if (extension != ".ips" && extension != ".bps") { + LOG_WARNING(Service_FS, "LayeredFS unknown ext file {}", path); + } + + const auto file_path = path.substr(0, path.size() - 4); + if (!file_path_map.count(file_path)) { + LOG_WARNING(Service_FS, "LayeredFS original file for patch {} not found", path); + continue; + } + + FileUtil::IOFile patch_file(entry.physicalName, "rb"); + if (!patch_file) { + LOG_ERROR(Service_FS, "LayeredFS Could not open file {}", entry.physicalName); + continue; + } + + const auto size = patch_file.GetSize(); + std::vector patch(size); + if (patch_file.ReadBytes(patch.data(), size) != size) { + LOG_ERROR(Service_FS, "LayeredFS Could not read file {}", entry.physicalName); + continue; + } + + auto& file = *file_path_map[file_path]; + std::vector buffer(file.relocation.size); // Original size + romfs->ReadFile(file.relocation.original_offset, buffer.size(), buffer.data()); + + bool ret = false; + if (extension == ".ips") { + ret = Patch::ApplyIpsPatch(patch, buffer); + } else { + ret = Patch::ApplyBpsPatch(patch, buffer); + } + + if (ret) { + LOG_INFO(Service_FS, "LayeredFS patched file {}", file_path); + + file.relocation.type = 2; + file.relocation.size = buffer.size(); + file.relocation.patched_file = std::move(buffer); + } else { + LOG_ERROR(Service_FS, "LayeredFS failed to patch file {}", file_path); + } + } else { + LOG_WARNING(Service_FS, "LayeredFS unknown ext file {}", path); + } + } +} + +std::size_t GetNameSize(const std::string& name) { + std::u16string u16name = Common::UTF8ToUTF16(name); + return Common::AlignUp(u16name.size() * 2, 4); +} + +void LayeredFS::PrepareBuildDirectory(Directory& current) { + directory_metadata_offset_map.emplace(¤t, current_directory_offset); + directory_list.emplace_back(¤t); + current_directory_offset += sizeof(DirectoryMetadata) + GetNameSize(current.name); +} + +void LayeredFS::PrepareBuildFile(File& current) { + if (current.relocation.type == 3) { // Deleted files are not counted + return; + } + file_metadata_offset_map.emplace(¤t, current_file_offset); + file_list.emplace_back(¤t); + current_file_offset += sizeof(FileMetadata) + GetNameSize(current.name); +} + +void LayeredFS::PrepareBuild(Directory& current) { + for (const auto& child : current.files) { + PrepareBuildFile(*child); + } + + for (const auto& child : current.directories) { + PrepareBuildDirectory(*child); + } + + for (const auto& child : current.directories) { + PrepareBuild(*child); + } +} + +// Implementation from 3dbrew +u32 CalcHash(const std::string& name, u32 parent_offset) { + u32 hash = parent_offset ^ 123456789; + std::u16string u16name = Common::UTF8ToUTF16(name); + for (char16_t c : u16name) { + hash = (hash >> 5) | (hash << 27); + hash ^= static_cast(c); + } + return hash; +} + +std::size_t WriteName(u8* dest, std::u16string name) { + const auto buffer_size = Common::AlignUp(name.size() * 2, 4); + std::vector buffer(buffer_size / 2); + std::transform(name.begin(), name.end(), buffer.begin(), [](char16_t character) { + return static_cast(static_cast(character)); + }); + std::memcpy(dest, buffer.data(), buffer_size); + + return buffer_size; +} + +void LayeredFS::BuildDirectories() { + directory_metadata_table.resize(current_directory_offset, 0xFF); + + std::size_t written = 0; + for (const auto& directory : directory_list) { + DirectoryMetadata metadata; + std::memset(&metadata, 0xFF, sizeof(metadata)); + metadata.parent_directory_offset = directory_metadata_offset_map.at(directory->parent); + + if (directory->parent != directory) { + bool flag = false; + for (const auto& sibling : directory->parent->directories) { + if (flag) { + metadata.next_sibling_offset = directory_metadata_offset_map.at(sibling.get()); + break; + } else if (sibling.get() == directory) { + flag = true; + } + } + } + + if (!directory->directories.empty()) { + metadata.first_child_directory_offset = + directory_metadata_offset_map.at(directory->directories.front().get()); + } + + if (!directory->files.empty()) { + metadata.first_file_offset = + file_metadata_offset_map.at(directory->files.front().get()); + } + + const auto bucket = CalcHash(directory->name, metadata.parent_directory_offset) % + directory_hash_table.size(); + metadata.hash_bucket_next = directory_hash_table[bucket]; + directory_hash_table[bucket] = directory_metadata_offset_map.at(directory); + + // Write metadata and name + std::u16string u16name = Common::UTF8ToUTF16(directory->name); + metadata.name_length = u16name.size() * 2; + + std::memcpy(directory_metadata_table.data() + written, &metadata, sizeof(metadata)); + written += sizeof(metadata); + + written += WriteName(directory_metadata_table.data() + written, u16name); + } + + ASSERT_MSG(written == directory_metadata_table.size(), + "Calculated size for directory metadata table is wrong"); +} + +void LayeredFS::BuildFiles() { + file_metadata_table.resize(current_file_offset, 0xFF); + + std::size_t written = 0; + for (const auto& file : file_list) { + FileMetadata metadata; + std::memset(&metadata, 0xFF, sizeof(metadata)); + + metadata.parent_directory_offset = directory_metadata_offset_map.at(file->parent); + + bool flag = false; + for (const auto& sibling : file->parent->files) { + if (sibling->relocation.type == 3) { // removed file + continue; + } + if (flag) { + metadata.next_sibling_offset = file_metadata_offset_map.at(sibling.get()); + break; + } else if (sibling.get() == file) { + flag = true; + } + } + + metadata.file_data_offset = current_data_offset; + metadata.file_data_length = file->relocation.size; + current_data_offset += Common::AlignUp(metadata.file_data_length, 16); + if (metadata.file_data_length != 0) { + data_offset_map.emplace(metadata.file_data_offset, file); + } + + const auto bucket = + CalcHash(file->name, metadata.parent_directory_offset) % file_hash_table.size(); + metadata.hash_bucket_next = file_hash_table[bucket]; + file_hash_table[bucket] = file_metadata_offset_map.at(file); + + // Write metadata and name + std::u16string u16name = Common::UTF8ToUTF16(file->name); + metadata.name_length = u16name.size() * 2; + + std::memcpy(file_metadata_table.data() + written, &metadata, sizeof(metadata)); + written += sizeof(metadata); + + written += WriteName(file_metadata_table.data() + written, u16name); + } + + ASSERT_MSG(written == file_metadata_table.size(), + "Calculated size for file metadata table is wrong"); +} + +// Implementation from 3dbrew +std::size_t GetHashTableSize(std::size_t entry_count) { + if (entry_count < 3) { + return 3; + } else if (entry_count < 19) { + return entry_count | 1; + } else { + std::size_t count = entry_count; + while (count % 2 == 0 || count % 3 == 0 || count % 5 == 0 || count % 7 == 0 || + count % 11 == 0 || count % 13 == 0 || count % 17 == 0) { + count++; + } + return count; + } +} + +void LayeredFS::RebuildMetadata() { + PrepareBuildDirectory(root); + PrepareBuild(root); + + directory_hash_table.resize(GetHashTableSize(directory_list.size()), 0xFFFFFFFF); + file_hash_table.resize(GetHashTableSize(file_list.size()), 0xFFFFFFFF); + + BuildDirectories(); + BuildFiles(); + + // Create header + RomFSHeader header; + header.header_length = sizeof(header); + header.directory_hash_table = { + /*offset*/ sizeof(header), + /*length*/ static_cast(directory_hash_table.size() * sizeof(u32_le))}; + header.directory_metadata_table = { + /*offset*/ + header.directory_hash_table.offset + header.directory_hash_table.length, + /*length*/ static_cast(directory_metadata_table.size())}; + header.file_hash_table = { + /*offset*/ + header.directory_metadata_table.offset + header.directory_metadata_table.length, + /*length*/ static_cast(file_hash_table.size() * sizeof(u32_le))}; + header.file_metadata_table = {/*offset*/ header.file_hash_table.offset + + header.file_hash_table.length, + /*length*/ static_cast(file_metadata_table.size())}; + header.file_data_offset = + Common::AlignUp(header.file_metadata_table.offset + header.file_metadata_table.length, 16); + + // Write hash table and metadata table + metadata.resize(header.file_data_offset); + std::memcpy(metadata.data(), &header, header.header_length); + std::memcpy(metadata.data() + header.directory_hash_table.offset, directory_hash_table.data(), + header.directory_hash_table.length); + std::memcpy(metadata.data() + header.directory_metadata_table.offset, + directory_metadata_table.data(), header.directory_metadata_table.length); + std::memcpy(metadata.data() + header.file_hash_table.offset, file_hash_table.data(), + header.file_hash_table.length); + std::memcpy(metadata.data() + header.file_metadata_table.offset, file_metadata_table.data(), + header.file_metadata_table.length); +} + +std::size_t LayeredFS::GetSize() const { + return metadata.size() + current_data_offset; +} + +std::size_t LayeredFS::ReadFile(std::size_t offset, std::size_t length, u8* buffer) { + ASSERT_MSG(offset + length <= GetSize(), "Out of bound"); + + std::size_t read_size = 0; + if (offset < metadata.size()) { + // First read the metadata + const auto to_read = std::min(metadata.size() - offset, length); + std::memcpy(buffer, metadata.data() + offset, to_read); + read_size += to_read; + offset = 0; + } else { + offset -= metadata.size(); + } + + // Read files + auto current = (--data_offset_map.upper_bound(offset)); + while (read_size < length) { + const auto relative_offset = offset - current->first; + std::size_t to_read{}; + if (current->second->relocation.size > relative_offset) { + to_read = std::min(current->second->relocation.size - relative_offset, + length - read_size); + } + const auto alignment = + std::min(Common::AlignUp(current->second->relocation.size, 16) - + relative_offset, + length - read_size) - + to_read; + + // Read the file in different ways depending on relocation type + auto& relocation = current->second->relocation; + if (relocation.type == 0) { // none + romfs->ReadFile(relocation.original_offset + relative_offset, to_read, + buffer + read_size); + } else if (relocation.type == 1) { // replace + FileUtil::IOFile replace_file(relocation.replace_file_path, "rb"); + if (replace_file) { + replace_file.Seek(relative_offset, SEEK_SET); + replace_file.ReadBytes(buffer + read_size, to_read); + } else { + LOG_ERROR(Service_FS, "Could not open replacement file for {}", + current->second->path); + } + } else if (relocation.type == 2) { // patch + std::memcpy(buffer + read_size, relocation.patched_file.data() + relative_offset, + to_read); + } else { + UNREACHABLE(); + } + + std::memset(buffer + read_size + to_read, 0, alignment); + + read_size += to_read + alignment; + offset += to_read + alignment; + current++; + } + + return read_size; +} + +bool LayeredFS::ExtractDirectory(Directory& current, const std::string& target_path) { + if (!FileUtil::CreateFullPath(target_path + current.path)) { + LOG_ERROR(Service_FS, "Could not create path {}", target_path + current.path); + return false; + } + + constexpr std::size_t BufferSize = 0x10000; + std::array buffer; + for (const auto& file : current.files) { + // Extract file + const auto path = target_path + file->path; + LOG_INFO(Service_FS, "Extracting {} to {}", file->path, path); + + FileUtil::IOFile target_file(path, "wb"); + if (!target_file) { + LOG_ERROR(Service_FS, "Could not open file {}", path); + return false; + } + + std::size_t written = 0; + while (written < file->relocation.size) { + const auto to_read = + std::min(buffer.size(), file->relocation.size - written); + if (romfs->ReadFile(file->relocation.original_offset + written, to_read, + buffer.data()) != to_read) { + LOG_ERROR(Service_FS, "Could not read from RomFS"); + return false; + } + + if (target_file.WriteBytes(buffer.data(), to_read) != to_read) { + LOG_ERROR(Service_FS, "Could not write to file {}", path); + return false; + } + + written += to_read; + } + } + + for (const auto& directory : current.directories) { + if (!ExtractDirectory(*directory, target_path)) { + return false; + } + } + + return true; +} + +bool LayeredFS::DumpRomFS(const std::string& target_path) { + std::string path = target_path; + if (path.back() == '/' || path.back() == '\\') { + path.erase(path.size() - 1, 1); + } + + return ExtractDirectory(root, path); +} + +} // namespace FileSys diff --git a/src/core/file_sys/layered_fs.h b/src/core/file_sys/layered_fs.h new file mode 100644 index 0000000000..956eedcfa7 --- /dev/null +++ b/src/core/file_sys/layered_fs.h @@ -0,0 +1,123 @@ +// Copyright 2020 Citra Emulator Project +// Licensed under GPLv2 or any later version +// Refer to the license.txt file included. + +#pragma once + +#include +#include +#include +#include +#include +#include "common/common_types.h" +#include "common/swap.h" +#include "core/file_sys/romfs_reader.h" + +namespace FileSys { + +struct RomFSHeader { + struct Descriptor { + u32_le offset; + u32_le length; + }; + u32_le header_length; + Descriptor directory_hash_table; + Descriptor directory_metadata_table; + Descriptor file_hash_table; + Descriptor file_metadata_table; + u32_le file_data_offset; +}; +static_assert(sizeof(RomFSHeader) == 0x28, "Size of RomFSHeader is not correct"); + +/** + * LayeredFS implementation. This basically adds a layer to another RomFSReader. + * + * patch_path: Path for RomFS replacements. Files present in this path replace or create + * corresponding files in RomFS. + * patch_ext_path: Path for RomFS extensions. Files present in this path: + * - When with an extension of ".stub", remove the corresponding file in the RomFS. + * - When with an extension of ".ips" or ".bps", patch the file in the RomFS. + */ +class LayeredFS : public RomFSReader { +public: + explicit LayeredFS(std::shared_ptr romfs, std::string patch_path, + std::string patch_ext_path, bool load_relocations = true); + ~LayeredFS() override; + + std::size_t GetSize() const override; + std::size_t ReadFile(std::size_t offset, std::size_t length, u8* buffer) override; + + bool DumpRomFS(const std::string& target_path); + +private: + struct File; + struct Directory { + std::string name; + std::string path; // with trailing '/' + std::vector> files; + std::vector> directories; + Directory* parent; + }; + + std::string ReadName(u32 offset, u32 name_length); + + // Loads the current directory, then its siblings, and then its children. + void LoadDirectory(Directory& current, u32 offset); + + // Load the file at offset, and then its siblings. + void LoadFile(Directory& parent, u32 offset); + + // Load replace/create relocations + void LoadRelocations(); + + // Load patch/remove relocations + void LoadExtRelocations(); + + // Calculate the offset of a single directory add it to the map and list of directories + void PrepareBuildDirectory(Directory& current); + + // Calculate the offset of a single file add it to the map and list of files + void PrepareBuildFile(File& current); + + // Recursively generate a sequence of files and directories and their offsets for all + // children of current. (The current directory itself is not handled.) + void PrepareBuild(Directory& current); + + void BuildDirectories(); + void BuildFiles(); + + // Recursively extract a directory and all its contents to target_path + // target_path should be without trailing '/'. + bool ExtractDirectory(Directory& current, const std::string& target_path); + + void RebuildMetadata(); + + std::shared_ptr romfs; + std::string patch_path; + std::string patch_ext_path; + + RomFSHeader header; + Directory root; + std::unordered_map file_path_map; + std::unordered_map directory_path_map; + std::map data_offset_map; // assigned data offset -> file + std::vector metadata; // Includes header, hash table and metadata + + // Used for rebuilding header + std::vector directory_hash_table; + std::vector file_hash_table; + + std::unordered_map + directory_metadata_offset_map; // directory -> metadata offset + std::vector directory_list; // sequence of directories to be written to metadata + u64 current_directory_offset{}; // current directory metadata offset + std::vector directory_metadata_table; // rebuilt directory metadata table + + std::unordered_map file_metadata_offset_map; // file -> metadata offset + std::vector file_list; // sequence of files to be written to metadata + u64 current_file_offset{}; // current file metadata offset + std::vector file_metadata_table; // rebuilt file metadata table + u64 current_data_offset{}; // current assigned data offset +}; + +} // namespace FileSys diff --git a/src/core/file_sys/ncch_container.cpp b/src/core/file_sys/ncch_container.cpp index f0687fa9e9..81525786fb 100644 --- a/src/core/file_sys/ncch_container.cpp +++ b/src/core/file_sys/ncch_container.cpp @@ -11,6 +11,7 @@ #include "common/common_types.h" #include "common/logging/log.h" #include "core/core.h" +#include "core/file_sys/layered_fs.h" #include "core/file_sys/ncch_container.h" #include "core/file_sys/patch.h" #include "core/file_sys/seed_db.h" @@ -25,6 +26,14 @@ namespace FileSys { static const int kMaxSections = 8; ///< Maximum number of sections (files) in an ExeFs static const int kBlockSize = 0x200; ///< Size of ExeFS blocks (in bytes) +u64 GetModId(u64 program_id) { + constexpr u64 UPDATE_MASK = 0x0000000e'00000000; + if ((program_id & 0x000000ff'00000000) == UPDATE_MASK) { // Apply the mods to updates + return program_id & ~UPDATE_MASK; + } + return program_id; +} + /** * Get the decompressed size of an LZSS compressed ExeFS file * @param buffer Buffer of compressed file @@ -303,8 +312,22 @@ Loader::ResultStatus NCCHContainer::Load() { } } - FileUtil::IOFile exheader_override_file{filepath + ".exheader", "rb"}; - const bool has_exheader_override = read_exheader(exheader_override_file); + const auto mods_path = + fmt::format("{}mods/{:016X}/", FileUtil::GetUserPath(FileUtil::UserPath::LoadDir), + GetModId(ncch_header.program_id)); + std::array exheader_override_paths{{ + mods_path + "exheader.bin", + filepath + ".exheader", + }}; + + bool has_exheader_override = false; + for (const auto& path : exheader_override_paths) { + FileUtil::IOFile exheader_override_file{path, "rb"}; + if (read_exheader(exheader_override_file)) { + has_exheader_override = true; + break; + } + } if (has_exheader_override) { if (exheader_header.system_info.jump_id != exheader_header.arm11_system_local_caps.program_id) { @@ -512,7 +535,13 @@ Loader::ResultStatus NCCHContainer::ApplyCodePatch(std::vector& code) const std::string path; bool (*patch_fn)(const std::vector& patch, std::vector& code); }; - const std::array patch_paths{{ + + const auto mods_path = + fmt::format("{}mods/{:016X}/", FileUtil::GetUserPath(FileUtil::UserPath::LoadDir), + GetModId(ncch_header.program_id)); + const std::array patch_paths{{ + {mods_path + "exefs/code.ips", Patch::ApplyIpsPatch}, + {mods_path + "exefs/code.bps", Patch::ApplyBpsPatch}, {filepath + ".exefsdir/code.ips", Patch::ApplyIpsPatch}, {filepath + ".exefsdir/code.bps", Patch::ApplyBpsPatch}, }}; @@ -551,23 +580,33 @@ Loader::ResultStatus NCCHContainer::LoadOverrideExeFSSection(const char* name, else return Loader::ResultStatus::Error; - std::string section_override = filepath + ".exefsdir/" + override_name; - FileUtil::IOFile section_file(section_override, "rb"); + const auto mods_path = + fmt::format("{}mods/{:016X}/", FileUtil::GetUserPath(FileUtil::UserPath::LoadDir), + GetModId(ncch_header.program_id)); + std::array override_paths{{ + mods_path + "exefs/" + override_name, + filepath + ".exefsdir/" + override_name, + }}; - if (section_file.IsOpen()) { - auto section_size = section_file.GetSize(); - buffer.resize(section_size); + for (const auto& path : override_paths) { + FileUtil::IOFile section_file(path, "rb"); - section_file.Seek(0, SEEK_SET); - if (section_file.ReadBytes(&buffer[0], section_size) == section_size) { - LOG_WARNING(Service_FS, "File {} overriding built-in ExeFS file", section_override); - return Loader::ResultStatus::Success; + if (section_file.IsOpen()) { + auto section_size = section_file.GetSize(); + buffer.resize(section_size); + + section_file.Seek(0, SEEK_SET); + if (section_file.ReadBytes(&buffer[0], section_size) == section_size) { + LOG_WARNING(Service_FS, "File {} overriding built-in ExeFS file", path); + return Loader::ResultStatus::Success; + } } } return Loader::ResultStatus::ErrorNotUsed; } -Loader::ResultStatus NCCHContainer::ReadRomFS(std::shared_ptr& romfs_file) { +Loader::ResultStatus NCCHContainer::ReadRomFS(std::shared_ptr& romfs_file, + bool use_layered_fs) { Loader::ResultStatus result = Load(); if (result != Loader::ResultStatus::Success) return result; @@ -597,14 +636,43 @@ Loader::ResultStatus NCCHContainer::ReadRomFS(std::shared_ptr& romf if (!romfs_file_inner.IsOpen()) return Loader::ResultStatus::Error; + std::shared_ptr direct_romfs; if (is_encrypted) { - romfs_file = std::make_shared(std::move(romfs_file_inner), romfs_offset, - romfs_size, secondary_key, romfs_ctr, 0x1000); + direct_romfs = + std::make_shared(std::move(romfs_file_inner), romfs_offset, + romfs_size, secondary_key, romfs_ctr, 0x1000); } else { - romfs_file = - std::make_shared(std::move(romfs_file_inner), romfs_offset, romfs_size); + direct_romfs = std::make_shared(std::move(romfs_file_inner), + romfs_offset, romfs_size); } + const auto path = + fmt::format("{}mods/{:016X}/", FileUtil::GetUserPath(FileUtil::UserPath::LoadDir), + GetModId(ncch_header.program_id)); + if (use_layered_fs && + (FileUtil::Exists(path + "romfs/") || FileUtil::Exists(path + "romfs_ext/"))) { + + romfs_file = std::make_shared(std::move(direct_romfs), path + "romfs/", + path + "romfs_ext/"); + } else { + romfs_file = std::move(direct_romfs); + } + + return Loader::ResultStatus::Success; +} + +Loader::ResultStatus NCCHContainer::DumpRomFS(const std::string& target_path) { + std::shared_ptr direct_romfs; + Loader::ResultStatus result = ReadRomFS(direct_romfs, false); + if (result != Loader::ResultStatus::Success) + return result; + + std::shared_ptr layered_fs = + std::make_shared(std::move(direct_romfs), "", "", false); + + if (!layered_fs->DumpRomFS(target_path)) { + return Loader::ResultStatus::Error; + } return Loader::ResultStatus::Success; } @@ -614,9 +682,10 @@ Loader::ResultStatus NCCHContainer::ReadOverrideRomFS(std::shared_ptr(std::move(romfs_file_inner), 0, - romfs_file_inner.GetSize()); + LOG_WARNING(Service_FS, "File {} overriding built-in RomFS; LayeredFS not enabled", + split_filepath); + romfs_file = std::make_shared(std::move(romfs_file_inner), 0, + romfs_file_inner.GetSize()); return Loader::ResultStatus::Success; } } diff --git a/src/core/file_sys/ncch_container.h b/src/core/file_sys/ncch_container.h index f06ee8ef61..8deda1fff0 100644 --- a/src/core/file_sys/ncch_container.h +++ b/src/core/file_sys/ncch_container.h @@ -247,7 +247,15 @@ public: * @param size The size of the romfs * @return ResultStatus result of function */ - Loader::ResultStatus ReadRomFS(std::shared_ptr& romfs_file); + Loader::ResultStatus ReadRomFS(std::shared_ptr& romfs_file, + bool use_layered_fs = true); + + /** + * Dump the RomFS of the NCCH container to the user folder. + * @param target_path target path to dump to + * @return ResultStatus result of function. + */ + Loader::ResultStatus DumpRomFS(const std::string& target_path); /** * Get the override RomFS of the NCCH container diff --git a/src/core/file_sys/romfs_reader.cpp b/src/core/file_sys/romfs_reader.cpp index 8e624cbfc2..64374684a1 100644 --- a/src/core/file_sys/romfs_reader.cpp +++ b/src/core/file_sys/romfs_reader.cpp @@ -5,7 +5,7 @@ namespace FileSys { -std::size_t RomFSReader::ReadFile(std::size_t offset, std::size_t length, u8* buffer) { +std::size_t DirectRomFSReader::ReadFile(std::size_t offset, std::size_t length, u8* buffer) { if (length == 0) return 0; // Crypto++ does not like zero size buffer file.Seek(file_offset + offset, SEEK_SET); diff --git a/src/core/file_sys/romfs_reader.h b/src/core/file_sys/romfs_reader.h index 72a02cde3d..df0318c99a 100644 --- a/src/core/file_sys/romfs_reader.h +++ b/src/core/file_sys/romfs_reader.h @@ -6,23 +6,39 @@ namespace FileSys { +/** + * Interface for reading RomFS data. + */ class RomFSReader { public: - RomFSReader(FileUtil::IOFile&& file, std::size_t file_offset, std::size_t data_size) + virtual ~RomFSReader() = default; + + virtual std::size_t GetSize() const = 0; + virtual std::size_t ReadFile(std::size_t offset, std::size_t length, u8* buffer) = 0; +}; + +/** + * A RomFS reader that directly reads the RomFS file. + */ +class DirectRomFSReader : public RomFSReader { +public: + DirectRomFSReader(FileUtil::IOFile&& file, std::size_t file_offset, std::size_t data_size) : is_encrypted(false), file(std::move(file)), file_offset(file_offset), data_size(data_size) {} - RomFSReader(FileUtil::IOFile&& file, std::size_t file_offset, std::size_t data_size, - const std::array& key, const std::array& ctr, - std::size_t crypto_offset) + DirectRomFSReader(FileUtil::IOFile&& file, std::size_t file_offset, std::size_t data_size, + const std::array& key, const std::array& ctr, + std::size_t crypto_offset) : is_encrypted(true), file(std::move(file)), key(key), ctr(ctr), file_offset(file_offset), crypto_offset(crypto_offset), data_size(data_size) {} - std::size_t GetSize() const { + ~DirectRomFSReader() override = default; + + std::size_t GetSize() const override { return data_size; } - std::size_t ReadFile(std::size_t offset, std::size_t length, u8* buffer); + std::size_t ReadFile(std::size_t offset, std::size_t length, u8* buffer) override; private: bool is_encrypted; diff --git a/src/core/loader/3dsx.cpp b/src/core/loader/3dsx.cpp index 7629ad3768..84321011bf 100644 --- a/src/core/loader/3dsx.cpp +++ b/src/core/loader/3dsx.cpp @@ -309,8 +309,8 @@ ResultStatus AppLoader_THREEDSX::ReadRomFS(std::shared_ptr if (!romfs_file_inner.IsOpen()) return ResultStatus::Error; - romfs_file = std::make_shared(std::move(romfs_file_inner), - romfs_offset, romfs_size); + romfs_file = std::make_shared(std::move(romfs_file_inner), + romfs_offset, romfs_size); return ResultStatus::Success; } diff --git a/src/core/loader/loader.h b/src/core/loader/loader.h index 20e84c6a9d..0414f181c4 100644 --- a/src/core/loader/loader.h +++ b/src/core/loader/loader.h @@ -186,6 +186,15 @@ public: return ResultStatus::ErrorNotImplemented; } + /** + * Dump the RomFS of the applciation + * @param target_path The target path to dump to + * @return ResultStatus result of function + */ + virtual ResultStatus DumpRomFS(const std::string& target_path) { + return ResultStatus::ErrorNotImplemented; + } + /** * Get the update RomFS of the application * Since the RomFS can be huge, we return a file reference instead of copying to a buffer @@ -196,6 +205,15 @@ public: return ResultStatus::ErrorNotImplemented; } + /** + * Dump the update RomFS of the applciation + * @param target_path The target path to dump to + * @return ResultStatus result of function + */ + virtual ResultStatus DumpUpdateRomFS(const std::string& target_path) { + return ResultStatus::ErrorNotImplemented; + } + /** * Get the title of the application * @param title Reference to store the application title into diff --git a/src/core/loader/ncch.cpp b/src/core/loader/ncch.cpp index 2e688e0119..1a966da5e0 100644 --- a/src/core/loader/ncch.cpp +++ b/src/core/loader/ncch.cpp @@ -254,6 +254,18 @@ ResultStatus AppLoader_NCCH::ReadUpdateRomFS(std::shared_ptr data; Loader::SMDH smdh; diff --git a/src/core/loader/ncch.h b/src/core/loader/ncch.h index 7c86f85d8a..041cfddbdc 100644 --- a/src/core/loader/ncch.h +++ b/src/core/loader/ncch.h @@ -59,6 +59,10 @@ public: ResultStatus ReadUpdateRomFS(std::shared_ptr& romfs_file) override; + ResultStatus DumpRomFS(const std::string& target_path) override; + + ResultStatus DumpUpdateRomFS(const std::string& target_path) override; + ResultStatus ReadTitle(std::string& title) override; private: