From c385b7767d32eccabbfeaa12764310cfc3d113b9 Mon Sep 17 00:00:00 2001
From: condut <>
Date: Fri, 10 Jul 2015 00:55:23 +0300
Subject: [PATCH] FS: Stream RomFS from file instead of loading all of it to
memory
---
src/core/file_sys/archive_romfs.cpp | 7 +++----
src/core/file_sys/archive_romfs.h | 4 +++-
src/core/file_sys/archive_savedatacheck.cpp | 11 ++++-------
src/core/file_sys/ivfc_archive.cpp | 13 ++++++-------
src/core/file_sys/ivfc_archive.h | 15 +++++++++++----
src/core/loader/loader.cpp | 2 +-
src/core/loader/loader.h | 7 +++++--
src/core/loader/ncch.cpp | 15 ++++++++++-----
src/core/loader/ncch.h | 7 +++++--
9 files changed, 48 insertions(+), 33 deletions(-)
diff --git a/src/core/file_sys/archive_romfs.cpp b/src/core/file_sys/archive_romfs.cpp
index c1e45dfeb2..b792b1c8cb 100644
--- a/src/core/file_sys/archive_romfs.cpp
+++ b/src/core/file_sys/archive_romfs.cpp
@@ -17,16 +17,15 @@
namespace FileSys {
-ArchiveFactory_RomFS::ArchiveFactory_RomFS(const Loader::AppLoader& app_loader)
- : romfs_data(std::make_shared<std::vector<u8>>()) {
+ArchiveFactory_RomFS::ArchiveFactory_RomFS(const Loader::AppLoader& app_loader) {
// Load the RomFS from the app
- if (Loader::ResultStatus::Success != app_loader.ReadRomFS(*romfs_data)) {
+ if (Loader::ResultStatus::Success != app_loader.ReadRomFS(romfs_file, data_offset, data_size)) {
LOG_ERROR(Service_FS, "Unable to read RomFS!");
}
}
ResultVal<std::unique_ptr<ArchiveBackend>> ArchiveFactory_RomFS::Open(const Path& path) {
- auto archive = Common::make_unique<IVFCArchive>(romfs_data);
+ auto archive = Common::make_unique<IVFCArchive>(romfs_file, data_offset, data_size);
return MakeResult<std::unique_ptr<ArchiveBackend>>(std::move(archive));
}
diff --git a/src/core/file_sys/archive_romfs.h b/src/core/file_sys/archive_romfs.h
index c69ff91c31..0ef67c5575 100644
--- a/src/core/file_sys/archive_romfs.h
+++ b/src/core/file_sys/archive_romfs.h
@@ -29,7 +29,9 @@ public:
ResultCode Format(const Path& path) override;
private:
- std::shared_ptr<std::vector<u8>> romfs_data;
+ std::shared_ptr<FileUtil::IOFile> romfs_file;
+ u64 data_offset;
+ u64 data_size;
};
} // namespace FileSys
diff --git a/src/core/file_sys/archive_savedatacheck.cpp b/src/core/file_sys/archive_savedatacheck.cpp
index dec838caee..ea1dfe2c71 100644
--- a/src/core/file_sys/archive_savedatacheck.cpp
+++ b/src/core/file_sys/archive_savedatacheck.cpp
@@ -37,17 +37,14 @@ ResultVal<std::unique_ptr<ArchiveBackend>> ArchiveFactory_SaveDataCheck::Open(co
auto vec = path.AsBinary();
const u32* data = reinterpret_cast<u32*>(vec.data());
std::string file_path = GetSaveDataCheckPath(mount_point, data[1], data[0]);
- FileUtil::IOFile file(file_path, "rb");
+ auto file = std::make_shared<FileUtil::IOFile>(file_path, "rb");
- if (!file.IsOpen()) {
+ if (!file->IsOpen()) {
return ResultCode(-1); // TODO(Subv): Find the right error code
}
- auto size = file.GetSize();
- auto raw_data = std::make_shared<std::vector<u8>>(size);
- file.ReadBytes(raw_data->data(), size);
- file.Close();
+ auto size = file->GetSize();
- auto archive = Common::make_unique<IVFCArchive>(std::move(raw_data));
+ auto archive = Common::make_unique<IVFCArchive>(file, 0, size);
return MakeResult<std::unique_ptr<ArchiveBackend>>(std::move(archive));
}
diff --git a/src/core/file_sys/ivfc_archive.cpp b/src/core/file_sys/ivfc_archive.cpp
index c88b39bcd6..2b88b1d5d2 100644
--- a/src/core/file_sys/ivfc_archive.cpp
+++ b/src/core/file_sys/ivfc_archive.cpp
@@ -16,15 +16,12 @@
namespace FileSys {
-IVFCArchive::IVFCArchive(std::shared_ptr<const std::vector<u8>> data) : data(data) {
-}
-
std::string IVFCArchive::GetName() const {
return "IVFC";
}
std::unique_ptr<FileBackend> IVFCArchive::OpenFile(const Path& path, const Mode mode) const {
- return Common::make_unique<IVFCFile>(data);
+ return Common::make_unique<IVFCFile>(romfs_file, data_offset, data_size);
}
bool IVFCArchive::DeleteFile(const Path& path) const {
@@ -66,8 +63,10 @@ std::unique_ptr<DirectoryBackend> IVFCArchive::OpenDirectory(const Path& path) c
size_t IVFCFile::Read(const u64 offset, const u32 length, u8* buffer) const {
LOG_TRACE(Service_FS, "called offset=%llu, length=%d", offset, length);
- memcpy(buffer, data->data() + offset, length);
- return length;
+ romfs_file->Seek(data_offset + offset, SEEK_SET);
+ u32 read_length = (u32)std::min((u64)length, data_size - offset);
+
+ return romfs_file->ReadBytes(buffer, read_length);
}
size_t IVFCFile::Write(const u64 offset, const u32 length, const u32 flush, const u8* buffer) const {
@@ -76,7 +75,7 @@ size_t IVFCFile::Write(const u64 offset, const u32 length, const u32 flush, cons
}
size_t IVFCFile::GetSize() const {
- return sizeof(u8) * data->size();
+ return data_size; // TODO: return value will overflow on 32-bit machines
}
bool IVFCFile::SetSize(const u64 size) const {
diff --git a/src/core/file_sys/ivfc_archive.h b/src/core/file_sys/ivfc_archive.h
index 1850b3b173..c25666223e 100644
--- a/src/core/file_sys/ivfc_archive.h
+++ b/src/core/file_sys/ivfc_archive.h
@@ -10,6 +10,7 @@
#include <vector>
#include "common/common_types.h"
+#include "common/file_util.h"
#include "core/file_sys/archive_backend.h"
#include "core/file_sys/directory_backend.h"
@@ -28,7 +29,8 @@ namespace FileSys {
*/
class IVFCArchive : public ArchiveBackend {
public:
- IVFCArchive(std::shared_ptr<const std::vector<u8>> data);
+ IVFCArchive(std::shared_ptr<FileUtil::IOFile> file, u64 offset, u64 size)
+ : romfs_file(file), data_offset(offset), data_size(size) {}
std::string GetName() const override;
@@ -42,12 +44,15 @@ public:
std::unique_ptr<DirectoryBackend> OpenDirectory(const Path& path) const override;
protected:
- std::shared_ptr<const std::vector<u8>> data;
+ std::shared_ptr<FileUtil::IOFile> romfs_file;
+ u64 data_offset;
+ u64 data_size;
};
class IVFCFile : public FileBackend {
public:
- IVFCFile(std::shared_ptr<const std::vector<u8>> data) : data(data) {}
+ IVFCFile(std::shared_ptr<FileUtil::IOFile> file, u64 offset, u64 size)
+ : romfs_file(file), data_offset(offset), data_size(size) {}
bool Open() override { return true; }
size_t Read(const u64 offset, const u32 length, u8* buffer) const override;
@@ -58,7 +63,9 @@ public:
void Flush() const override { }
private:
- std::shared_ptr<const std::vector<u8>> data;
+ std::shared_ptr<FileUtil::IOFile> romfs_file;
+ u64 data_offset;
+ u64 data_size;
};
class IVFCDirectory : public DirectoryBackend {
diff --git a/src/core/loader/loader.cpp b/src/core/loader/loader.cpp
index 2e450fce4d..b6549daf24 100644
--- a/src/core/loader/loader.cpp
+++ b/src/core/loader/loader.cpp
@@ -124,7 +124,7 @@ ResultStatus LoadFile(const std::string& filename) {
case FileType::CXI:
case FileType::CCI:
{
- AppLoader_NCCH app_loader(std::move(file));
+ AppLoader_NCCH app_loader(std::move(file), filename);
// Load application and RomFS
if (ResultStatus::Success == app_loader.Load()) {
diff --git a/src/core/loader/loader.h b/src/core/loader/loader.h
index 52bbf35b89..ff298222ba 100644
--- a/src/core/loader/loader.h
+++ b/src/core/loader/loader.h
@@ -99,10 +99,13 @@ public:
/**
* Get the RomFS of the application
- * @param buffer Reference to buffer to store data
+ * Since the RomFS can be huge, we return a file reference instead of copying to a buffer
+ * @param romfs_file The file containing the RomFS
+ * @param offset The offset the romfs begins on
+ * @param size The size of the romfs
* @return ResultStatus result of function
*/
- virtual ResultStatus ReadRomFS(std::vector<u8>& buffer) const {
+ virtual ResultStatus ReadRomFS(std::shared_ptr<FileUtil::IOFile>& romfs_file, u64& offset, u64& size) const {
return ResultStatus::ErrorNotImplemented;
}
diff --git a/src/core/loader/ncch.cpp b/src/core/loader/ncch.cpp
index 87603d198a..2bf1a6a261 100644
--- a/src/core/loader/ncch.cpp
+++ b/src/core/loader/ncch.cpp
@@ -299,7 +299,7 @@ ResultStatus AppLoader_NCCH::ReadLogo(std::vector<u8>& buffer) const {
return LoadSectionExeFS("logo", buffer);
}
-ResultStatus AppLoader_NCCH::ReadRomFS(std::vector<u8>& buffer) const {
+ResultStatus AppLoader_NCCH::ReadRomFS(std::shared_ptr<FileUtil::IOFile>& romfs_file, u64& offset, u64& size) const {
if (!file->IsOpen())
return ResultStatus::Error;
@@ -311,12 +311,17 @@ ResultStatus AppLoader_NCCH::ReadRomFS(std::vector<u8>& buffer) const {
LOG_DEBUG(Loader, "RomFS offset: 0x%08X", romfs_offset);
LOG_DEBUG(Loader, "RomFS size: 0x%08X", romfs_size);
- buffer.resize(romfs_size);
-
- file->Seek(romfs_offset, SEEK_SET);
- if (file->ReadBytes(&buffer[0], romfs_size) != romfs_size)
+ if (file->GetSize () < romfs_offset + romfs_size)
return ResultStatus::Error;
+ // We reopen the file, to allow its position to be independent from file's
+ romfs_file = std::make_shared<FileUtil::IOFile>(filepath, "rb");
+ if (!romfs_file->IsOpen())
+ return ResultStatus::Error;
+
+ offset = romfs_offset;
+ size = romfs_size;
+
return ResultStatus::Success;
}
LOG_DEBUG(Loader, "NCCH has no RomFS");
diff --git a/src/core/loader/ncch.h b/src/core/loader/ncch.h
index 29e39d2c0c..d180e77ed3 100644
--- a/src/core/loader/ncch.h
+++ b/src/core/loader/ncch.h
@@ -163,7 +163,8 @@ namespace Loader {
/// Loads an NCCH file (e.g. from a CCI, or the first NCCH in a CXI)
class AppLoader_NCCH final : public AppLoader {
public:
- AppLoader_NCCH(std::unique_ptr<FileUtil::IOFile>&& file) : AppLoader(std::move(file)) { }
+ AppLoader_NCCH(std::unique_ptr<FileUtil::IOFile>&& file, const std::string& filepath)
+ : AppLoader(std::move(file)), filepath(filepath) { }
/**
* Returns the type of the file
@@ -211,7 +212,7 @@ public:
* @param buffer Reference to buffer to store data
* @return ResultStatus result of function
*/
- ResultStatus ReadRomFS(std::vector<u8>& buffer) const override;
+ ResultStatus ReadRomFS(std::shared_ptr<FileUtil::IOFile>& romfs_file, u64& offset, u64& size) const override;
private:
@@ -244,6 +245,8 @@ private:
NCCH_Header ncch_header;
ExeFs_Header exefs_header;
ExHeader_Header exheader_header;
+
+ std::string filepath;
};
} // namespace Loader