ggml-org/llama.cpp · #26014
Windows unbuffered model load
src/llama-mmap.cpp265 + / 35 −
@@ -30,6 +30,7 @@ #ifndef NOMINMAX #define NOMINMAX #endif+ #include <malloc.h> #include <windows.h> #ifndef PATH_MAX #define PATH_MAX MAX_PATH@@ -68,7 +69,10 @@ static std::string llama_format_win_err(DWORD err) { struct llama_file::impl { #if defined(_WIN32)- HANDLE fp_win32;+ // Keep the buffered handle as the logical cursor; the direct handle only issues aligned I/O.+ HANDLE fp_win32 = INVALID_HANDLE_VALUE;+ HANDLE fp_win32_direct = INVALID_HANDLE_VALUE;+ std::string GetErrorMessageWin32(DWORD error_code) const { std::string ret; LPSTR lpMsgBuf = NULL;@@ -81,21 +85,101 @@ struct llama_file::impl { LocalFree(lpMsgBuf); } + while (!ret.empty() && (ret.back() == '\r' || ret.back() == '\n')) {+ ret.pop_back();+ }+ return ret; } - impl(const char * fname, const char * mode, [[maybe_unused]] const bool use_direct_io = false) {- fp = ggml_fopen(fname, mode);+ impl(const char * fname, const char * mode, const bool use_direct_io = false) : fname(fname) {+ init_buffered_io(mode);+ if (use_direct_io && std::strcmp(mode, "rb") == 0) {+ DWORD error = ERROR_SUCCESS;+ if (!init_direct_io(error)) {+ LLAMA_LOG_WARN("Failed to open file '%s' with unbuffered I/O: %s. Falling back to buffered I/O\n",+ fname, GetErrorMessageWin32(error).c_str());+ }+ }+ }++ bool init_direct_io(DWORD& error) {+ error = ERROR_SUCCESS;+ alignment = 1;++ fp_win32_direct = ReOpenFile(+ fp_win32,+ GENERIC_READ,+ FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,+ FILE_FLAG_NO_BUFFERING);++ if (fp_win32_direct == INVALID_HANDLE_VALUE) {+ error = GetLastError();+ return false;+ }++ const bool ok = [&]() -> bool {+#if defined(NTDDI_VERSION) && defined(NTDDI_WIN8) && (NTDDI_VERSION >= NTDDI_WIN8)+ FILE_STORAGE_INFO storage_info{};+ if (!GetFileInformationByHandleEx(+ fp_win32_direct,+ FileStorageInfo,+ &storage_info,+ static_cast<DWORD>(sizeof(storage_info)))) {+ error = GetLastError();+ return false;+ }++ alignment = std::max<size_t>({+ static_cast<size_t>(storage_info.LogicalBytesPerSector),+ static_cast<size_t>(storage_info.PhysicalBytesPerSectorForAtomicity),+ static_cast<size_t>(storage_info.PhysicalBytesPerSectorForPerformance),+ static_cast<size_t>(storage_info.FileSystemEffectivePhysicalBytesPerSectorForAtomicity)+ });+#else+ error = ERROR_NOT_SUPPORTED;+ return false;+#endif+ // Required for bit-mask rounding and aligned allocation schemes.+ if (alignment <= 1 || (alignment & (alignment - 1)) != 0) {+ error = ERROR_NOT_SUPPORTED;+ return false;+ }++ // ReOpenFile creates a distinct handle; explicitly start direct I/O at offset 0.+ LARGE_INTEGER zero{};+ if (!SetFilePointerEx(fp_win32_direct, zero, nullptr, FILE_BEGIN)) {+ error = GetLastError();+ return false;+ }++ return true;+ }();++ if (!ok) {+ CloseHandle(fp_win32_direct);+ fp_win32_direct = INVALID_HANDLE_VALUE;+ alignment = 1;+ return false;+ }++ LLAMA_LOG_INFO("%s: file '%s' opened with unbuffered I/O (alignment: %zu bytes)\n",+ __func__, fname.c_str(), alignment);+ return true;+ }++ void init_buffered_io(const char * mode) {+ fp = ggml_fopen(fname.c_str(), mode); if (fp == NULL) {- throw std::runtime_error(format("failed to open %s: %s", fname, strerror(errno)));+ throw std::runtime_error(format("failed to open %s: %s", fname.c_str(), strerror(errno))); } fp_win32 = (HANDLE) _get_osfhandle(_fileno(fp)); seek(0, SEEK_END); size = tell(); seek(0, SEEK_SET); } - impl(FILE * file) : owns_fp(false) {+ impl(FILE * file) : fname("(file*)"), owns_fp(false) { fp = file; fp_win32 = (HANDLE) _get_osfhandle(_fileno(fp)); seek(0, SEEK_END);@@ -108,7 +192,7 @@ struct llama_file::impl { li.QuadPart = 0; BOOL ret = SetFilePointerEx(fp_win32, li, &li, FILE_CURRENT); if (!ret) {- throw std::runtime_error(format("read error: %s", GetErrorMessageWin32(GetLastError()).c_str()));+ throw std::runtime_error(format("tell error: %s", GetErrorMessageWin32(GetLastError()).c_str())); } return li.QuadPart;@@ -123,24 +207,159 @@ struct llama_file::impl { li.QuadPart = offset; BOOL ret = SetFilePointerEx(fp_win32, li, NULL, whence); if (!ret) {- throw std::runtime_error(format("read error: %s", GetErrorMessageWin32(GetLastError()).c_str()));+ throw std::runtime_error(format("seek error: %s", GetErrorMessageWin32(GetLastError()).c_str())); } } - void read_raw(void * ptr, size_t len) {+ void read_raw_unsafe(void * ptr, size_t len) {+ if (len == 0) {+ return;+ }++ const size_t read_offset = tell();+ if (read_offset > size) {+ throw std::runtime_error("unexpectedly reached end of file");+ }++ if (!has_direct_io()) {+ const size_t read_size = std::min(len, size - read_offset);+ size_t bytes_read = 0;+ while (bytes_read < read_size) {+ const DWORD to_read = (DWORD) std::min<size_t>(read_size - bytes_read, MAXDWORD);+ DWORD chunk_read = 0;+ BOOL result = ReadFile(fp_win32, reinterpret_cast<char *>(ptr) + bytes_read,+ to_read, &chunk_read, NULL);+ if (!result) {+ throw std::runtime_error(format("read error: %s", GetErrorMessageWin32(GetLastError()).c_str()));+ }+ if (chunk_read < to_read) {+ throw std::runtime_error("unexpectedly reached end of file");+ }++ bytes_read += chunk_read;+ }+ return;+ }++ DWORD error = ERROR_SUCCESS;+ if ((read_offset & (alignment - 1)) != 0 ||+ (len & (alignment - 1)) != 0 ||+ (reinterpret_cast<uintptr_t>(ptr) & (alignment - 1)) != 0) {+ error = ERROR_INVALID_PARAMETER;+ } else {+ LARGE_INTEGER li;+ li.QuadPart = read_offset;+ if (!SetFilePointerEx(fp_win32_direct, li, NULL, FILE_BEGIN)) {+ error = GetLastError();+ }+ }+ size_t bytes_read = 0;- while (bytes_read < len) {- size_t chunk_size = std::min<size_t>(len - bytes_read, 64*1024*1024);+ const size_t max_read_size = (size_t) MAXDWORD & ~(alignment - 1);+ while (error == ERROR_SUCCESS && bytes_read < len) {+ const DWORD to_read = (DWORD) std::min(len - bytes_read, max_read_size); DWORD chunk_read = 0;- BOOL result = ReadFile(fp_win32, reinterpret_cast<char*>(ptr) + bytes_read, chunk_size, &chunk_read, NULL);+ BOOL result = ReadFile(fp_win32_direct, reinterpret_cast<char *>(ptr) + bytes_read,+ to_read, &chunk_read, NULL); if (!result) {- throw std::runtime_error(format("read error: %s", GetErrorMessageWin32(GetLastError()).c_str()));- }- if (chunk_read < chunk_size || chunk_read == 0) {- throw std::runtime_error("unexpectedly reached end of file");+ error = GetLastError();+ if (error == ERROR_HANDLE_EOF && bytes_read == size - read_offset) {+ error = ERROR_SUCCESS;+ break;+ }+ break; } bytes_read += chunk_read;+ if (chunk_read < to_read) {+ if (bytes_read == size - read_offset) {+ break;+ }+ error = ERROR_HANDLE_EOF;+ }+ }++ if (error == ERROR_INVALID_PARAMETER || error == ERROR_INVALID_USER_BUFFER || error == ERROR_NOACCESS) {+ LLAMA_LOG_WARN("%s: file '%s' falling back to buffered I/O due to %s\n",+ __func__, fname.c_str(), GetErrorMessageWin32(error).c_str());+ CloseHandle(fp_win32_direct);+ fp_win32_direct = INVALID_HANDLE_VALUE;+ alignment = 1;+ seek(read_offset, SEEK_SET);+ read_raw_unsafe(ptr, len);+ return;+ }+ if (error != ERROR_SUCCESS) {+ throw std::runtime_error(format("read error: %s", GetErrorMessageWin32(error).c_str()));+ }++ seek(len > size - read_offset ? size : read_offset + len, SEEK_SET);+ }++ void read_aligned_chunk(void * dest, size_t len) {+ if (len == 0) {+ return;+ }++ const size_t offset = tell();+ if (offset > size || len > size - offset) {+ throw std::runtime_error("unexpectedly reached end of file");+ }++ const size_t aligned_offset = offset & ~(alignment - 1);+ const size_t offset_from_alignment = offset - aligned_offset;+ const size_t bytes_to_read = (offset_from_alignment + len + alignment - 1) & ~(alignment - 1);+ const size_t buffer_size = std::min<size_t>(bytes_to_read, LLAMA_DIRECT_IO_BUFFER_SIZE);++ struct aligned_buffer_deleter {+ void operator()(void * p) const { _aligned_free(p); }+ };+ std::unique_ptr<void, aligned_buffer_deleter> buffer(_aligned_malloc(buffer_size, alignment));+ if (!buffer) {+ LLAMA_LOG_WARN("%s: Falling back to buffered I/O due to %s\n",+ __func__, GetErrorMessageWin32(ERROR_NOT_ENOUGH_MEMORY).c_str());+ CloseHandle(fp_win32_direct);+ fp_win32_direct = INVALID_HANDLE_VALUE;+ alignment = 1;+ seek(offset, SEEK_SET);+ read_raw_unsafe(dest, len);+ return;+ }++ seek(aligned_offset, SEEK_SET);++ size_t skip = offset_from_alignment;+ size_t copied = 0;+ for (size_t done = 0; done < bytes_to_read; ) {+ const size_t n = std::min(buffer_size, bytes_to_read - done);+ read_raw_unsafe(buffer.get(), n);++ const size_t chunk_skip = std::min(skip, n);+ const size_t count = std::min(n - chunk_skip, len - copied);+ memcpy(reinterpret_cast<char *>(dest) + copied, reinterpret_cast<char *>(buffer.get()) + chunk_skip, count);++ copied += count;+ skip -= chunk_skip;+ done += n;+ }++ seek(offset + len, SEEK_SET);+ }++ void read_raw(void * ptr, size_t len) {+ if (len == 0) {+ return;+ }++ const size_t offset = tell();+ if (offset > size || len > size - offset) {+ throw std::runtime_error("unexpectedly reached end of file");+ }++ if (has_direct_io()) {+ read_aligned_chunk(ptr, len);+ } else {+ read_raw_unsafe(ptr, len); } } @@ -153,7 +372,7 @@ struct llama_file::impl { void write_raw(const void * ptr, size_t len) const { size_t bytes_written = 0; while (bytes_written < len) {- size_t chunk_size = std::min<size_t>(len - bytes_written, 64*1024*1024);+ size_t chunk_size = std::min<size_t>(len - bytes_written, LLAMA_DIRECT_IO_BUFFER_SIZE); DWORD chunk_written = 0; BOOL result = WriteFile(fp_win32, reinterpret_cast<char const*>(ptr) + bytes_written, chunk_size, &chunk_written, NULL); if (!result) {@@ -172,23 +391,29 @@ struct llama_file::impl { } bool has_direct_io() const {- return true;+ return fp_win32_direct != INVALID_HANDLE_VALUE && alignment > 1; } ~impl() {+ if (fp_win32_direct != INVALID_HANDLE_VALUE) {+ CloseHandle(fp_win32_direct);+ } if (fp && owns_fp) { std::fclose(fp); } }+ std::string fname; #else impl(const char * fname, const char * mode, [[maybe_unused]] const bool use_direct_io = false) : fname(fname) { #ifdef __linux__ // Try unbuffered I/O for read only if (use_direct_io && std::strcmp(mode, "rb") == 0) { if (init_fd()) {+ LLAMA_LOG_INFO("%s: file '%s' opened with direct I/O (alignment: %zu bytes)\n",+ __func__, fname, alignment); return; }- LLAMA_LOG_WARN("Failed to open file '%s' with error: %s. Falling back to buffered I/O",+ LLAMA_LOG_WARN("Failed to open file '%s' with unbuffered I/O: %s. Falling back to buffered I/O\n", fname, strerror(errno)); } #endif@@ -268,8 +493,8 @@ struct llama_file::impl { } errno = 0; if (fd == -1) {- const size_t curr_off = tell();- const size_t to_read = std::min(len, size - curr_off);+ const size_t read_offset = tell();+ const size_t to_read = std::min(len, size - read_offset); std::size_t ret = std::fread(ptr, to_read, 1, fp); if (ferror(fp)) {@@ -286,24 +511,23 @@ struct llama_file::impl { if (ret == -1) { if (errno == EINTR) {- continue; // Interrupted by signal, retry+ continue; }- // Fallback to std::fread in case the DMA controller cannot access the buffer if (errno == EFAULT || errno == EINVAL) {- LLAMA_LOG_WARN("%s: Falling back to buffered IO due to %s\n", __func__, strerror(errno));- auto curr_off = tell();+ LLAMA_LOG_WARN("%s: file '%s' falling back to buffered I/O due to %s\n",+ __func__, fname.c_str(), strerror(errno));+ const size_t read_offset = tell(); close(fd); fd = -1; alignment = 1; init_fp("rb");- seek(curr_off, SEEK_SET);+ seek(read_offset, SEEK_SET); read_raw_unsafe(ptr, len); return; } throw std::runtime_error(format("read error: %s", strerror(errno))); } if (ret == 0) {- // EOF: allow if this read was only pulling alignment padding past file end off_t pos = lseek(fd, 0, SEEK_CUR); if (pos != -1 && (size_t) pos == size) { std::memset(reinterpret_cast<char *>(ptr) + bytes_read, 0, len - bytes_read);@@ -317,19 +541,27 @@ struct llama_file::impl { } } - void read_aligned_chunk(void * dest, size_t size) {+ void read_aligned_chunk(void * dest, size_t len) { size_t offset = tell(); off_t aligned_offset = offset & ~(alignment - 1); off_t offset_from_alignment = offset - aligned_offset;- size_t bytes_to_read = (offset_from_alignment + size + alignment - 1) & ~(alignment - 1);+ size_t bytes_to_read = (offset_from_alignment + len + alignment - 1) & ~(alignment - 1); // stage through a bounded buffer, so that a large tensor is not held in memory twice while it loads const size_t buffer_size = std::min<size_t>(bytes_to_read, LLAMA_DIRECT_IO_BUFFER_SIZE); void * raw_buffer = nullptr; int ret = posix_memalign(&raw_buffer, alignment, buffer_size); if (ret != 0) {- throw std::runtime_error(format("posix_memalign failed with error %d", ret));+ LLAMA_LOG_WARN("%s: file '%s' falling back to buffered I/O due to %s\n",+ __func__, fname.c_str(), strerror(ret));+ close(fd);+ fd = -1;+ alignment = 1;+ init_fp("rb");+ seek(offset, SEEK_SET);+ read_raw_unsafe(dest, len);+ return; } struct aligned_buffer_deleter {@@ -345,13 +577,15 @@ struct llama_file::impl { const size_t n = std::min(buffer_size, bytes_to_read - done); read_raw_unsafe(buffer.get(), n); - const size_t count = std::min(n - skip, size - copied);+ const size_t count = std::min(n - skip, len - copied); memcpy(reinterpret_cast<char *>(dest) + copied, reinterpret_cast<char *>(buffer.get()) + skip, count); copied += count; skip = 0; done += n; }++ seek(offset + len, SEEK_SET); } void read_raw(void * ptr, size_t len) {@@ -390,7 +624,7 @@ struct llama_file::impl { ~impl() { if (fd != -1) { close(fd);- } else if (owns_fp) {+ } else if (fp && owns_fp) { std::fclose(fp); } }@@ -439,11 +673,7 @@ int llama_file::file_id() const { void llama_file::seek(size_t offset, int whence) const { pimpl->seek(offset, whence); } void llama_file::read_raw(void * ptr, size_t len) { pimpl->read_raw(ptr, len); }-#ifdef _WIN32-void llama_file::read_raw_unsafe(void * ptr, size_t len) { pimpl->read_raw(ptr, len); }-#else void llama_file::read_raw_unsafe(void * ptr, size_t len) { pimpl->read_raw_unsafe(ptr, len); }-#endif uint32_t llama_file::read_u32() { return pimpl->read_u32(); } tests/CMakeLists.txt7 + / 0 −
@@ -355,6 +355,13 @@ llama_build(test-backend-ops.cpp LINK ggml) llama_build_and_test(test-backend-tensor.cpp LINK ggml) +if (BUILD_SHARED_LIBS)+ llama_build_and_test(test-llama-mmap.cpp ${PROJECT_SOURCE_DIR}/src/llama-impl.cpp ${PROJECT_SOURCE_DIR}/src/llama-mmap.cpp LABEL behavior)+ target_compile_definitions(test-llama-mmap PRIVATE LLAMA_BUILD)+else()+ llama_build_and_test(test-llama-mmap.cpp LABEL behavior)+endif()+ llama_build_and_test(test-tiled-mulmat.cpp LABEL "main behavior backend" LINK ggml) llama_build_and_test(test-model-load-cancel.cpp LABEL "model behavior") llama_build_and_test(test-autorelease.cpp LABEL "model behavior")tests/test-llama-mmap.cppadded232 + / 0 −
@@ -0,0 +1,232 @@+#include "testing.h"++#include "../src/llama-mmap.h"++#include <algorithm>+#include <chrono>+#include <cstddef>+#include <cstdio>+#include <cstdlib>+#include <filesystem>+#include <fstream>+#include <stdexcept>+#include <string>+#include <system_error>+#include <utility>+#include <vector>++#if defined(_WIN32)+#include <malloc.h>+#endif++static constexpr size_t test_file_size = LLAMA_DIRECT_IO_BUFFER_SIZE + 3*1024*1024 + 1;+static constexpr size_t test_io_size = 1024*1024;++static uint8_t test_byte(size_t offset) {+ // Mix high offset bits into each byte to detect repeated chunks.+ uint64_t value = offset;+ value = (value ^ (value >> 30)) * 0xbf58476d1ce4e5b9ULL;+ value = (value ^ (value >> 27)) * 0x94d049bb133111ebULL;+ return static_cast<uint8_t>(value ^ (value >> 31));+}++static bool matches(const void * data, size_t offset, size_t size) {+ const auto * bytes = static_cast<const uint8_t *>(data);+ for (size_t i = 0; i < size; ++i) {+ if (bytes[i] != test_byte(offset + i)) {+ return false;+ }+ }+ return true;+}++static void check_test_pattern(testing & t) {+ uint8_t buffer[256];+ for (size_t i = 0; i < sizeof(buffer); ++i) {+ buffer[i] = test_byte(i);+ }+ t.assert_true("reject data repeated after 512 KiB", !matches(buffer, 512 * 1024, sizeof(buffer)));+ t.assert_true("reject data repeated after a direct I/O chunk", !matches(buffer, LLAMA_DIRECT_IO_BUFFER_SIZE, sizeof(buffer)));+}++struct temporary_file {+ std::filesystem::path path;++ temporary_file() {+ const auto now = std::chrono::steady_clock::now().time_since_epoch().count();+ path = std::filesystem::temp_directory_path() / ("test-llama-mmap-" + std::to_string(now) + ".bin");+ }++ ~temporary_file() {+ std::error_code error;+ std::filesystem::remove(path, error);+ }+};++struct aligned_buffer {+ void * data = nullptr;++ aligned_buffer(size_t alignment, size_t size) {+ alignment = std::max(alignment, alignof(std::max_align_t));+#if defined(_WIN32)+ data = _aligned_malloc(size, alignment);+#else+ if (posix_memalign(&data, alignment, size) != 0) {+ data = nullptr;+ }+#endif+ }++ ~aligned_buffer() {+#if defined(_WIN32)+ _aligned_free(data);+#else+ free(data);+#endif+ }++ aligned_buffer(const aligned_buffer &) = delete;+ aligned_buffer & operator=(const aligned_buffer &) = delete;+};++static void write_test_file(const std::filesystem::path & path) {+ std::ofstream file(path, std::ios::binary);+ if (!file) {+ throw std::runtime_error("failed to create test file");+ }++ std::vector<uint8_t> buffer(test_io_size);+ for (size_t offset = 0; offset < test_file_size; offset += buffer.size()) {+ const size_t size = std::min(buffer.size(), test_file_size - offset);+ for (size_t i = 0; i < size; ++i) {+ buffer[i] = test_byte(offset + i);+ }+ file.write(reinterpret_cast<const char *>(buffer.data()), size);+ if (!file) {+ throw std::runtime_error("failed to write test file");+ }+ }+}++static void check_file(testing & t, const std::filesystem::path & path, bool use_direct_io) {+ llama_file file(path.string().c_str(), "rb", use_direct_io);+ if (use_direct_io && !file.has_direct_io()) {+ t.skip("direct I/O is unavailable for the test file");+ return;+ }+ t.assert_equal("requested I/O mode", use_direct_io, file.has_direct_io());+ t.assert_equal(test_file_size, file.size());+ t.assert_true("read alignment is nonzero", file.read_alignment() != 0);++ const size_t alignment = file.read_alignment();+ const std::vector<std::pair<size_t, size_t>> ranges = {+ { 0, 1 },+ { 1, 31 },+ { alignment - 1, alignment + 3 },+ { LLAMA_DIRECT_IO_BUFFER_SIZE - 11, 27 },+ { test_file_size - 31, 31 },+ { 1, LLAMA_DIRECT_IO_BUFFER_SIZE + 31 },+ { 1, test_file_size - 1 },+ {0, test_file_size}+ };+ for (const auto & range : ranges) {+ std::vector<uint8_t> buffer(range.second);+ file.seek(range.first, SEEK_SET);+ file.read_raw(buffer.data(), buffer.size());+ t.assert_equal("I/O mode after read", use_direct_io, file.has_direct_io());+ t.assert_true("read expected bytes", matches(buffer.data(), range.first, buffer.size()));+ t.assert_equal(range.first + range.second, file.tell());+ }++ std::vector<uint8_t> buffer(test_io_size - 1);+ file.seek(0, SEEK_SET);+ for (size_t offset = 0; offset < test_file_size; offset += buffer.size()) {+ const size_t size = std::min(buffer.size(), test_file_size - offset);+ file.read_raw(buffer.data(), size);+ t.assert_equal("I/O mode after sequential read", use_direct_io, file.has_direct_io());+ t.assert_true("sequential read expected bytes", matches(buffer.data(), offset, size));+ t.assert_equal(offset + size, file.tell());+ }++ const size_t unsafe_size = ((LLAMA_DIRECT_IO_BUFFER_SIZE + 3 * alignment - 1) / alignment) * alignment;+ aligned_buffer buffer_aligned(alignment, unsafe_size);+ t.assert_true("allocate aligned buffer", buffer_aligned.data != nullptr);+ if (buffer_aligned.data != nullptr) {+ const size_t offset = 7 * alignment;+ file.seek(offset, SEEK_SET);+ file.read_raw_unsafe(buffer_aligned.data, unsafe_size);+ t.assert_equal("I/O mode after unsafe read", use_direct_io, file.has_direct_io());+ t.assert_true("unsafe read expected bytes", matches(buffer_aligned.data, offset, unsafe_size));+ t.assert_equal(offset + unsafe_size, file.tell());++ const size_t final_offset = (test_file_size - 1) & ~(alignment - 1);+ file.seek(final_offset, SEEK_SET);+ file.read_raw_unsafe(buffer_aligned.data, alignment);+ t.assert_equal("I/O mode after unsafe EOF read", use_direct_io, file.has_direct_io());+ t.assert_true("unsafe EOF read expected bytes", matches(buffer_aligned.data, final_offset, test_file_size - final_offset));+ t.assert_equal(test_file_size, file.tell());+ }+}++static void check_direct_io_fallback(testing & t, const std::filesystem::path & path) {+ llama_file file(path.string().c_str(), "rb", true);+ if (!file.has_direct_io()) {+ t.skip("direct I/O is unavailable for the test file");+ return;+ }+ t.assert_true("direct I/O is enabled before fallback", file.has_direct_io());++ const size_t alignment = file.read_alignment();+ aligned_buffer buffer(alignment, alignment + 1);+ if (!t.assert_true("allocate aligned buffer", buffer.data != nullptr)) {+ return;+ }+ auto * unaligned = static_cast<uint8_t *>(buffer.data) + 1;+ const size_t offset = 7 * alignment;+ file.seek(offset, SEEK_SET);+ file.read_raw_unsafe(unaligned, alignment);++#ifdef _WIN32+ t.assert_true("unaligned read disables direct I/O", !file.has_direct_io());+ t.assert_equal(size_t(1), file.read_alignment());+#endif+ t.assert_true("fallback read expected bytes", matches(unaligned, offset, alignment));+ t.assert_equal(offset + alignment, file.tell());++ file.read_raw(unaligned, alignment);+#ifdef _WIN32+ t.assert_true("subsequent read stays buffered", !file.has_direct_io());+#endif+ t.assert_true("subsequent read expected bytes", matches(unaligned, offset + alignment, alignment));+ t.assert_equal(offset + 2 * alignment, file.tell());+}++static void check_mmap(testing & t, const std::filesystem::path & path) {+ if (!llama_mmap::SUPPORTED) {+ return;+ }++ llama_file file(path.string().c_str(), "rb");+ llama_mmap mapping(&file, 0, false);+ t.assert_equal(test_file_size, mapping.size());+ const auto * data = static_cast<const uint8_t *>(mapping.addr());+ t.assert_true("mapping has an address", data != nullptr);+ if (data != nullptr) {+ t.assert_true("first byte", matches(data, 0, 1));+ t.assert_true("chunk boundary", matches(data + LLAMA_DIRECT_IO_BUFFER_SIZE - 11, LLAMA_DIRECT_IO_BUFFER_SIZE - 11, 27));+ t.assert_true("last bytes", matches(data + test_file_size - 31, test_file_size - 31, 31));+ }+}++int main() {+ testing t;+ temporary_file test_file;+ t.test("test_pattern", [&] (testing & t) { check_test_pattern(t); });+ t.test("write_test_file", [&] (testing &) { write_test_file(test_file.path); });++ t.test("llama_file.buffered", [&] (testing & t) { check_file(t, test_file.path, false); });+ t.test("llama_file.direct", [&] (testing & t) { check_file(t, test_file.path, true); });+ t.test("llama_file.direct_fallback", [&] (testing & t) { check_direct_io_fallback(t, test_file.path); });+ t.test("llama_mmap", [&] (testing & t) { check_mmap(t, test_file.path); });+ return t.summary();+}