72 , offset(offset), size(size), buf((void*)buf)
114 : alignment(alignment), blockSize(blockSize), queueDepth(queueDepth)
129 ENSURE(1 <= queueDepth && queueDepth <= maxQueueDepth);
137 static const size_t maxQueueDepth = 32;
146 #define IO_OVERLAPPED io::Parameters(io::Parameters::OverlappedTag()) 191 const bool temporaryBuffersRequested = (op.
buf == 0);
192 if(temporaryBuffersRequested)
195 for(
size_t i = 0; i <
ARRAY_SIZE(controlBlocks); i++)
197 aiocb& cb = operator[](i);
201 if(temporaryBuffersRequested)
202 cb.
aio_buf = (
volatile void*)(uintptr_t(buffers.get()) + i * blockSize);
208 return controlBlocks[counter %
ARRAY_SIZE(controlBlocks)];
213 aiocb controlBlocks[Parameters::maxQueueDepth];
226 #ifndef ENABLE_IO_STATS 227 #define ENABLE_IO_STATS 0 232 template<
class CompletedHook,
class IssueHook>
246 for(
off_t blocksIssued = 0, blocksCompleted = 0; blocksCompleted < numBlocks; blocksCompleted++)
248 for(; blocksIssued != numBlocks && blocksIssued < blocksCompleted + (
off_t)p.queueDepth; blocksIssued++)
250 aiocb& cb = controlBlockRingBuffer[blocksIssued];
253 cb.
aio_buf = (
volatile void*)(uintptr_t(op.
buf) + blocksIssued * p.blockSize);
254 if(blocksIssued == numBlocks-1)
262 aiocb& cb = controlBlockRingBuffer[blocksCompleted];
271 const off_t totalSize = p.blockSize? numBlocks*p.blockSize : op.
size;
272 debug_printf(
"IO: %.2f MB/s (%.2f)\n", totalSize/(t1-t0)/1e6, (t1-t0)*1e3);
279 template<
class CompletedHook>
297 template<
class CompletedHook,
class IssueHook>
298 static inline Status Store(
const OsPath& pathname,
const void* data,
size_t size,
const Parameters& p =
Parameters(),
const CompletedHook& completedHook = CompletedHook(),
const IssueHook& issueHook = IssueHook())
301 int oflag = O_WRONLY;
302 if(p.queueDepth != 1)
320 template<
class CompletedHook>
336 template<
class CompletedHook,
class IssueHook>
337 static inline Status Load(
const OsPath& pathname,
void* buf,
size_t size,
const Parameters& p =
Parameters(),
const CompletedHook& completedHook = CompletedHook(),
const IssueHook& issueHook = IssueHook())
340 int oflag = O_RDONLY;
341 if(p.queueDepth != 1)
345 return io::Run(op, p, completedHook, issueHook);
348 template<
class CompletedHook>
361 #endif // #ifndef INCLUDED_IO #define RETURN_STATUS_FROM_CALLBACK(expression)
Definition: status.h:338
#define O_DIRECT
Definition: filesystem.h:67
ControlBlockRingBuffer(const Operation &op, const Parameters &p)
Definition: io.h:186
static const size_t pageSize
Definition: alignment.h:83
#define UNUSED(param)
mark a function parameter as unused and avoid the corresponding compiler warning. ...
Definition: code_annotation.h:38
size_t aio_nbytes
Definition: waio.h:68
Status operator()(const u8 *block, size_t blockSize) const
called after a block I/O has completed.
Definition: io.h:158
#define COMPILER_FENCE
prevent the compiler from reordering loads or stores across this point.
Definition: code_annotation.h:269
Status waio_Preallocate(int fd, off_t size)
Definition: waio.cpp:437
T DivideRoundUp(T dividend, T divisor)
Definition: lib.h:76
Flags
Definition: cache.cpp:308
size_t queueDepth
Definition: io.h:143
const Status OK
Definition: status.h:386
const Status IO
Definition: io.h:43
static ICounter * counter
Definition: whrt.cpp:96
Definition: unique_range.h:75
static const uintptr_t maxSectorSize
Definition: alignment.h:104
off_t alignment
Definition: io.h:139
T round_up(T n, T multiple)
round number up/down to the next given multiple.
Definition: bits.h:265
Status WaitUntilComplete(aiocb &cb, size_t queueDepth)
Definition: io.cpp:67
static Status Store(const OsPath &pathname, const void *data, size_t size, const Parameters &p=Parameters())
Definition: io.h:326
Parameters()
Definition: io.h:99
void Close()
Definition: file.h:73
uint8_t u8
Definition: types.h:37
LIB_API int wtruncate(const OsPath &pathname, off_t length)
Definition: ufilesystem.cpp:123
volatile void * aio_buf
Definition: waio.h:67
#define ARRAY_SIZE(name)
Definition: code_annotation.h:336
bool IsAligned(T t, uintptr_t multiple)
Definition: alignment.h:30
uint64_t u64
Definition: types.h:40
#define ENSURE(expr)
ensure the expression <expr> evaluates to non-zero.
Definition: debug.h:287
INLINE aiocb & operator[](size_t counter)
Definition: io.h:206
int aio_lio_opcode
Definition: waio.h:71
__int64 off_t
Definition: wposix_types.h:91
static const size_t KiB
Definition: alignment.h:93
UniqueRange buffers
Definition: io.h:212
size_t blockSize
Definition: io.h:141
int fd
Definition: io.h:86
static UniqueRange Allocate(size_t size, size_t alignment=maxSectorSize)
Definition: io.h:55
i64 Status
Error handling system.
Definition: status.h:171
void debug_printf(const char *fmt,...)
write a formatted string to the debug channel, subject to filtering (see below).
Definition: debug.cpp:142
double timer_Time()
Definition: timer.cpp:98
u8 Descriptor
Definition: cache.cpp:255
Status Open(const OsPath &pathname, int oflag)
Definition: file.h:63
Introduction
Definition: debug.h:404
Parameters(OverlappedTag)
Definition: io.h:108
static Status AllocateAligned(shared_ptr< T > &p, size_t size, size_t alignment=cacheLineSize)
Definition: shared_ptr.h:66
off_t size
Definition: io.h:90
bool is_pow2(T n)
Definition: bits.h:164
off_t aio_offset
Definition: waio.h:66
void Validate() const
Definition: io.h:76
Parameters(size_t blockSize, size_t queueDepth, off_t alignment=maxSectorSize)
Definition: io.h:113
Operation(const File &file, void *buf, off_t size, off_t offset=0)
Definition: io.h:70
off_t offset
Definition: io.h:89
void * buf
Definition: io.h:91
static Status Run(const Operation &op, const Parameters &p=Parameters())
Definition: io.h:285
Status operator()(aiocb &cb) const
called before a block I/O is issued.
Definition: io.h:174
static int Issue(aiocb *cb)
Definition: waio.cpp:494
static Status Load(const OsPath &pathname, void *buf, size_t size, const Parameters &p=Parameters())
Definition: io.h:354
int opcode
Definition: io.h:87
int aio_fildes
Definition: waio.h:65
void Validate(const Operation &op) const
Definition: io.h:118
#define RETURN_STATUS_IF_ERR(expression)
Definition: status.h:276
#define INLINE
Definition: code_annotation.h:363
static Status Run(const Operation &op, const Parameters &p=Parameters(), const CompletedHook &completedHook=CompletedHook(), const IssueHook &issueHook=IssueHook())
Definition: io.h:233