cml/libcw
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
fe74e41a0d
libcw/cwDataSets.cpp
kevin 9acb24e9e1 Apply licensing information to project.
2024-12-01 14:35:24 -05:00

2685 lines
74 KiB
C++
Raw Blame History

//| Copyright: (C) 2020-2024 Kevin Larke <contact AT larke DOT org>
//| License: GNU GPL version 3.0 or above. See the accompanying LICENSE file.
#include "cwCommon.h"
#include "cwLog.h"
#include "cwCommonImpl.h"
#include "cwTest.h"
#include "cwMem.h"
#include "cwObject.h"
#include "cwFile.h"
#include "cwFileSys.h"
#include "cwVectOps.h"
#include "cwMtx.h"
#include "cwVariant.h"
#include "cwDataSets.h"
#include "cwSvg.h"
#include "cwTime.h"
#include "cwText.h"
#include "cwMath.h"
//----------------------------------------------------------------------------------------------------------------------------
//----------------------------------------------------------------------------------------------------------------------------
//----------------------------------------------------------------------------------------------------------------------------
namespace cw
{
namespace dataset
{
namespace wtr
{
typedef struct col_str
{
rdr::col_t col; // Public fields - See rdr::col_t.
unsigned char* cur; // Cache of the current column data contents.
unsigned curByteN; // Count of bytes in cur[].
unsigned* curDimV; // Cache of the current column dimensions.
struct col_str* link; // Link to next col_t record.
} col_t;
typedef struct wtr_str
{
file::handle_t fH; // Output file handle
unsigned record_count; // Total count of rows.
col_t* colL; // Linked list of column descriptions
unsigned totalVarDimN; // Total count of unknown dim's among all columns
} wtr_t;
inline wtr_t* _handleToPtr( handle_t h )
{ return handleToPtr<handle_t,wtr_t>(h); }
rc_t _destroy( wtr_t* p )
{
col_t* c = p->colL;
while( c != nullptr )
{
col_t* c0 = c->link;
mem::free(const_cast<char*>(c->col.label));
mem::release(c->col.dimV);
mem::release(c->col.maxDimV);
mem::release(c->cur);
mem::release(c->curDimV);
mem::release(c);
c = c0;
}
file::close(p->fH);
mem::release(p);
return kOkRC;
}
col_t* _columnIdToPtr( wtr_t* p, unsigned columnId )
{
col_t* c = p->colL;
for(; c!=nullptr; c=c->link )
if( c->col.id == columnId )
return c;
cwLogError(kInvalidArgRC,"The dataset column id %i was not found.",columnId);
return nullptr;
}
// eleN = count of elements in dV[]
// dimV[ dimN ] = dimensions for variable sized data elements. cumprod(dimV) must equal eleN
rc_t _write_column_to_buf( wtr_t* p, unsigned columnId, unsigned eleN, const unsigned* dimV, unsigned dimN, const void* dV, unsigned typeFlags, col_t*& colPtrRef )
{
col_t* c = _columnIdToPtr(p,columnId);
if( c == nullptr )
return cwLogError(kInvalidArgRC,"Unable to locate the column description associated with id: %i.",columnId);
// if this is a fixed size column
if( c->col.varDimN == 0 )
{
// verify that the element count matches the fixed element count
if( eleN != c->col.maxEleN )
return cwLogError(kInvalidArgRC,"Data vector in column '%s' has %i elements but should have %i elements.", cwStringNullGuard(c->col.label), eleN, c->col.maxEleN );
if( dimV != nullptr || dimN != 0 )
cwLogWarning("The dimension vector for the fixed sized column '%s' is ignored in the write() function.",cwStringNullGuard(c->col.label));
}
else // this is a variable sized column
{
unsigned tmpEleN = 1;
for(unsigned i=0; i<c->col.rankN; ++i)
{
tmpEleN *= dimV[i]; // track the count of elements
c->col.maxDimV[i] = std::max(c->col.maxDimV[i], dimV[i] ); // track the max. dimension
c->curDimV[i] = dimV[i]; // store the this columns dimensions
}
// verify that the sizeof the data matches the size given in the dimensions
if( tmpEleN != eleN )
return cwLogError(kInvalidArgRC,"The product of the dimension vector does not equal the count of elements in column '%s'.",c->col.label);
}
if( p->record_count == 0)
{
// set data type
c->col.max.flags = typeFlags;
c->col.min.flags = typeFlags;
}
else
{
// verify data type is the same for all elements
if( c->col.max.flags != typeFlags )
return cwLogError(kInvalidArgRC,"The data vector type '%s' does not match the column type '%s'.", variant::flagsToLabel(typeFlags), variant::flagsToLabel(c->col.max.flags));
}
// store the bytes associated with col/row
unsigned bytesPerEle = variant::flagsToBytes(typeFlags);
if( bytesPerEle == 0 )
return cwLogError(kInvalidArgRC,"Invalid type identifier in column '%s'.", cwStringNullGuard(c->col.label));
else
{
c->curByteN = bytesPerEle * eleN;
c->cur = mem::resize<unsigned char>(c->cur,c->curByteN);
memcpy(c->cur,dV,c->curByteN);
}
colPtrRef = c;
return kOkRC;
}
rc_t _write_hdr( wtr_t* p )
{
col_t* c;
rc_t rc;
p->totalVarDimN = 0;
// get the count of columns
unsigned col_count = 0;
for(c=p->colL; c!=nullptr; c=c->link)
++col_count;
if((rc = file::write( p->fH, p->record_count )) != kOkRC ) goto errLabel;
if((rc = file::write( p->fH, col_count )) != kOkRC ) goto errLabel;
for(c=p->colL; c!=nullptr; c=c->link)
{
if((rc = file::writeStr( p->fH, c->col.label )) != kOkRC ) goto errLabel;
if((rc = file::write( p->fH, c->col.id )) != kOkRC ) goto errLabel;
if((rc = file::write( p->fH, c->col.varDimN )) != kOkRC ) goto errLabel;
if((rc = file::write( p->fH, c->col.rankN )) != kOkRC ) goto errLabel;
if((rc = file::write( p->fH, c->col.maxEleN )) != kOkRC ) goto errLabel;
if((rc = variant::write( p->fH, c->col.max)) != kOkRC ) goto errLabel;
if((rc = variant::write( p->fH, c->col.min )) != kOkRC ) goto errLabel;
for(unsigned i=0; i<c->col.rankN; ++i)
{
if((rc = file::write( p->fH, c->col.dimV[i] )) != kOkRC ) goto errLabel;
if((rc = file::write( p->fH, c->col.maxDimV[i])) != kOkRC ) goto errLabel;
}
p->totalVarDimN += c->col.varDimN;
}
errLabel:
return rc;
}
rc_t _re_write_hdr( wtr_t* p )
{
rc_t rc;
if((rc = file::seek( p->fH, file::kBeginFl, 0)) != kOkRC )
return cwLogError( kSeekFailRC, "Data file Header seek failed.");
if((rc = _write_hdr( p )) != kOkRC )
return cwLogError( rc, "Header re-write failed.");
return rc;
}
}
}
}
cw::rc_t cw::dataset::wtr::create( handle_t& h, const char* fn )
{
rc_t rc;
if((rc = destroy(h)) != kOkRC )
return rc;
auto p = mem::allocZ<wtr_t>(1);
if((rc = file::open(p->fH,fn,file::kWriteFl)) != kOkRC )
{
rc = cwLogError(rc,"Data file creation failed.");
goto errLabel;
}
h.set(p);
errLabel:
if(rc != kOkRC )
_destroy(p);
return rc;
}
cw::rc_t cw::dataset::wtr::destroy( handle_t& h )
{
rc_t rc = kOkRC;
if( !h.isValid())
return rc;
wtr_t* p = _handleToPtr(h);
if(( rc = _re_write_hdr( p )) != kOkRC )
return rc;
if((rc = _destroy(p)) != kOkRC )
return rc;
h.clear();
return rc;
}
cw::rc_t cw::dataset::wtr::define_columns( handle_t h, const char* label, unsigned columnId, unsigned rankN, const unsigned* dimV )
{
rc_t rc = kOkRC;
wtr_t* p = _handleToPtr(h);
col_t* c = mem::allocZ<col_t>(1);
c->col.label = mem::duplStr(label);
c->col.id = columnId;
c->col.rankN = rankN;
c->col.varDimN = 0;
c->col.dimV = mem::allocDupl<unsigned>( dimV, rankN );
c->col.maxDimV = mem::allocDupl<unsigned>( dimV, rankN );
c->curDimV = mem::allocDupl<unsigned>( dimV, rankN );
c->col.maxEleN = 1;
for(unsigned i=0; i<rankN; ++i)
{
c->col.maxEleN *= dimV[i];
if( dimV[i] == 0 )
{
c->col.varDimN +=1;
}
}
// link the new col recd to the end of the column list
col_t* c0 = p->colL;
col_t* c1 = nullptr;
for(; c0!=nullptr; c0=c0->link)
c1 = c0;
if( c1 == nullptr )
p->colL = c;
else
c1->link = c;
return rc;
}
cw::rc_t cw::dataset::wtr::write( handle_t h, unsigned columnId, const int* dV, unsigned eleN, const unsigned* dimV, unsigned rankN )
{
rc_t rc;
wtr_t* p = _handleToPtr(h);
col_t* c = nullptr;
if((rc = _write_column_to_buf( p, columnId, eleN, dimV, rankN, dV, variant::kInt32VFl, c)) != kOkRC )
return rc;
if( p->record_count == 0 )
{
c->col.min.u.i32 = vop::min( dV, eleN );
c->col.max.u.i32 = vop::max( dV, eleN );
//printf("0i %i %i\n", columnId, c->col.min.u.i32 );
}
else
{
//printf("1i %i %i\n", columnId, c->col.min.u.i32 );
c->col.min.u.i32 = std::min(c->col.min.u.i32, vop::min( dV, eleN ));
c->col.max.u.i32 = std::max(c->col.max.u.i32, vop::max( dV, eleN ));
}
return rc;
}
cw::rc_t cw::dataset::wtr::write( handle_t h, unsigned columnId, const float* dV, unsigned eleN, const unsigned* dimV, unsigned rankN )
{
rc_t rc;
wtr_t* p = _handleToPtr(h);
col_t* c = nullptr;
if((rc = _write_column_to_buf( p, columnId, eleN, dimV, rankN, dV, variant::kFloatVFl, c)) != kOkRC )
return rc;
if( p->record_count == 0 )
{
c->col.min.u.f = vop::min( dV, eleN );
c->col.max.u.f = vop::max( dV, eleN );
}
else
{
c->col.min.u.f = std::min(c->col.min.u.f, vop::min( dV, eleN ));
c->col.max.u.f = std::max(c->col.max.u.f, vop::max( dV, eleN ));
}
return rc;
}
cw::rc_t cw::dataset::wtr::write( handle_t h, unsigned columnId, const double* dV, unsigned eleN, const unsigned* dimV, unsigned rankN )
{
rc_t rc;
wtr_t* p = _handleToPtr(h);
col_t* c = nullptr;
if((rc = _write_column_to_buf( p, columnId, eleN, dimV, rankN, dV, variant::kDoubleVFl, c)) != kOkRC )
return rc;
if( p->record_count == 0 )
{
c->col.min.u.d = vop::min( dV, eleN );
c->col.max.u.d = vop::max( dV, eleN );
}
else
{
c->col.min.u.d = std::min(c->col.min.u.d, vop::min( dV, eleN ));
c->col.max.u.d = std::max(c->col.max.u.d, vop::max( dV, eleN ));
}
return rc;
}
cw::rc_t cw::dataset::wtr::write_record( handle_t h )
{
rc_t rc;
wtr_t* p = _handleToPtr(h);
col_t* c;
// if this is the first row in the file then write the file header
if( p->record_count == 0 )
if((rc = _write_hdr(p)) != kOkRC )
return rc;
unsigned rowByteN = 0;
// calculate the size of the row data
for(c=p->colL; c!=nullptr; c=c->link)
rowByteN += c->col.varDimN * sizeof(unsigned) + c->curByteN;
// write the size of this row
if((rc = file::write(p->fH, rowByteN)) != kOkRC )
goto errLabel;
// for each column
for(c=p->colL; c!=nullptr; c=c->link)
{
// if this is a variable sized column
if( c->col.varDimN > 0 )
{
// then write the variable sized dimensions
for(unsigned i=0; i<c->col.rankN; ++i)
if( c->col.dimV[i] == 0 )
if((rc = file::write( p->fH, c->curDimV[i] )) != kOkRC )
goto errLabel;
}
// write the column field value
if((rc = file::write( p->fH, c->cur, c->curByteN)) != kOkRC )
goto errLabel;
}
errLabel:
if( rc != kOkRC )
rc = cwLogError(rc,"Example index %i write failed", p->record_count);
else
p->record_count += 1;
return rc;
}
/*
File Format for the following data.
where the data record itself is repeated 3 time.
unsigned dim0V[] = {1};
unsigned dim1V[] = {3};
unsigned dim2V[] = {2,0};
unsigned dim3V[] = {2,2};
int val0[] = {0};
int val1[] = {1,2,3};
int val2[] = {4,5,6,7,8,9};
int val3[] = {10,11,13,14};
0300 0000 3 recd_count
0400 0000 4 col_count
0400 0000 label size - col0
636f 6c30 label
0000 0000 id
0000 0000 varDimN
0100 0000 rankN
0100 0000 maxEleN
4000 0000 max type
0000 0000 max value
0000 0000
4000 0000 min type
0000 0000 min value
0000 0000
0100 0000 dimV[0]
0100 0000 maxDimV[0]
0400 0000 label size - col 1
636f 6c31 label
0100 0000 id
0000 0000 varDimN
0100 0000 rankN
0300 0000 maxEleN
4000 0000 max type
0300 0000 max value
0000 0000
4000 0000 min type
0100 0000 max value
0000 0000
0300 0000 dimV[0]
0300 0000 maxDimV[0]
0400 0000 label size - col 2
636f 6c32 label
0200 0000 id
0100 0000 varDimN
0200 0000 rankN
0000 0000 maxEleN
4000 0000 max type
0900 0000 max value
0000 0000
4000 0000 min type
0400 0000 min value
0000 0000
0200 0000 dimV[0]
0200 0000 maxDimV[0]
0000 0000 dimV[1]
0300 0000 maxDimV[1]
0400 0000 label size - col 3
636f 6c33 label
0300 0000 id
0000 0000 varDimN
0200 0000 rankN
0400 0000 maxEleN
4000 0000 max type
0e00 0000 max value
0000 0000
4000 0000 min type
0a00 0000 min value
0000 0000
0200 0000 dimV[0]
0200 0000 maxDimV[0]
0200 0000 dimV[1]
0200 0000 maxDimV[1]
3c00 0000 recd0 size (60 bytes)
0000 0000 0 col0
0100 0000 1 col1[0]
0200 0000 2 col1[1]
0300 0000 3 col1[2]
0300 0000 dimV[1] col2 <- variable dimension
0400 0000 4 col2[0]
0500 0000 5
0600 0000 6
0700 0000 7
0800 0000 8
0900 0000 9
0a00 0000 10 col3
0b00 0000 11
0d00 0000 12
0e00 0000 13
3c00 0000 recd1 size (60 bytes)
0100 0000 1 col0
0100 0000
0200 0000
0300 0000
0300 0000
0400 0000
0500 0000
0600 0000
0700 0000
0800 0000
0900 0000
0a00 0000
0b00 0000
0d00 0000
0e00 0000
3c00 0000 recd2 size (60 bytes)
0200 0000 2 col0
0100 0000
0200 0000
0300 0000
0300 0000
0400 0000
0500 0000
0600 0000
0700 0000
0800 0000
0900 0000
0a00 0000
0b00 0000
0d00 0000
0e00 0000
*/
cw::rc_t cw::dataset::wtr::test( const object_t* cfg )
{
rc_t rc = kOkRC;
char* outFn = nullptr;
handle_t h;
if((rc = cfg->getv("outFn",outFn)) != kOkRC )
return cwLogError(rc,"wtr test failed. Argument parse failed.");
outFn = filesys::expandPath(outFn);
if((rc = create(h,outFn)) != kOkRC )
{
rc = cwLogError(rc,"rdr create failed.");
goto errLabel;
}
else
{
enum { kId0, kId1, kId2, kId3 };
unsigned dim0V[] = {1};
unsigned dim1V[] = {3};
unsigned dim2V[] = {2,0};
unsigned dim3V[] = {2,2};
unsigned dim0N = cwCountOf(dim0V);
unsigned dim1N = cwCountOf(dim1V);
unsigned dim2N = cwCountOf(dim2V);
unsigned dim3N = cwCountOf(dim3V);
int val0[] = {0};
int val1[] = {1,2,3};
int val2[] = {4,5,6,7,8,9};
int val3[] = {10,11,13,14};
if((rc = define_columns(h, "col0", kId0, dim0N, dim0V )) != kOkRC )
{
rc = cwLogError(rc,"Define column 0 failed.");
goto errLabel;
}
if((rc = define_columns(h, "col1", kId1, dim1N, dim1V )) != kOkRC )
{
rc = cwLogError(rc,"Define column 1 failed.");
goto errLabel;
}
if((rc = define_columns(h, "col2", kId2, dim2N, dim2V )) != kOkRC )
{
rc = cwLogError(rc,"Define column 2 failed.");
goto errLabel;
}
if((rc = define_columns(h, "col3", kId3, dim3N, dim3V )) != kOkRC )
{
rc = cwLogError(rc,"Define column 3 failed.");
goto errLabel;
}
for(unsigned i=0; i<3; ++i)
{
val0[0] = i;
write( h, kId0, val0, dim0V[0] );
write( h, kId1, val1, dim1V[0] );
dim2V[1] = 3;
write( h, kId2, val2, dim2V[0]*dim2V[1], dim2V, dim2N );
write( h, kId3, val3, dim3V[0]*dim3V[1] );
write_record(h);
}
}
errLabel:
destroy(h);
mem::release(outFn);
return rc;
}
//----------------------------------------------------------------------------------------------------------------------------
//----------------------------------------------------------------------------------------------------------------------------
//----------------------------------------------------------------------------------------------------------------------------
namespace cw
{
namespace dataset
{
namespace rdr
{
enum
{
kSizeofRecordHeader = sizeof(unsigned)
};
typedef struct cache_str
{
file::handle_t fH;
unsigned totalRecdN; // Total count of records in the file
std::uint8_t * buf; // File buffer memory
unsigned bufMaxByteN; // Allocated size of buf[]
unsigned bufByteN; // Bytes in buf[]
unsigned baseFileOffs; // Offset of the first record in the file
unsigned* tocV; // tocV[tocN] Cached record byte offsets
unsigned tocN; // Count of records in the cache
unsigned tocBaseIdx; // Record index of the first record in the cache
unsigned tocIdx; // Record index of next record to return
unsigned state; // See rdr::k???State
bool shuffleFl; // shuffle the file buffer each time it is filled
} cache_t;
// Backup the file position to the beginning of the last (partial) record in the cache.
// Note that the last record overlaps the end of the cache is is therefore incomplete.
rc_t _cache_backup( cache_t* p, unsigned actByteN, unsigned cacheByteN )
{
rc_t rc = kOkRC;
if( p->state == kEofRC )
return kEofRC;
assert( actByteN >= cacheByteN);
if((rc = file::seek(p->fH,file::kCurFl, -(int)(actByteN-cacheByteN) )) != kOkRC )
return cwLogError(rc,"Dataset rdr cache align failed.");
return rc;
}
// Count the records in the case and re-align the current file position to the last (partial) record in the cache
rc_t _cache_count_and_align( cache_t* p, unsigned actByteN )
{
p->bufByteN = 0;
for(p->tocN=0; p->bufByteN < actByteN; ++p->tocN )
{
if( p->bufByteN + kSizeofRecordHeader >= actByteN )
{
_cache_backup( p, actByteN, p->bufByteN);
break;
}
unsigned recdByteN = *reinterpret_cast<unsigned*>(p->buf + p->bufByteN);
// TODO: handle case where the whole buffer has less than one record
if( p->tocN==0 && actByteN < kSizeofRecordHeader + recdByteN )
{
assert(0);
}
if( p->bufByteN + recdByteN > actByteN )
{
_cache_backup( p, actByteN, p->bufByteN);
break;
}
p->bufByteN += kSizeofRecordHeader + recdByteN;
}
return kOkRC;
}
void _cache_shuffle_toc( cache_t* p )
{
// for each record address in tocV[]
for(unsigned i=0; i<p->tocN; ++i)
{
// generate a random index into tocV[]
unsigned idx = math::randUInt(0,p->tocN-1);
// swap location i with a random location
unsigned tmp = p->tocV[i];
p->tocV[i] = p->tocV[idx];
p->tocV[idx] = tmp;
}
}
void _cache_fill_toc( cache_t* p )
{
unsigned cacheByteOffs = 0;
for(unsigned i=0; i<p->tocN; ++i)
{
p->tocV[i] = cacheByteOffs;
unsigned recdByteN = *reinterpret_cast<unsigned*>(p->buf + cacheByteOffs);
cacheByteOffs += kSizeofRecordHeader + recdByteN;
}
}
rc_t _cache_fill( cache_t* p )
{
rc_t rc = kOkRC;
unsigned actByteN = 0;
// Note that his function is always called when the file is pointing to the
// record length at the start of a record
// Fill the cache with as much data as possible from the file
if((rc = file::read(p->fH, p->buf, p->bufMaxByteN, &actByteN)) != kOkRC )
{
if(rc == kEofRC)
p->state = kEofState;
else
return cwLogError(rc,"dataset rdr cache fill failed.");
}
// Get a count of the records in the cache (p->tocN) and adjust the file position such that
// it is left pointing to the beginning of the first record after the cache.
if((rc = _cache_count_and_align(p,actByteN)) != kOkRC )
return rc;
// alllocate the TOC
p->tocV = mem::resize<unsigned>(p->tocV,p->tocN);
// Fill the p->tocV[]
_cache_fill_toc(p);
if( p->shuffleFl)
_cache_shuffle_toc(p);
return rc;
}
rc_t _cache_rewind( cache_t* p )
{
rc_t rc;
// rewind the file to the beginning of the
if(( rc = file::seek(p->fH,file::kBeginFl,p->baseFileOffs)) != kOkRC )
{
rc = cwLogError(rc,"rdr cache file seek failed.");
goto errLabel;
}
if((rc = _cache_fill(p)) != kOkRC )
goto errLabel;
p->tocBaseIdx = 0;
p->tocIdx = 0;
errLabel:
return rc;
}
rc_t _cache_advance( cache_t* p )
{
rc_t rc = kOkRC;
unsigned n = p->tocN;
if((rc = _cache_fill(p)) != kOkRC )
goto errLabel;
p->tocBaseIdx += n;
errLabel:
return rc;
}
rc_t cache_setup( cache_t* p, file::handle_t fH, unsigned bufMaxByteN, unsigned baseFileOffs, unsigned totalRecordN, bool shuffleFl )
{
rc_t rc = kOkRC;
p->fH = fH;
p->buf = mem::alloc<uint8_t>( bufMaxByteN );
p->bufMaxByteN = bufMaxByteN;
p->bufByteN = 0;
p->baseFileOffs = baseFileOffs;
p->state = kOkState;
p->totalRecdN = totalRecordN;
p->shuffleFl = shuffleFl;
rc = _cache_rewind(p);
return rc;
}
rc_t cache_close( cache_t* p )
{
mem::release(p->tocV);
mem::release(p->buf);
return kOkRC;
}
rc_t cache_read( cache_t* p, const std::uint8_t*& recdRef, unsigned& recdByteN )
{
rc_t rc = kOkRC;
unsigned tocIdx;
if( p->tocIdx == p->totalRecdN )
{
rc = kEofRC;
p->state = kEofState;
goto errLabel;
}
if( p->tocIdx == p->tocBaseIdx + p->tocN )
if((rc = _cache_advance(p)) != kOkRC )
goto errLabel;
tocIdx = p->tocIdx - p->tocBaseIdx;
recdByteN = *reinterpret_cast<unsigned*>(p->buf + p->tocV[ tocIdx ]);
recdRef = p->buf + (kSizeofRecordHeader + p->tocV[ tocIdx ]);
p->tocIdx += 1;
errLabel:
return rc;
}
rc_t cache_seek( cache_t* p, unsigned recordIdx )
{
rc_t rc = kOkRC;
if( recordIdx >= p->totalRecdN )
return cwLogError(kSeekFailRC,"rdr cache seek index %i greater than last index: %i.",recordIdx,p->totalRecdN-1);
// if the requested record index is inside the cache
if( p->tocBaseIdx <= recordIdx && recordIdx < p->tocBaseIdx + p->tocN )
p->tocIdx = recordIdx;
else
{
// if the requested record index is prior to the cache
if( recordIdx < p->tocBaseIdx )
if((rc = _cache_rewind(p)) != kOkRC )
goto errLabel;
// recordIdx now must be past the beginning of the cache
assert( recordIdx >= p->tocBaseIdx );
// advance the cache until recordIdx is inside of it
while( recordIdx >= p->tocBaseIdx + p->tocN )
{
if((rc = _cache_advance(p)) != kOkRC )
goto errLabel;
}
assert( p->tocBaseIdx <= recordIdx && recordIdx < p->tocBaseIdx + p->tocN );
p->tocIdx = recordIdx;
}
errLabel:
return rc;
}
typedef struct
{
col_t col; // Public record
unsigned* varDimIdxV; // varDimIdxV[] Dimension indexes that are variable in this column.
unsigned varDimIdxN; // Count of values in varDimIdxV[].
} c_t;
typedef struct rdr_str
{
c_t* colA; // colA[ column_count ] Per column data.
unsigned column_count; // Count of elements in colA[].
unsigned record_count; // Count of total examples.
file::handle_t fH; // Backing data file handle.
const std::uint8_t* buf; // buf[ bufMaxByteN ] File read buffer
unsigned bufMaxByteN; // Allocated size of buf[] in bytes. (also sizeof fixed size records)
unsigned bufCurByteN; // Current count of bytes used in buf[].
bool isFixedSizeFl; // True if all fields are fixed size
unsigned flags; // kShuffleFl
unsigned curRecordIdx; // Index of record in buf[].
unsigned nextRecordIdx; // Index of the next record to read.
long baseFileByteOffs; // File byte offset of the first data record
cache_t* cache;
unsigned state; // See k???State enum
} rdr_t;
typedef struct type_str
{
const char* label;
unsigned typeId;
unsigned variantFl;
} type_t;
type_t _typeRefA[] = {
{ "int", kIntRdrFl, variant::kInt32VFl },
{ "float", kFloatRdrFl, variant::kFloatVFl },
{ "double", kDoubleRdrFl, variant::kDoubleVFl },
{ nullptr, 0, 0 }
};
rdr_t* _handleToPtr(handle_t h )
{ return handleToPtr<handle_t,rdr_t>(h); }
const type_t* _typeIdToDesc( unsigned typeId )
{
for(const type_t* t=_typeRefA; t->label!=nullptr; ++t)
if( t->typeId == typeId )
return t;
cwLogError(kInvalidArgRC,"The dataset rdr typeId %i is not valid.", typeId);
return nullptr;
}
const type_t* _varTypeFlagsToDesc( unsigned variantFl )
{
for(const type_t* t=_typeRefA; t->label!=nullptr; ++t)
if( t->variantFl == variantFl )
return t;
return nullptr;
}
const char* _typeIdToLabel( unsigned typeId )
{
const type_t* t;
if((t = _typeIdToDesc(typeId)) == nullptr )
return nullptr;
return t->label;
}
bool _typeIdMatch( unsigned typeId, unsigned variantTypeFl )
{
const type_t* t;
if((t = _typeIdToDesc(typeId)) == nullptr )
return false;
return t->typeId==typeId && t->variantFl==variantTypeFl;
}
const c_t* _colFromId( rdr_t* p, unsigned columnId )
{
for(unsigned i=0; i<p->column_count; ++i)
if( p->colA[i].col.id == columnId )
return p->colA + i;
cwLogError(kInvalidArgRC,"Invalid columnId (%i).", columnId );
return nullptr;
}
const c_t* _colFromLabel( rdr_t* p, const char* colLabel )
{
for(unsigned i=0; i<p->column_count; ++i)
if( textCompare(p->colA[i].col.label, colLabel) == 0 )
return p->colA + i;
cwLogError(kInvalidArgRC,"Invalid column label:%s.", colLabel );
return nullptr;
}
rc_t _destroy( rdr_t* p )
{
for(unsigned i=0; i<p->column_count; ++i)
{
mem::release( p->colA[i].col.dimV );
mem::release( p->colA[i].col.maxDimV );
mem::release( p->colA[i].varDimIdxV);
mem::free( const_cast<char*>(p->colA[i].col.label) );
}
cache_close(p->cache);
mem::release(p->cache);
file::close(p->fH);
mem::release(p->colA);
//mem::free(const_cast<std::uint8_t*>(p->buf));
mem::release(p);
return kOkRC;
}
rc_t _readHdr( rdr_t* p, unsigned cacheByteN, unsigned flags )
{
rc_t rc = kOkRC;
unsigned bufOffsByteN = 0;
p->bufMaxByteN = 0;
p->isFixedSizeFl = true;
if((rc = read(p->fH,p->record_count)) != kOkRC ) goto errLabel;
if((rc = read(p->fH,p->column_count)) != kOkRC ) goto errLabel;
p->colA = mem::allocZ<c_t>( p->column_count);
// for each column
for(unsigned i=0; i<p->column_count; ++i)
{
c_t* c = p->colA + i;
if((rc = readStr( p->fH,(char**)&c->col.label,255)) != kOkRC ) goto errLabel;
if((rc = read(p->fH,c->col.id)) != kOkRC ) goto errLabel;
if((rc = read(p->fH,c->col.varDimN)) != kOkRC ) goto errLabel;
if((rc = read(p->fH,c->col.rankN )) != kOkRC ) goto errLabel;
if((rc = read(p->fH,c->col.maxEleN )) != kOkRC ) goto errLabel;
if((rc = variant::read( p->fH, c->col.max)) != kOkRC ) goto errLabel;
if((rc = variant::read( p->fH, c->col.min )) != kOkRC ) goto errLabel;
c->col.dimV = mem::allocZ<unsigned>( c->col.rankN );
c->col.maxDimV = mem::allocZ<unsigned>( c->col.rankN );
c->varDimIdxV = mem::allocZ<unsigned>( c->col.rankN );
c->varDimIdxN = 0;
c->col.maxEleN = c->col.rankN==0 ? 0 : 1;
for(unsigned j=0; j<c->col.rankN; ++j)
{
if((rc = file::read( p->fH, c->col.dimV[j] )) != kOkRC ) goto errLabel;
if((rc = file::read( p->fH, c->col.maxDimV[j])) != kOkRC ) goto errLabel;
if( c->col.dimV[j] == 0 )
c->varDimIdxV[c->varDimIdxN++] = j;
c->col.maxEleN *= c->col.maxDimV[j];
}
unsigned bytesPerEle = variant::flagsToBytes(c->col.max.flags);
const type_t* t;
if((t = _varTypeFlagsToDesc(c->col.max.flags)) == nullptr )
rc = cwLogError(kInvalidDataTypeRC,"The column %s is not a valid data type (e.g. int, float double).",cwStringNullGuard(c->col.label));
else
c->col.typeId = t->typeId;
// TODO: why maintain both eleN and maxEleN and byteN and maxByteN?
c->col.eleN = c->col.maxEleN;
c->col.maxByteN = bytesPerEle * c->col.maxEleN;
c->col.byteOffset = bufOffsByteN;
c->col.byteN = c->col.maxByteN;
p->bufMaxByteN += c->col.maxByteN + c->varDimIdxN * sizeof(unsigned); // Track the max file buffer size
if( c->col.varDimN != 0 && p->isFixedSizeFl )
p->isFixedSizeFl = false;
bufOffsByteN = p->bufMaxByteN;
}
p->buf = nullptr; //mem::alloc<std::uint8_t>(p->bufMaxByteN);
p->cache = mem::allocZ<cache_t>(1);
// store the file offset to the first data record
if((rc = tell(p->fH,&p->baseFileByteOffs)) != kOkRC )
{
rc = cwLogError(rc,"rdr dataset tell file position failed.");
goto errLabel;
}
rc = cache_setup( p->cache, p->fH, cacheByteN, p->baseFileByteOffs, p->record_count, cwIsFlag(flags,kShuffleFl) );
errLabel:
if( rc != kOkRC )
{
rc = cwLogError(rc,"Data set file header read failed.");
p->state = kErrorState;
}
return rc;
}
// Seek to the a record, but don't actually read it.
rc_t _seek( rdr_t* p, unsigned recdIdx )
{
rc_t rc;
if((rc = cache_seek(p->cache,recdIdx)) != kOkRC )
p->state = p->cache->state;
return rc;
}
rc_t _parse_var_record( rdr_t* p )
{
rc_t rc = kOkRC;
p->bufCurByteN = 0;
for(unsigned i=0; i<p->column_count; ++i)
{
c_t* c = p->colA + i;
// if this is a variabled sized column
if( c->col.varDimN != 0 )
{
const unsigned* varDimV = reinterpret_cast<const unsigned*>(p->buf + p->bufCurByteN );
unsigned eleN = c->col.rankN==0 ? 0 : 1;
// for each dim. of this column
for(unsigned j=0,k=0; j<c->col.rankN; ++j)
{
// if this is a variable sized dimension then set the actual dim. size
if( k<c->varDimIdxN && c->varDimIdxV[k] == j )
{
c->col.dimV[j] = varDimV[k];
k += 1;
p->bufCurByteN += sizeof(varDimV[k]);
}
// calc the count of elements in this field
eleN *= c->col.dimV[j];
}
// set the size and count of elements in this field
c->col.eleN = eleN;
c->col.byteN = variant::flagsToBytes( c->col.max.flags ) * eleN;
}
c->col.byteOffset = p->bufCurByteN;
p->bufCurByteN += c->col.byteN;
}
return rc;
}
rc_t _read_record( rdr_t* p )
{
rc_t rc = kOkRC;
unsigned recordByteN;
if((rc = cache_read( p->cache, p->buf, recordByteN )) != kOkRC )
{
p->state = p->cache->state;
goto errLabel;
}
// if all columns in the record do not have a fixed size then update
// the column pointers into the data record
if( !p->isFixedSizeFl )
if((rc = _parse_var_record( p )) != kOkRC )
goto errLabel;
p->curRecordIdx = p->nextRecordIdx;
p->nextRecordIdx += 1;
errLabel:
return rc;
}
rc_t _get( rdr_t* p, unsigned columnId, const void*& vpRef, unsigned& nRef, const unsigned*& dimVRef, unsigned reqTypeId )
{
const c_t* c;;
if((c = _colFromId(p,columnId)) == nullptr )
return kInvalidArgRC;
if( c->col.typeId != reqTypeId )
return cwLogError(kInvalidArgRC,"Cannot convert the column '%s' from type:%s to type:%s.", _typeIdToLabel(c->col.typeId), _typeIdToLabel(reqTypeId));
nRef = c->col.eleN;
dimVRef = c->col.dimV;
vpRef = p->buf + c->col.byteOffset;
return kOkRC;
}
}
}
}
cw::rc_t cw::dataset::rdr::create( handle_t& h, const char* fn, unsigned cacheBufByteN, unsigned flags )
{
rc_t rc;
if((rc = destroy(h)) != kOkRC )
return rc;
auto p = mem::allocZ<rdr_t>(1);
if((rc = file::open(p->fH, fn,file::kReadFl)) == kOkRC )
if((rc = _readHdr(p,cacheBufByteN,flags)) != kOkRC )
goto errLabel;
p->state = kOkState;
p->curRecordIdx = kInvalidIdx;
p->flags = flags;
h.set(p);
errLabel:
if(rc != kOkRC )
_destroy(p);
return rc;
}
cw::rc_t cw::dataset::rdr::destroy( handle_t& h )
{
rc_t rc = kOkRC;
if( !h.isValid())
return rc;
rdr_t* p = _handleToPtr(h);
if((rc = _destroy(p)) != kOkRC )
return rc;
h.clear();
return rc;
}
unsigned cw::dataset::rdr::column_count( handle_t h )
{
rdr_t* p = _handleToPtr(h);
return p->column_count;
}
const cw::dataset::rdr::col_t* cw::dataset::rdr::column_cfg( handle_t h, unsigned colIdx )
{
rdr_t* p = _handleToPtr(h);
if( colIdx >= p->column_count )
return nullptr;
return &p->colA[colIdx].col;
}
const cw::dataset::rdr::col_t* cw::dataset::rdr::column_cfg( handle_t h, const char* colLabel )
{
rdr_t* p = _handleToPtr(h);
const c_t* c;
if((c = _colFromLabel(p, colLabel )) == nullptr )
return nullptr;
return &c->col;
}
unsigned cw::dataset::rdr::record_count( handle_t h)
{
rdr_t* p = _handleToPtr(h);
return p->record_count;
}
unsigned cw::dataset::rdr::cur_record_index( handle_t h )
{
rdr_t* p = _handleToPtr(h);
return p->curRecordIdx;
}
unsigned cw::dataset::rdr::next_record_index( handle_t h )
{
rdr_t* p = _handleToPtr(h);
return p->nextRecordIdx;
}
unsigned cw::dataset::rdr::state( handle_t h )
{
rdr_t* p = _handleToPtr(h);
return p->state;
}
cw::rc_t cw::dataset::rdr::seek( handle_t h, unsigned recordIdx )
{
rdr_t* p = _handleToPtr(h);
return _seek(p,recordIdx);
}
cw::rc_t cw::dataset::rdr::read( handle_t h, unsigned record_index )
{
rc_t rc = kOkRC;
rdr_t* p = _handleToPtr(h);
if( record_index != kInvalidIdx )
if((rc = _seek(p,record_index)) != kOkRC )
return rc;
return _read_record(p);
}
cw::rc_t cw::dataset::rdr::get( handle_t h, unsigned columnId, const int*& vRef, unsigned& nRef, const unsigned*& dimVRef )
{
rdr_t* p = _handleToPtr(h);
const void* vp = nullptr;
rc_t rc = _get(p, columnId, vp, nRef, dimVRef, kIntRdrFl );
vRef = rc!=kOkRC ? nullptr : static_cast<const int*>(vp);
return rc;
}
cw::rc_t cw::dataset::rdr::get( handle_t h, unsigned columnId, const float*& vRef, unsigned& nRef, const unsigned*& dimVRef )
{
rdr_t* p = _handleToPtr(h);
const void* vp = nullptr;
rc_t rc = _get(p, columnId, vp, nRef, dimVRef, kFloatRdrFl );
vRef = rc!=kOkRC ? nullptr : static_cast<const float*>(vp);
return rc;
}
cw::rc_t cw::dataset::rdr::get( handle_t h, unsigned columnId, const double*& vRef, unsigned& nRef, const unsigned*& dimVRef )
{
rdr_t* p = _handleToPtr(h);
const void* vp = nullptr;
rc_t rc = _get(p, columnId, vp, nRef, dimVRef, kDoubleRdrFl );
vRef = rc!=kOkRC ? nullptr : static_cast<const double*>(vp);
return rc;
}
cw::rc_t cw::dataset::rdr::report( handle_t h )
{
rc_t rc = kOkRC;
rdr_t* p = _handleToPtr(h);
for(unsigned i=0; i<p->column_count; ++i)
{
const c_t* c = p->colA + i;
printf("id:%5i vdN:%5i mxEleN:%5i rank:%3i %8s", c->col.id, c->col.varDimN, c->col.maxEleN, c->col.rankN, _typeIdToLabel(c->col.typeId) );
printf(" min:"); variant::print(c->col.min);
printf(" max:"); variant::print(c->col.max);
printf(" | ");
for(unsigned j=0; j<c->col.rankN; ++j)
printf("%i ",c->col.dimV[j]);
printf(" | ");
for(unsigned j=0; j<c->col.rankN; ++j)
printf("%i ",c->col.maxDimV[j]);
printf("\n");
}
return rc;
}
cw::rc_t cw::dataset::rdr::test( const object_t* cfg )
{
rc_t rc = kOkRC;
char* inFn = nullptr;
unsigned cacheByteN = 128;
handle_t h;
if((rc = cfg->getv("inFn",inFn,"cacheByteN",cacheByteN)) != kOkRC )
return cwLogError(rc,"rdr test failed. Argument parse failed.");
inFn = filesys::expandPath(inFn);
if((rc = create(h,inFn,cacheByteN,kShuffleFl)) != kOkRC )
{
rc = cwLogError(rc,"rdr create failed.");
}
else
{
const int* v = nullptr;
unsigned vN = 0;
const unsigned* dimV = nullptr;
report(h);
while( (rc=read(h)) == kOkRC )
{
get(h,0,v,vN,dimV); vop::print(v,vN,"%i ","c0:");
get(h,1,v,vN,dimV); vop::print(v,vN,"%i ","c1:");
get(h,2,v,vN,dimV); vop::print(v,vN,"%i ","c2:");
get(h,3,v,vN,dimV); vop::print(v,vN,"%i ","c3:");
}
if( rc != kEofRC )
rc = cwLogError(kOpFailRC,"The read operation failed.");
destroy(h);
}
mem::release(inFn);
return rc;
}
//----------------------------------------------------------------------------------------------------------------------------
//----------------------------------------------------------------------------------------------------------------------------
//----------------------------------------------------------------------------------------------------------------------------
namespace cw {
namespace dataset {
namespace adapter {
typedef struct col_str
{
const rdr::col_t* col; // Column description
bool oneHotFl; // Convert this column to a one-hot vector
unsigned maxEleN; // Max count of elements in the buffer from this column
int oneHotMax; // Max value in this column
int oneHotMin; // Min value in this column
unsigned* batchDimV; // batchDivV[ col.rankN, batchN ] or nullptr for fixed size columns
struct col_str* link; //
} col_t;
typedef struct field_str
{
unsigned id; // Field Id
unsigned flags; // Field flags
bool isFixedSizeFl; // Do all columns in this field have a fixed size.
unsigned bytesPerEle; // Size of each element in buf[] (determined by flags | k<DataType>fl)
unsigned bufMaxEleN; // Allocated size of buf[] for a batch size of maxBatchN
unsigned bufEleN; // Current count of elements in buf[] for the entire batch.
unsigned bufMaxFieldByteN; // Max. size in bytes of one field record.
unsigned bufByteN; // Current count of bytes in buf.
std::uint8_t* buf; // buf[ bufMaxFieldByteN*batchN ]
unsigned* batchEleNV; // batchEleN[ maxBatchN ] Count of ele's in each record of a batch.
col_t* colL; // List of columns assigned to this field
colMap_t** colMapM; // colMapM[ batchN ]
colMap_t* colMapA; // colMapA[ batchN*columnN ] Storage for colMapM[]
struct field_str* link; //
} field_t;
typedef struct adapter_str
{
unsigned maxBatchN; // Max. possible value of batchN in a call to read().
unsigned batchN; // Count of records returned in the last call to read().
rdr::handle_t rdrH; // Source data file
field_t* fieldL; // List of field descriptions
unsigned state; // Exception state
} adapter_t;
inline adapter_t* _handleToPtr(handle_t h )
{ return handleToPtr<handle_t,adapter_t>(h); }
rc_t _destroy( adapter_t* p )
{
rc_t rc = kOkRC;
field_t* f = p->fieldL;
while( f != nullptr )
{
field_t* f0 = f->link;
col_t* c = f->colL;
while( c != nullptr )
{
col_t* c0 = c->link;
// if this is a var width column
if( c->col->varDimN > 0 )
mem::release(c->batchDimV);
mem::release(c);
c = c0;
}
mem::release(f->batchEleNV);
mem::release(f->buf);
mem::release(f->colMapM);
mem::release(f->colMapA);
mem::release(f);
f=f0;
}
rdr::destroy(p->rdrH);
mem::release(p);
return rc;
}
field_t* _fieldIdToRecd( adapter_t* p, unsigned fieldId )
{
field_t* f = p->fieldL;
for(; f!=nullptr; f=f->link)
if( f->id == fieldId )
return f;
cwLogError(kInvalidArgRC,"Invalid field id '%i'.",fieldId);
return nullptr;
}
rc_t _calc_one_hot_ele_count( col_t* c, unsigned& eleN_Ref )
{
rc_t rc = kOkRC;
if( !variant::isInt(c->col->min) || !variant::isInt(c->col->max) )
return cwLogError(kInvalidArgRC,"One-hot columns must be integer valued.");
if( c->col->rankN != 1 || c->col->maxDimV[0] != 1 )
return cwLogError(kInvalidArgRC,"One-hot columns must be scalar integers.");
if((rc = variant::get(c->col->min,c->oneHotMin)) != kOkRC )
return cwLogError(rc,"Unable to obtain the one-hot minimum value.");
if((rc = variant::get(c->col->max,c->oneHotMax)) != kOkRC )
return cwLogError(rc,"Unable to obtain the maximum value.");
eleN_Ref = (c->oneHotMax - c->oneHotMin) + 1;
return rc;
}
rc_t _assign_column( adapter_t* p, field_t* f, const char* colLabel, bool oneHotFl )
{
rc_t rc = kOkRC;
col_t* c = mem::allocZ<col_t>(1);
if((c->col = rdr::column_cfg(p->rdrH, colLabel)) == nullptr )
rc = kInvalidArgRC;
else
{
c->oneHotFl = oneHotFl;
// locate the last link in the column list
col_t* c0 = f->colL;
while( c0!=nullptr && c0->link != nullptr )
c0=c0->link;
// add the new record to the end of the list
if( c0 == nullptr )
f->colL = c;
else
c0->link = c;
// if one-hot encoding was requested
if( oneHotFl )
rc = _calc_one_hot_ele_count(c,c->maxEleN);
else
c->maxEleN = c->col->maxEleN;
// update the size of the field buffer to account for the column size
f->bufMaxEleN += c->col->maxEleN;
// if this is a variable length column
if( c->col->varDimN > 0 )
f->isFixedSizeFl = false;
if( cwIsFlag(f->flags,kTrackColDimFl) )
{
// if this is a fixed size column then batchDimV is null
// otherwise it is a [batchN,rankN] matrix used to hold the dim's of each returned data ele from this column
c->batchDimV = c->col->varDimN == 0 ? nullptr : mem::allocZ<unsigned>(p->maxBatchN*c->col->rankN);
}
}
if( rc != kOkRC )
rc = cwLogError(rc,"'%s' Column assignment failed.", cwStringNullGuard(colLabel));
return rc;
}
rc_t _allocate_field_buffer( adapter_t* p, field_t* f )
{
rc_t rc = kOkRC;
f->bufMaxEleN = 0;
// calc the field width as the sum of the max column widths
unsigned colN = 0;
for(col_t* c=f->colL; c!=nullptr; c=c->link)
{
f->bufMaxEleN += c->maxEleN;
colN += 1;
}
f->bufMaxFieldByteN = f->bufMaxEleN * f->bytesPerEle;
f->buf = mem::alloc<std::uint8_t>(p->maxBatchN * f->bufMaxFieldByteN);
// if col. dim tracking is enabled for this field
if( cwIsFlag(f->flags,kTrackColDimFl) )
{
// allocate the column dim tracking data structures
f->colMapM = mem::allocZ<colMap_t*>( p->maxBatchN );
f->colMapA = mem::allocZ<colMap_t>( p->maxBatchN * colN );
// initialize the fixed portion of the col. tracking records
for(unsigned i=0; i<p->maxBatchN; ++i)
{
f->colMapM[i] = f->colMapA + i*colN;
// for batch index i for each column
unsigned j=0, eleOffs=0;
for(col_t* c=f->colL; c!=nullptr; c=c->link,++j)
{
f->colMapM[i][j].colId = c->col->id;
f->colMapM[i][j].rankN = c->col->rankN;
// if this is a fixed size field then the col. map can be completely populated in advance of reading the data
// TODO: don't allocate the complete colMapA[] array because every colN records are duplicates anyway.
// just point colMapM[] to a single row of colMapA[].
if( !f->isFixedSizeFl )
{
f->colMapM[i][j].eleN = c->oneHotFl ? c->maxEleN : c->col->eleN;
f->colMapM[i][j].fieldEleOffset = eleOffs;
f->colMapM[i][j].dimV = c->col->dimV;
eleOffs += c->col->eleN;
}
else
{
f->colMapM[i][j].dimV = c->batchDimV + (i*c->col->rankN);
}
}
}
}
return rc;
}
template< typename S, typename D >
rc_t _translate_one_hot( std::uint8_t* buf, unsigned bufByteN, const S* src, unsigned srcEleN, const col_t* c, unsigned& dstByteNRef )
{
rc_t rc = kOkRC;
dstByteNRef = 0;
unsigned dstEleN = (c->oneHotMax - c->oneHotMin) + 1;
unsigned dstByteN = dstEleN * sizeof(D);
if( dstByteN > bufByteN )
return cwLogError(kBufTooSmallRC,"The field buffer is too small (src:%i > buf:%i) during one-hot conversion.",dstByteN,bufByteN);
if( srcEleN != 1 )
return cwLogError(kInvalidArgRC,"One-hot encoded fields must be scalars. (srcEleN:%i)",srcEleN);
unsigned oneHotIdx = src[0] - c->oneHotMin;
if( oneHotIdx >= dstEleN )
return cwLogError(kInvalidArgRC,"The one-hot index (%i) is out of the one-hot vector size:%i.",oneHotIdx,dstEleN);
memset(buf,0,dstByteN);
D* dst = reinterpret_cast<D*>(buf);
dst[ oneHotIdx ] = 1;
dstByteNRef = dstByteN;
return rc;
}
template< typename S, typename D >
rc_t _translate_datatype( const col_t* c, std::uint8_t* buf, unsigned bufByteN, const S* src, unsigned srcEleN, unsigned& dstByteNRef )
{
if( c->oneHotFl )
return _translate_one_hot<S,D>( buf, bufByteN, src, srcEleN, c, dstByteNRef );
unsigned dstByteN = srcEleN * sizeof(D);
D* dst = reinterpret_cast<D*>(buf);
dstByteNRef = 0;
if( dstByteN > bufByteN )
return cwLogError(kBufTooSmallRC,"The field buffer is too small (src:%i > buf:%i).",dstByteN,bufByteN);
// copy, and translate, the rdr::col into the field->buf[]
for(unsigned i=0; i<srcEleN; ++i)
dst[i] = src[i];
dstByteNRef = dstByteN;
return kOkRC;
}
template< typename T >
rc_t _translate_column_tpl(adapter_t* p, field_t* f, col_t* c, std::uint8_t* buf, unsigned bufN, unsigned& dstByteNRef)
{
rc_t rc = kOkRC;
const T* v = nullptr;
unsigned vN = 0;
const unsigned* dimV = nullptr;
// read the column
if((rc = rdr::get(p->rdrH, c->col->id, v, vN, dimV )) != kOkRC )
return rc;
switch( f->flags & kTypeMask )
{
case kIntFl: rc = _translate_datatype<T,int>( c, buf, bufN, v, vN, dstByteNRef ); break;
case kFloatFl: rc = _translate_datatype<T,float>( c, buf, bufN, v, vN, dstByteNRef ); break;
case kDoubleFl: rc = _translate_datatype<T,double>( c, buf, bufN, v, vN, dstByteNRef ); break;
default:
assert(0);
}
return rc;
}
rc_t _translate_column( adapter_t* p, field_t* f, col_t* c, std::uint8_t* buf, unsigned bufN, unsigned& dstByteNRef )
{
rc_t rc = kOkRC;
switch( c->col->typeId )
{
case rdr::kIntRdrFl: rc = _translate_column_tpl<int>( p,f,c,buf,bufN,dstByteNRef); break;
case rdr::kFloatRdrFl: rc = _translate_column_tpl<float>( p,f,c,buf,bufN,dstByteNRef); break;
case rdr::kDoubleRdrFl: rc = _translate_column_tpl<double>(p,f,c,buf,bufN,dstByteNRef); break;
default:
assert(0);
}
return rc;
}
rc_t _read_field( adapter_t* p, unsigned batchIdx, field_t* f, unsigned& byteNRef )
{
rc_t rc = kOkRC;
byteNRef = 0;
// on the first use the buffer will not yet be allocated
if( f->buf == nullptr )
if((rc = _allocate_field_buffer(p,f)) != kOkRC )
return rc;
unsigned availBufByteN = f->bufMaxFieldByteN;
unsigned fieldBufByteOffs = 0;
// for each column of this field
for(col_t* c=f->colL; c!=nullptr; c=c->link)
{
unsigned colByteN = 0;
// translate each source column into the field buffer
if((rc = _translate_column( p, f, c, f->buf + f->bufByteN + fieldBufByteOffs, availBufByteN, colByteN )) != kOkRC )
return rc;
assert( availBufByteN >= colByteN );
availBufByteN -= colByteN;
fieldBufByteOffs += colByteN;
// if column dim. tracking is enabled and this is a variable with column ...
if( cwIsFlag(f->flags,kTrackColDimFl) && c->col->varDimN>0 )
for( unsigned i=0; i<c->col->rankN; ++i)
c->batchDimV[ batchIdx * c->col->rankN + i] = c->col->dimV[i]; // ... get the dim's of this column
}
byteNRef = fieldBufByteOffs;
return rc;
}
template< typename T >
cw::rc_t _get( handle_t h, unsigned fieldId, const T*& vV, const unsigned*& nV )
{
rc_t rc = kOkRC;
adapter_t* p = _handleToPtr(h);
field_t* f;
if( p->state != kInitState )
return cwLogError(kInvalidStateRC,"get() failed The adapter is in an invalid state (%i != %i).",p->state,kInitState);
if((f = _fieldIdToRecd(p,fieldId)) == nullptr )
return kInvalidArgRC;
if(f->buf == nullptr )
return cwLogError( kInvalidStateRC, "read() must be called begore get().");
vV = reinterpret_cast<const T*>(f->buf);
nV = f->batchEleNV;
return rc;
}
template< typename T >
cw::rc_t _print_field( adapter_t* p, field_t* f, const char* fmt, unsigned batchIdx, const T* v, unsigned vN )
{
rc_t rc = kOkRC;
unsigned i = 0,k = 0;
for(col_t* c=f->colL; c!=nullptr; c=c->link,++i)
{
colMap_t* cm = f->colMapM[batchIdx] + i;
printf("| %s %i : ", c->col->label, cm->eleN );
for(unsigned j=0; j<cm->eleN; ++j)
printf(fmt,v[k++]);
}
return rc;
}
template< typename T >
cw::rc_t _print_field( adapter_t* p, field_t* f, const char* fmt )
{
rc_t rc = kOkRC;
printf("Field:%3i \n",f->id);
for(unsigned i=0,k=0; i<p->batchN; ++i)
{
printf("%i : ",i);
T* v = reinterpret_cast<T*>(f->buf) + k;
unsigned vN = f->batchEleNV[i];
if( cwIsFlag(f->flags,kTrackColDimFl) )
rc = _print_field(p,f,fmt,i,v,vN);
else
for(unsigned j=0; j<vN; ++j)
printf(fmt,v[j]);
k += vN;
printf("\n");
}
return rc;
}
}
}
}
cw::rc_t cw::dataset::adapter::create( handle_t& hRef, const char* fn, unsigned maxBatchN, unsigned cacheByteN, unsigned flags )
{
rc_t rc = kOkRC;
if((rc = destroy(hRef)) != kOkRC )
return rc;
adapter_t* p = mem::allocZ<adapter_t>(1);
if((rc = rdr::create(p->rdrH,fn,cacheByteN,flags)) != kOkRC )
goto errLabel;
p->maxBatchN = maxBatchN;
p->state = kPreInitState;
hRef.set(p);
errLabel:
if( rc != kOkRC )
_destroy(p);
return rc;
}
cw::rc_t cw::dataset::adapter::destroy( handle_t& hRef )
{
rc_t rc = kOkRC;
if( !hRef.isValid() )
return rc;
adapter_t* p = _handleToPtr(hRef);
if((rc = _destroy(p)) != kOkRC )
return rc;
hRef.clear();
return rc;
}
cw::rc_t cw::dataset::adapter::create_field( handle_t h, unsigned fieldId, unsigned flags, const char* colLabel, bool oneHotFl )
{
rc_t rc = kOkRC;
adapter_t* p = _handleToPtr(h);
field_t* f = mem::allocZ<field_t>(1);
unsigned typeFlags = flags & kTypeMask;
f->id = fieldId;
f->flags = flags;
f->link = p->fieldL;
f->batchEleNV = mem::alloc<unsigned>(p->maxBatchN);
p->fieldL = f;
switch( typeFlags )
{
case kIntFl: f->bytesPerEle = sizeof(int); break;
case kFloatFl: f->bytesPerEle = sizeof(float); break;
case kDoubleFl: f->bytesPerEle = sizeof(double); break;
default:
rc = cwLogError(kInvalidArgRC,"The field data type value 0x%x is not valid.", typeFlags );
}
if( colLabel != nullptr )
rc = _assign_column( p, f, colLabel, oneHotFl );
return rc;
}
cw::rc_t cw::dataset::adapter::assign_column( handle_t h, unsigned fieldId, const char* colLabel, bool oneHotFl )
{
adapter_t* p = _handleToPtr(h);
const rdr::col_t* c = nullptr;
field_t* f;
if(( c = rdr::column_cfg(p->rdrH,colLabel)) == nullptr )
return kInvalidArgRC;
if((f = _fieldIdToRecd(p,fieldId)) == nullptr )
return kInvalidArgRC;
return _assign_column( p, f, colLabel, oneHotFl );
}
unsigned cw::dataset::adapter::record_count( handle_t h )
{
adapter_t* p = _handleToPtr(h);
return rdr::record_count(p->rdrH);
}
unsigned cw::dataset::adapter::field_fixed_ele_count( handle_t h, unsigned fieldId )
{
adapter_t* p = _handleToPtr(h);
field_t* f;
if((f = _fieldIdToRecd(p,fieldId)) == nullptr )
return 0;
return f->bufEleN;;
}
cw::rc_t cw::dataset::adapter::read( handle_t h, unsigned batchN, const unsigned* recordIdxV )
{
rc_t rc = kOkRC;
adapter_t* p = _handleToPtr(h);
switch( p->state )
{
case kInitState:
break;
case kPreInitState:
p->state = kInitState;
break;
default:
return cwLogError(kInvalidStateRC,"Invalid adapter state (%i != %i).",p->state,kInitState);
}
if( batchN > p->maxBatchN )
return cwLogError(kInvalidArgRC,"The batch count:%i is greater than the max batch count:%i.",batchN,p->maxBatchN);
p->batchN = 0;
// for each record in this batch
for(unsigned i=0; i<batchN; ++i)
{
// read the data record
if((rc = rdr::read(p->rdrH, recordIdxV==nullptr ? kInvalidIdx : recordIdxV[i] )) != kOkRC )
{
if( rc == kEofRC )
p->state = kEofState;
goto errLabel;
}
// translate each field
for(field_t* f=p->fieldL; f!=nullptr; f=f->link)
{
unsigned fieldByteN = 0;
if( i == 0 )
{
f->bufEleN = 0;
f->bufByteN = 0;
}
// read the field into f->buf[]
if((rc = _read_field(p,i,f,fieldByteN)) != kOkRC )
{
rc = cwLogError(rc,"Field (id:%i) read failed.",f->id);
goto errLabel;
}
assert( fieldByteN % f->bytesPerEle == 0 );
// update the buffer state
unsigned fieldEleN = fieldByteN / f->bytesPerEle;
f->bufEleN += fieldEleN;
f->bufByteN += fieldByteN;
f->batchEleNV[i] = fieldEleN;
}
p->batchN += 1;
}
errLabel:
if( rc != kOkRC )
p->state = kErrorState;
return rc;
}
cw::rc_t cw::dataset::adapter::get( handle_t h, unsigned fieldId, const int*& vV, const unsigned*& nV )
{ return _get<int>(h,fieldId,vV,nV); }
cw::rc_t cw::dataset::adapter::get( handle_t h, unsigned fieldId, const float*& vV, const unsigned*& nV )
{ return _get<float>(h,fieldId,vV,nV); }
cw::rc_t cw::dataset::adapter::get( handle_t h, unsigned fieldId, const double*& vV, const unsigned*& nV )
{ return _get<double>(h,fieldId,vV,nV); }
cw::rc_t cw::dataset::adapter::column_map( handle_t h, unsigned fieldId, colMap_t const * const *& colMapV_Ref )
{
rc_t rc = kOkRC;
adapter_t* p = _handleToPtr(h);
field_t* f;
if( p->state != kInitState )
return cwLogError(kInvalidStateRC,"Invalid adapter state (%i != %i).",p->state,kInitState);
if((f = _fieldIdToRecd(p,fieldId)) == nullptr )
return kInvalidArgRC;
colMapV_Ref = f->colMapM;
return rc;
}
unsigned cw::dataset::adapter::state( handle_t h )
{
adapter_t* p = _handleToPtr(h);
return p->state;
}
cw::rc_t cw::dataset::adapter::print_field( handle_t h, unsigned fieldId, const char* fmt )
{
rc_t rc = kOkRC;
adapter_t* p = _handleToPtr(h);
field_t* f;
if((f = _fieldIdToRecd(p,fieldId)) == nullptr )
return cwLogError(kInvalidArgRC,"Invalid field id (%i).",fieldId);
switch( f->flags & kTypeMask )
{
case kIntFl: rc = _print_field<int>( p, f, fmt==nullptr ? "%i " : fmt ); break;
case kFloatFl: rc = _print_field<float>( p, f, fmt==nullptr ? "%f " : fmt ); break;
case kDoubleFl: rc = _print_field<double>(p, f, fmt==nullptr ? "%f " : fmt ); break;
default:
rc = cwLogError(kInvalidArgRC,"Unknown type flag: 0x%x.",f->flags & kTypeMask);
}
return rc;
}
cw::rc_t cw::dataset::adapter::test( const object_t* cfg )
{
rc_t rc = kOkRC;
char* inFn = nullptr;
unsigned batchN = 0;
unsigned cacheByteN = 128;
unsigned shuffleFl = rdr::kShuffleFl;
handle_t h;
enum {
kField0Id = 0,
kField1Id = 1
};
// read the cfg args
if((rc = cfg->getv("inFn",inFn,"batchN",batchN,"cacheByteN",cacheByteN)) != kOkRC )
return cwLogError(rc,"adapter test failed. Argument parse failed.");
inFn = filesys::expandPath(inFn);
// create the adapter
if((rc = create(h, inFn, batchN, cacheByteN, shuffleFl)) != kOkRC )
{
rc = cwLogError(rc,"Unable to create dataset adapter for '%s'.",inFn);
goto errLabel;
}
else
{
const int* xV = nullptr;
const float* yV = nullptr;
const unsigned* xNV = nullptr;
const unsigned* yNV = nullptr;
unsigned recdIdxV[] = { 2,1,0 };
if((rc = create_field( h, kField0Id, kIntFl | kTrackColDimFl, "col0", true )) != kOkRC )
goto errLabel;
if((rc = create_field( h, kField1Id, kFloatFl | kTrackColDimFl, "col1" )) != kOkRC )
goto errLabel;
if((rc = assign_column( h, kField1Id, "col2" )) != kOkRC )
goto errLabel;
if((rc = assign_column( h, kField1Id, "col3" )) != kOkRC )
goto errLabel;
assert( cwCountOf(recdIdxV) == batchN );
if((rc = read(h, batchN, recdIdxV )) != kOkRC )
goto errLabel;
if((rc = get(h, kField0Id, xV, xNV )) != kOkRC )
goto errLabel;
if((rc = get(h, kField1Id, yV, yNV )) != kOkRC )
goto errLabel;
for(unsigned i=0,n0=0,n1=0; i<batchN; ++i)
{
for(unsigned j=0; j<xNV[i]; ++j)
printf("%i ", xV[n0+j]);
n0 += xNV[i];
printf(": ");
for(unsigned j=0; j<yNV[i]; ++j)
printf("%f ", yV[n1+j] );
n1 = yNV[i];
printf("\n");
}
print_field(h, kField0Id);
print_field(h, kField1Id);
}
errLabel:
destroy(h);
mem::release(inFn);
return rc;
}
//----------------------------------------------------------------------------------------------------------------------------
//----------------------------------------------------------------------------------------------------------------------------
//----------------------------------------------------------------------------------------------------------------------------
namespace cw
{
namespace dataset
{
namespace mnist
{
typedef struct mnist_str
{
char* trainFn;
char* testFn;
char* validFn;
unsigned* data_dimV;
unsigned* label_dimV;
unsigned exampleN;
float* dataM;
unsigned* labelV;
unsigned kPixN;
unsigned curIdx;
} mnist_t;
inline mnist_t* _handleToPtr(handle_t h )
{ return handleToPtr<handle_t,mnist_t>(h); }
rc_t _destroy( mnist_t* p )
{
rc_t rc = kOkRC;
mem::release(p->dataM);
mem::release(p->labelV);
mem::release(p->trainFn);
mem::release(p->validFn);
mem::release(p->testFn);
mem::release(p);
return rc;
}
rc_t _read_file_record_count( const char* fn, unsigned& nRef )
{
rc_t rc;
file::handle_t fH;
// open the file
if((rc = file::open(fH, fn, file::kReadFl | file::kBinaryFl )) != kOkRC )
{
rc = cwLogError(rc,"MNIST file open failed on '%s'.",cwStringNullGuard(fn));
goto errLabel;
}
// read the count of examples
if((rc = read(fH,nRef)) != kOkRC )
{
rc = cwLogError(rc,"Unable to read MNIST example count.");
goto errLabel;
}
// close file
if((rc = file::close(fH)) != kOkRC )
{
rc = cwLogError(rc,"MNIST file close failed on '%s'.",cwStringNullGuard(fn));
goto errLabel;
}
errLabel:
return rc;
}
rc_t _read_file( mnist_t* p, const char* fn, unsigned n, float* dataM, unsigned* labelV )
{
file::handle_t fH;
rc_t rc = kOkRC;
unsigned exampleN = 0;
// open the file
if((rc = file::open(fH, fn, file::kReadFl | file::kBinaryFl )) != kOkRC )
{
rc = cwLogError(rc,"MNIST file open failed on '%s'.",cwStringNullGuard(fn));
goto errLabel;
}
// read the count of examples
if((rc = read(fH,exampleN)) != kOkRC )
{
rc = cwLogError(rc,"Unable to read MNIST example count.");
goto errLabel;
}
assert( exampleN == n );
// read each example
for(unsigned i=0; i<exampleN; ++i)
{
// read the digit image label
if((rc = read(fH, labelV[i])) != kOkRC )
{
rc = cwLogError(rc,"Unable to read MNIST label on example %i.",i);
goto errLabel;
}
// read the image pixels
if((rc = readFloat(fH, dataM + i*p->kPixN, p->kPixN)) != kOkRC )
{
rc = cwLogError(rc,"Unable to read MNIST data vector on example %i.",i);
goto errLabel;
}
}
errLabel:
if( rc != kOkRC)
rc = cwLogError(rc,"Load failed on MNIST file %s.",cwStringNullGuard(fn));
file::close(fH);
return rc;
}
}
}
}
cw::rc_t cw::dataset::mnist::create( handle_t& h, const char* dir )
{
rc_t rc;
mnist_t* p = nullptr;
unsigned trainN = 0;
unsigned validN = 0;
unsigned testN = 0;
if((rc = destroy(h)) != kOkRC )
return rc;
char* inDir = filesys::expandPath(dir);
// allocate the object
p = mem::allocZ<mnist_t>(1);
p->kPixN = 784;
p->trainFn = filesys::makeFn(inDir, "mnist_train", ".bin", NULL );
p->validFn = filesys::makeFn(inDir, "mnist_valid", ".bin", NULL );
p->testFn = filesys::makeFn(inDir, "mnist_test", ".bin", NULL );
mem::release(inDir);
_read_file_record_count( p->trainFn, trainN );
p->exampleN += trainN;
_read_file_record_count( p->validFn, validN );
p->exampleN += validN;
_read_file_record_count( p->testFn, testN );
p->exampleN += testN;
// allocate the data memory
p->dataM = mem::alloc<float>( p->kPixN * p->exampleN );
p->labelV = mem::alloc<unsigned>( p->exampleN );
// read the training data
if((rc = _read_file( p, p->trainFn, trainN, p->dataM, p->labelV )) != kOkRC )
{
rc = cwLogError(rc,"MNIST training set load failed.");
goto errLabel;
}
// read the validation data
if((rc = _read_file( p, p->validFn, validN, p->dataM + p->kPixN*trainN, p->labelV + trainN )) != kOkRC )
{
rc = cwLogError(rc,"MNIST validation set load failed.");
goto errLabel;
}
// read the testing data
if((rc = _read_file( p, p->testFn, testN, p->dataM + p->kPixN*(trainN +validN), p->labelV + (trainN + validN) )) != kOkRC )
{
rc = cwLogError(rc,"MNIST test set load failed.");
goto errLabel;
}
h.set(p);
errLabel:
if( rc != kOkRC )
_destroy(p);
mem::release(inDir);
return rc;
}
cw::rc_t cw::dataset::mnist::destroy( handle_t& h )
{
rc_t rc = kOkRC;
if( !h.isValid())
return rc;
mnist_t* p = _handleToPtr(h);
if((rc = _destroy(p)) != kOkRC )
return rc;
h.clear();
return rc;
}
unsigned cw::dataset::mnist::record_count( handle_t h )
{
mnist_t* p = _handleToPtr(h);
return p->exampleN;
}
cw::rc_t cw::dataset::mnist::seek( handle_t h, unsigned exampleIdx )
{
rc_t rc = kOkRC;
mnist_t* p = _handleToPtr(h);
if( exampleIdx <= p->exampleN )
p->curIdx = exampleIdx;
else
rc = cwLogError(kSeekFailRC,"Illegal seek index. Seek failed.");
return rc;
}
cw::rc_t cw::dataset::mnist::dataM( handle_t h, const float*& dataM_Ref, const unsigned*& labelV_Ref, unsigned exampleN, unsigned& actualExampleN_Ref, unsigned exampleIdx )
{
rc_t rc = kOkRC;
mnist_t* p = _handleToPtr(h);
if( exampleIdx == kInvalidIdx )
exampleIdx = p->curIdx;
if( exampleIdx >= p->exampleN )
return kEofRC;
if( exampleIdx + exampleN > p->exampleN )
exampleN = p->exampleN - exampleIdx;
//memcpy(dataM, p->dataM + exampleIdx * p->kPixN, exampleN * p->kPixN * sizeof(p->dataM[0]) );
//memcpy(labelV, p->labelV + exampleIdx, exampleN * sizeof(p->labelV[0]) );
dataM_Ref = p->dataM + exampleIdx * p->kPixN;
labelV_Ref = p->labelV + exampleIdx;
actualExampleN_Ref = exampleN;
p->curIdx += exampleN;
return rc;
}
cw::rc_t cw::dataset::mnist::write( handle_t h, const char* fn )
{
rc_t rc = kOkRC;
unsigned recdN = record_count(h);
wtr::handle_t wtrH;
if((rc = wtr::create(wtrH,fn)) != kOkRC )
return cwLogError(rc,"Dataset wtr create failed.");
enum { kImagId, kNumbId };
unsigned numbDimV[] = {1};
unsigned imagDimV[] = {kPixelRowN,kPixelColN};
unsigned imagEleN = imagDimV[0]*imagDimV[1];
if((rc = define_columns( wtrH, "numb", kNumbId, cwCountOf(numbDimV), numbDimV )) != kOkRC )
goto errLabel;
if((rc = define_columns( wtrH, "imag", kImagId, cwCountOf(imagDimV), imagDimV )) != kOkRC )
goto errLabel;
printf("recdN: %i\n",recdN);
for(unsigned i=0; i < recdN; )
{
const float* imagM = nullptr;
const unsigned* numbV = nullptr;
unsigned cacheRecdN = std::min(100u,recdN-i);
unsigned actRecdN = 0;
if((rc = dataM(h, imagM, numbV, cacheRecdN, actRecdN, i )) != kOkRC )
{
cwLogError(rc,"Extract image data failed.");
goto errLabel;
}
for(unsigned j=0; j<actRecdN; ++j)
{
// write the digit this imag represents as an 'int'.
if((rc = wtr::write( wtrH, kNumbId, ((int*)numbV) + j, 1 )) != kOkRC )
goto errLabel;
// write the image data as 'floats'
if((rc = wtr::write( wtrH, kImagId, imagM + j*imagEleN, imagEleN )) != kOkRC )
goto errLabel;
if((rc = wtr::write_record( wtrH )) != kOkRC )
goto errLabel;
}
i += actRecdN;
}
errLabel:
if(rc != kOkRC )
cwLogError(rc, "MNIST data file write failed.");
wtr::destroy(wtrH);
return rc;
}
cw::rc_t cw::dataset::mnist::test( const object_t* cfg )
{
handle_t h;
rc_t rc = kOkRC;
char* inDir = nullptr;
char* outHtmlFn = nullptr;
if((rc = cfg->getv("inDir",inDir,"outHtmlFn",outHtmlFn)) != kOkRC )
return cwLogError(rc,"MNIST test failed. Argument parse failed.");
inDir = filesys::expandPath(inDir);
outHtmlFn = filesys::expandPath(outHtmlFn);
if((rc = create(h, inDir )) == kOkRC )
{
svg::handle_t svgH;
if((rc = svg::create(svgH)) != kOkRC )
rc = cwLogError(rc,"SVG Test failed on create.");
else
{
const float* dataM = nullptr;
const unsigned* labelV = nullptr;
unsigned exampleN = 100;
unsigned actualExampleN = 0;
//mnist::seek( h, 10 );
mnist::dataM( h, dataM, labelV, exampleN, actualExampleN );
for(unsigned i=0; i<actualExampleN; ++i)
{
printf("label: %i ", labelV[i] );
svg::offset(svgH, 0, i*30*5 );
svg::image(svgH, dataM + (kPixelRowN*kPixelColN)*i, kPixelRowN, kPixelColN, 5, svg::kInvGrayScaleColorMapId);
}
svg::write(svgH, outHtmlFn, nullptr, svg::kStandAloneFl, 10,10,10,10);
svg::destroy(svgH);
}
rc = destroy(h);
}
mem::release(outHtmlFn);
mem::release(inDir);
return rc;
}
//----------------------------------------------------------------------------------------------------------------------------
//----------------------------------------------------------------------------------------------------------------------------
//----------------------------------------------------------------------------------------------------------------------------
cw::rc_t cw::dataset::test( const object_t* cfg )
{
rc_t rc = kOkRC;
char* inDir = nullptr;
char* dsFn = nullptr;
char* outHtmlFn = nullptr;
mnist::handle_t mniH;
adapter::handle_t adpH;
svg::handle_t svgH;
unsigned batchN = 100;
unsigned cacheByteN = 4096 * 10;
unsigned shuffleFl = rdr::kShuffleFl;
if((rc = cfg->getv("inDir",inDir,"dsFn",dsFn,"outHtmlFn",outHtmlFn,"batchN",batchN,"cacheByteN",cacheByteN)) != kOkRC )
return cwLogError(rc,"MNIST test failed. Argument parse failed.");
inDir = filesys::expandPath(inDir);
dsFn = filesys::expandPath(dsFn);
outHtmlFn = filesys::expandPath(outHtmlFn);
// open the native MNIST object
if((rc = mnist::create(mniH, inDir )) != kOkRC )
{
cwLogError(rc,"Unable to open the native MNIST object.");
goto errLabel;
}
else
{
// write the MNIST data to a dataset file
if((rc = mnist::write(mniH, dsFn)) != kOkRC )
{
cwLogError(rc,"MNIST dataset write failed");
goto errLabel;
}
mnist::destroy(mniH);
}
// open a dataset adapter
if((rc = adapter::create(adpH,dsFn,batchN,cacheByteN,shuffleFl)) != kOkRC )
{
cwLogError(rc,"Dataset reader create failed.");
goto errLabel;
}
else
{
// create an SVG file
if((rc = svg::create(svgH)) != kOkRC )
rc = cwLogError(rc,"SVG writer create failed.");
else
{
enum { kImagId, kNumbId };
// create a field for the image data
if((rc = create_field( adpH, kImagId, adapter::kFloatFl, "imag" )) != kOkRC )
{
cwLogError(rc,"Dataset rdr column define failed.");
goto errLabel;
}
// create a field for the image lable
if((rc = create_field( adpH, kNumbId, adapter::kIntFl, "numb" )) != kOkRC )
{
cwLogError(rc,"Dataset rdr column define failed.");
goto errLabel;
}
for(unsigned j=0,imageN=0; true; ++j )
{
// read a batch of data
if((rc = adapter::read( adpH, batchN)) != kOkRC )
{
if( rc == kEofRC )
cwLogInfo("Done!.");
else
cwLogError(rc,"Batch read failed.");
goto errLabel;
}
else
{
const int* numbV = nullptr;
const unsigned* numbNV = nullptr;
const float* imagV = nullptr;
const unsigned* imagNV = nullptr;
const unsigned kPixelSize = 5;
adapter::get(adpH, kNumbId, numbV, numbNV ); // get the labels
adapter::get(adpH, kImagId, imagV, imagNV ); // get the image data
printf("%3i : ",j);
// print the first 5 images from each batch to an SVG file
for(unsigned i=0; i<5; ++i,++imageN)
{
printf("%i ", numbV[i] );
// offset the image vertically
svg::offset(svgH, 0, imageN*30*kPixelSize );
svg::image(svgH, imagV + (mnist::kPixelRowN*mnist::kPixelColN)*i, mnist::kPixelRowN, mnist::kPixelColN, kPixelSize, svg::kInvGrayScaleColorMapId);
}
printf("\n");
}
}
svg::write(svgH, outHtmlFn, nullptr, svg::kStandAloneFl, 10,10,10,10);
}
}
errLabel:
adapter::destroy(adpH);
svg::destroy(svgH);
mem::release(inDir);
mem::release(dsFn);
mem::release(outHtmlFn);
return rc;
}
Reference in New Issue View Git Blame Copy Permalink