24 #ifndef __FILE_DATA_SOURCE_H__
25 #define __FILE_DATA_SOURCE_H__
28 #include "services/daal_memory.h"
29 #include "data_management/data_source/data_source.h"
30 #include "data_management/data_source/csv_data_source.h"
31 #include "data_management/data/data_dictionary.h"
32 #include "data_management/data/numeric_table.h"
33 #include "data_management/data/homogen_numeric_table.h"
37 namespace data_management
52 template<
typename FeatureManager,
typename SummaryStatisticsType = DAAL_SUMMARY_STATISTICS_TYPE>
53 class FileDataSource :
public CsvDataSource<FeatureManager, SummaryStatisticsType>
56 typedef CsvDataSource<FeatureManager, SummaryStatisticsType> super;
59 using super::_rawLineBuffer;
60 using super::_rawLineBufferLen;
61 using super::_rawLineLength;
74 FileDataSource(
const std::string &fileName,
75 DataSourceIface::NumericTableAllocationFlag doAllocateNumericTable = DataSource::notAllocateNumericTable,
76 DataSourceIface::DictionaryCreationFlag doCreateDictionaryFromContext = DataSource::notDictionaryFromContext,
77 size_t initialMaxRows = 10) :
78 super(doAllocateNumericTable, doCreateDictionaryFromContext, initialMaxRows)
80 _status |= initialize(fileName);
89 FileDataSource(
const std::string &fileName,
90 CsvDataSourceOptions options,
91 size_t initialMaxRows = 10) :
92 super(options, initialMaxRows)
94 _status |= initialize(fileName);
97 virtual ~FileDataSource()
101 daal::services::daal_free( _fileBuffer );
105 services::Status createDictionaryFromContext() DAAL_C11_OVERRIDE
107 services::Status s = super::createDictionaryFromContext();
108 fseek(_file, 0, SEEK_SET);
109 _fileBufferPos = _fileBufferLen;
113 DataSourceIface::DataSourceStatus getStatus() DAAL_C11_OVERRIDE
115 return (iseof() ? DataSourceIface::endOfData : DataSourceIface::readyForLoad);
119 bool iseof() const DAAL_C11_OVERRIDE
121 return (_fileBufferPos == _readedFromFileLen && feof(_file));
124 bool readLine(
char *buffer,
int count,
int& pos)
128 while (pos + 1 < count)
130 if (_fileBufferPos < _readedFromFileLen)
132 if (_fileBuffer[_fileBufferPos] ==
'\0')
136 buffer[pos] = _fileBuffer[_fileBufferPos];
139 if (buffer[pos - 1] ==
'\n')
147 _readedFromFileLen = (int)fread(_fileBuffer, 1, _fileBufferLen, _file);
159 services::Status readLine() DAAL_C11_OVERRIDE
165 if(!readLine(_rawLineBuffer + _rawLineLength, _rawLineBufferLen - _rawLineLength, readLen))
167 return services::Status(services::ErrorOnFileRead);
175 _rawLineLength += readLen;
176 if (_rawLineBuffer[_rawLineLength - 1] ==
'\n' || _rawLineBuffer[_rawLineLength - 1] ==
'\r')
178 while (_rawLineLength > 0 && (_rawLineBuffer[_rawLineLength - 1] ==
'\n' || _rawLineBuffer[_rawLineLength - 1] ==
'\r'))
182 _rawLineBuffer[_rawLineLength] =
'\0';
185 if(!super::enlargeBuffer())
186 return services::Status(services::ErrorMemoryAllocationFailed);
188 return services::Status();
192 services::Status initialize(
const std::string &fileName)
195 _fileName = fileName;
196 _fileBufferLen = (int)INITIAL_FILE_BUFFER_LENGTH;
197 _fileBufferPos = _fileBufferLen;
199 _readedFromFileLen = 0;
200 if (fileName.find(
'\0') != std::string::npos)
202 return services::throwIfPossible(services::ErrorNullByteInjection);
204 #if (defined(_MSC_VER)&&(_MSC_VER >= 1400))
206 error = fopen_s( &_file, fileName.c_str(),
"r" );
207 if (error != 0 || !_file)
209 return services::throwIfPossible(services::ErrorOnFileOpen);
212 _file = fopen( (
char*)(fileName.c_str()),
"r" );
215 return services::throwIfPossible(services::ErrorOnFileOpen);
218 _fileBuffer = (
char *)daal::services::daal_malloc(_fileBufferLen);
223 return services::throwIfPossible(services::ErrorMemoryAllocationFailed);
225 return services::Status();
229 std::string _fileName;
236 int _readedFromFileLen;
239 static const size_t INITIAL_FILE_BUFFER_LENGTH = 1048576;
245 using interface1::FileDataSource;
daal::data_management::interface1::DataSourceIface::NumericTableAllocationFlag
NumericTableAllocationFlag
Specifies whether a Numeric Table is allocated inside of the Data Source object.
Definition: data_source.h:80
daal::services::ErrorOnFileOpen
Definition: error_indexes.h:392
daal::data_management::interface1::CsvDataSource
Specifies methods to access data stored in files.
Definition: csv_data_source.h:100
daal::data_management::interface1::FileDataSource
Specifies methods to access data stored in files.
Definition: file_data_source.h:53
daal::data_management::interface1::CsvDataSource::createDictionaryFromContext
services::Status createDictionaryFromContext() DAAL_C11_OVERRIDE
Definition: csv_data_source.h:327
daal::data_management::interface1::DataSourceIface::notAllocateNumericTable
Definition: data_source.h:82
daal::data_management::interface1::DataSourceIface::notDictionaryFromContext
Definition: data_source.h:72
daal::data_management::interface1::DataSourceIface::readyForLoad
Definition: data_source.h:60
daal::data_management::interface1::FileDataSource::FileDataSource
FileDataSource(const std::string &fileName, CsvDataSourceOptions options, size_t initialMaxRows=10)
Definition: file_data_source.h:89
daal::data_management::interface1::FileDataSource::FileDataSource
FileDataSource(const std::string &fileName, DataSourceIface::NumericTableAllocationFlag doAllocateNumericTable=DataSource::notAllocateNumericTable, DataSourceIface::DictionaryCreationFlag doCreateDictionaryFromContext=DataSource::notDictionaryFromContext, size_t initialMaxRows=10)
Definition: file_data_source.h:74
daal::services::ErrorNullByteInjection
Definition: error_indexes.h:394
daal::data_management::interface1::DataSourceIface::DataSourceStatus
DataSourceStatus
Specifies the status of the Data Source.
Definition: data_source.h:58
daal::data_management::interface1::CsvDataSourceOptions
Options of CSV data source.
Definition: csv_data_source.h:50
daal::data_management::interface1::FileDataSource::getStatus
DataSourceIface::DataSourceStatus getStatus() DAAL_C11_OVERRIDE
Definition: file_data_source.h:113
daal::data_management::interface1::DataSourceIface::endOfData
Definition: data_source.h:62
daal::data_management::interface1::DataSourceIface::doAllocateNumericTable
Definition: data_source.h:83
daal::services::ErrorOnFileRead
Definition: error_indexes.h:393
daal::data_management::interface1::FileDataSource::createDictionaryFromContext
services::Status createDictionaryFromContext() DAAL_C11_OVERRIDE
Definition: file_data_source.h:105
daal::data_management::interface1::DataSourceIface::DictionaryCreationFlag
DictionaryCreationFlag
Specifies whether a Data Dictionary is created from the context of a Data Source. ...
Definition: data_source.h:70
daal::services::daal_malloc
DAAL_EXPORT void * daal_malloc(size_t size, size_t alignment=DAAL_MALLOC_DEFAULT_ALIGNMENT)
daal::services::daal_free
DAAL_EXPORT void daal_free(void *ptr)
daal::services::ErrorMemoryAllocationFailed
Definition: error_indexes.h:150
daal::data_management::interface1::DataSourceTemplate
Implements the abstract DataSourceIface interface.
Definition: data_source.h:464