c++ file IO

[James C. Fuller]

Patrice,
  Would you be willing to share your c++ api File IO routines?
Currently I am using <fstream> and as you said in the other post calling api's will be faster and smaller.

James

[Patrice Terrier]

Generic Open file function

Code Select Expand

long zFOpen (IN wstring sFilName, IN long AccessMode, IN long ShareMode, OUT HANDLE &hFile) {

    long AccessIs, ShareIs, FlagAndAttribute, nRet;
    nRet = 0; hFile = 0;

    AccessMode = min(max(AccessMode, 0), 2);    // Coherce between 0-2
    if (AccessMode == 0) {                      // 0 Open for read only.
       AccessIs = GENERIC_READ; }
    else if (AccessMode == 1) {                 // 1 Open for write only.
       AccessIs = GENERIC_WRITE; }
    else {                                      // 2 Open for read and write.
       AccessIs = GENERIC_READ | GENERIC_WRITE;
    }

    ShareMode = min(max(ShareMode, 1), 4);      // Coherce between 1-4
    if (ShareMode == 1) {                       // 1 Deny read/write access.
       ShareIs = 0; }
    else if (ShareMode == 2) {                  // 2 Deny write access.
       ShareIs =  FILE_SHARE_READ; }
    else if (ShareMode == 3) {                  // 3 Deny read access.
       ShareIs =  FILE_SHARE_WRITE; }
    else {                                      // 4 Deny none (full share mode).
       ShareIs =  FILE_SHARE_READ | FILE_SHARE_WRITE;
    }

    if (hFile == INVALID_HANDLE_VALUE) {
       FlagAndAttribute = FILE_ATTRIBUTE_NORMAL | FILE_FLAG_WRITE_THROUGH; }
    else {
       FlagAndAttribute = FILE_ATTRIBUTE_NORMAL;
    }

    hFile = CreateFile(sFilName.c_str(), AccessIs, ShareIs, NULL, OPEN_ALWAYS, FlagAndAttribute, NULL);

    if (hFile == INVALID_HANDLE_VALUE) {        // -1 Fail to create the file
       nRet = GetLastError();                   // Set the error code
       hFile = 0;                               // Reset handle number
    }
    return nRet;
}


LOF replacement

Code Select Expand

DWORD zFlof (IN HANDLE hFile) {
    DWORD dwFileSize = 0;
    if (hFile) {
       if (GetFileType(hFile) == FILE_TYPE_DISK) {
           dwFileSize = GetFileSize(hFile, NULL);
           if (dwFileSize == INVALID_FILE_SIZE) { dwFileSize = 0; } // Error
       }
    }
    return dwFileSize;
}

PUT replacement

Code Select Expand

long zFPut (IN HANDLE hFile, IN string sBuffer) {
    long nRet = 0;
    if (hFile) {
       DWORD ByttesWritten = 0;
       DWORD LenBuf = (DWORD) (sBuffer.length());
       if (LenBuf) {
          if (WriteFile(hFile, (CHAR*) sBuffer.c_str(), LenBuf, &ByttesWritten, NULL) == 0) {
             nRet = GetLastError();
          }
       }
    }
    return nRet;
}

GET replacement

Code Select Expand

long zFGet (IN HANDLE hFile, OUT string &sBuffer) {
    long nRet = 0;
    if (hFile) {
       DWORD ByttesReaded = 0;
       DWORD LenBuf = (DWORD) (sBuffer.length());
       if (LenBuf) {
          if (ReadFile(hFile, (CHAR*) sBuffer.c_str(), LenBuf, &ByttesReaded, NULL) == 0) {
              nRet = GetLastError();
          }
       }
    }
    return nRet;
}


SEEK replacement

Code Select Expand

long zFSeek (IN HANDLE hFile, IN DWORD PosByte) {
    LARGE_INTEGER li = {0};
    li.LowPart = PosByte;
    long nRet = 0; if (SetFilePointerEx(hFile, li, NULL, FILE_BEGIN) < 0) { nRet = GetLastError(); }
    return nRet;
}

Ethan Winer FGetAt replacement

Code Select Expand

long zFGetAt (IN HANDLE hFile, IN DWORD nPosByte, OUT string &sBuffer) {
    long nErrCode = 0;
    nErrCode = zFSeek(hFile, nPosByte);
    if (nErrCode == 0) { nErrCode = zFGet(hFile, sBuffer); }
    return nErrCode;
}

CLOSE replacement

Code Select Expand

void zFClose (OUT HANDLE &hFile) {
    if (hFile) { CloseHandle(hFile); }
    hFile = NULL;
}

[James C. Fuller]

Patrice,
  Thank you very much.
I did find this on Codeproject but it only works with VC++ and I use MinGW also.
http://www.codeproject.com/Articles/3936/CFile-Replacement-with-Overlapped-I-O-and-User-Def

Hopefully these will also work with MinGW (haven't tested yret)

James

[James C. Fuller]

Patrice,
  I had to add min max macros for MinGW.
I also  found the GET a bit restrictive for wstring use only
I added a zFRead function.
Thanks again
James

Code Select Expand


long zFRead(HANDLE hFile, BYTE* pBuffer, DWORD dwSize){
    DWORD dwRead = 0;
    BOOL nRet = FALSE;
    if(pBuffer) {
        if (ReadFile(hFile,pBuffer,dwSize,&dwRead,NULL) == 0) {
    nRet = GetLastError();
        }
    }
    return nRet;
}



[Patrice Terrier]

Here is a more complete list, including FPutR and FGetR.

Code Select Expand

' Determines the presence of a file.
function zExist (byval sFileName as string) as long
    local hFind as long, fd AS WIN32_FIND_DATA
    if (len(sFileName)) then
        hFind = FindFirstFile(byval STRPTR(sFileName), fd)
        if (hFind <> %INVALID_HANDLE_VALUE) then
            FindClose(hFind)
            function = -1
        end if
    end if
end function

' Generic open file function
function zFOpen (byval sFileName as string, byval nAccessMode as long, byval nShareMode as long, byref hFile as long) as long
    local szName AS ASCIIZ * %MAX_PATH, nAccessIs, nShareIs, nFlagAndAttribute as long
   
    szName = sFileName
    nAccessMode = min&(max&(nAccessMode, 0), 2) ' Coherce between 0-2
    if nAccessMode = 0 then                     ' 0 Open for read only.
       nAccessIs = %GENERIC_READ
    elseif nAccessMode = 1 then                 ' 1 Open for write only.
       nAccessIs = %GENERIC_WRITE
    else                                        ' 2 Open for read and write.
       nAccessIs = %GENERIC_READ or %GENERIC_WRITE
    end if

    nShareMode = min&(max&(nShareMode, 1), 4)  ' Coherce between 1-4
    if (nShareMode = 1) then                    ' 1 Deny read/write access.
       nShareIs = 0
    elseif nShareMode = 2 then                 ' 2 Deny write access.
       nShareIs =  %FILE_SHARE_READ
    elseif nShareMode = 3 then                 ' 3 Deny read access.
       nShareIs =  %FILE_SHARE_WRITE
    else                                        ' 4 Deny none (full share mode).
       nShareIs =  %FILE_SHARE_READ or %FILE_SHARE_WRITE
    end if

    if (hFile = -1) then
       nFlagAndAttribute = %FILE_ATTRIBUTE_NORMAL or %FILE_FLAG_WRITE_THROUGH
    else
       nFlagAndAttribute = %FILE_ATTRIBUTE_NORMAL
    end if

    hFile = CreateFile(szName, nAccessIs, nShareIs, byval %NULL, %OPEN_ALWAYS, nFlagAndAttribute, byval %NULL)

    if (hFile = %INVALID_HANDLE_VALUE) then     ' -1 Fail to create the file
        function = GetLastError()               ' Set the error code
        hFile = 0                               ' Reset handle number
    end if
end function

' Dito CLOSE
SUB zFClose (byval hFile as long)
    if (hFile) then CloseHandle(hFile)
END SUB

' Dito LOF
function zFlof (byval hFile as long) as long
    if (GetFileType(hFile) = %FILE_TYPE_DISK) then
        local nSize as long
        nSize = GetFileSize(hFile, byval %NULL)
        if (nSize > -1) then function = nSize
    end if
end function

' Dito SEEK
function zFSeek (hFile, byval nPosByte as long) as long
    if (SetFilePointer(hFile, nPosByte, byval %NULL, %FILE_BEGIN) < 0) then
        function = GetLastError()
    end if
end function

' Dito GET
function zFGet (byval hFile as long, sBuffer as string) as long
    if (hFile) then
        local nLenBuf, nByttesReaded as dword
        nLenBuf = len(sBuffer)
        if (nLenBuf) then
            if (ReadFile(hFile, byval STRPTR(sBuffer), nLenBuf, nByttesReaded, byval %NULL) = 0) then
                function = GetLastError()
            end if
       end if
    end if
end function

' GET at a specific offset
function zFGetAt (byval hFile as long, byval nPosByte as long, sBuffer as string) as long
    local nErrcode, nPosByte as long
    nErrCode = zFSeek(hFile, nPosByte)
    if (nErrCode = 0) then nErrCode = zFGet(hFile, sBuffer)
    function = nErrCode
end function

' Dito PUT
function zFPut (byval hFile as long, sBuffer as string) as long
    if (hFile) then
        local nLenBuf, nByttesWritten as dword
        nLenBuf = len(sBuffer)
        if (nLenBuf) then
            if (WriteFile(hFile, byval STRPTR(sBuffer), nLenBuf, nByttesWritten, byval %NULL) = 0) then
                function = GetLastError()
            end if
        end if
    end if
end function

function zFPutAt (byval hFile as long, byval nPosByte as long, sBuffer as string) as long
    local nErrcode as long
    nErrCode = zFSeek(hFile, nPosByte)
    if (nErrCode = 0) then nErrCode = zFPut(hFile, sBuffer)
    function = nErrCode
end function

function zFLoc (byval hFile as long) as long
    '// Warning this function returns &hFFFFFFFF (-1) in case of error
    function = CLNG(SetFilePointer(hFile, 0&, byval %NULL, %FILE_CURRENT))
end function

function zFPutR (byval hFile as long, sBuffer as string, byval nRecord as long) as long
    local nErrcode, nLenBuf long
    nLenBuf = len(sBuffer)
    nRecord = nRecord * nLenBuf - nLenBuf
    nErrCode = zFSeek(hFile, nRecord)
    if (nErrCode = 0) then nErrCode = zFPut(hFile, sBuffer)
    function = nErrCode
end function

function zFGetR (byval hFile as long, sBuffer as sring, byval Record as long) as long
    local nErrcode, nLenBuf long
    nLenBuf = len(sBuffer)
    nRecord = nRecord * nLenBuf - nLenBuf
    nErrCode = zFSeek(hFile, nRecord)
    if (nErrCode = 0) then nErrCode = zFGet(hFile, sBuffer)
    function = nErrCode
end function

function zFlush (byval hFile as long) as long
    if (hFile) then
       if (FlushFileBuffers(hFile) = 0) then function = GetLastError()
    end if
end function

function zSetEof (byval hFile as long) as long
    if (SetEndOfFile(hFile) = 0) then function = GetLastError()
end function

function zSetEofAt (byval hFile as long, byval nPosByte as long) as long
    local nErrCode as long
    nErrCode = zFSeek(hFile, nPosByte)
    if (nErrCode = 0) then ErrCode = zSetEof(hFile)
    function = nErrCode
end function

'function zFSetSize (byval sFileName as string, byval nPosByte as long) as long
'    local hFile, nErrCode as long
'    nErrCode = zFOpen(sFileName, 1, 1, hFile)
'    if (nErrCode = 0) then ErrCode = zSetEofAt(hFile, nPosByte)
'    zFClose(hFile)
'    function = nErrCode
'end function

function zFLock (byval hFile as long, byval nOffSet as long, byval nLength as long) as long
    if (LockFile(hFile, nOffSet, byval %NULL, nLength, byval %NULL) = 0) then
       function = GetLastError()
    end if
end function

function zFUnLock (byval hFile as long, byval nOffSet as long, byval nLength as long) as long
    if (UnLockFile(hFile, nOffSet, byval %NULL, nLength, byval %NULL) = 0) then
       function = GetLastError()
    end if
end function


And here is the C++ min max macro, as well as a few others:
#define min(a,b)  (((a) < (b)) ? (a) : (b))
#define max(a,b) (((a) > (b)) ? (a) : (b))
#define LOINT(a) ((SHORT)(a))
#define HIINT(a) ((SHORT)(((DWORD)(a) >> 16) & 0xFFFF))
#define LOWORD(l) ((WORD)(((DWORD_PTR)(l)) & 0xffff))
#define HIWORD(l) ((WORD)((((DWORD_PTR)(l)) >> 16) & 0xffff))
#define LOBYTE(w) ((BYTE)(((DWORD_PTR)(w)) & 0xff))
#define HIBYTE(w) ((BYTE)((((DWORD_PTR)(w)) >> 8 ) & 0xff))
#define MAKLNG(a, b) ((LONG) (((WORD) (a)) | ((DWORD) ((WORD) (b))) << 16))
#define UBOUND(T) ((LONG) T.size())

[James C. Fuller]

Patrice,
  Thanks again for posting your code.
It's a bit too restrictive for my use as it appears to use strings for all the buffers.

I still have not found any (non-STL) code other than mfc,.net, or cstdio to replicate this very simple line input routine.

James

Code Select Expand


#include <iostream>
#include <fstream>
#include <string>

using namespace std;

int main ()
{
    std::string  sLine;
    std::ifstream  f("Test.txt");
    if(!f )
    {
        return 0;
    }
    while(getline(f, sLine))
    {
        cout << sLine << endl;
    }
    f.close();
}


[Patrice Terrier]

It is just a matter to replace string with char* or WCHAR*.

I did post already a replacement for LineInput, let me find the correct link for you.

...

[Patrice Terrier]

Here is the link:
Bufin (fast line Input replacement, based on direct call to I/O API)
This one is modeled on Ethan Winer code, from his book "BASIC Techniques and Utilities".

[James C. Fuller]

Patrice,
  Your last two posts (code and link) were PowerBASIC not C++.
  I am searching for a non -> STL,mfc,.net, or cstdio  C++ solution for my little snippet if it exists.
  Also in your first posts (c++) here you get the length from the wstring so it's not just a simple replacement.
  I do appreciate your insight.
James

[Patrice Terrier]

Then use WCHAR rather than wstring, and use wcslen to get the length of the WCHAR (UNICODE).

[Frederick J. Harris]

I am searching for a non -> STL,mfc,.net, or cstdio  C++ solution for my little snippet if it exists.

I don't quite understand that line James.  Does 'non' apply to cstdio, or just to STL, mfc, and .net.  I always use fgets or fgetws for that sort of thing, which I do fairly frequently.  I suppose there are MS specific techniques for doing it, but I don't see the advantages of that.  Here is a link...

http://www.cplusplus.com/reference/cstdio/fgets/

[James C. Fuller]

Fred,
  A very enlightening topic discussion.

http://forums.codeguru.com/showthread.php?547711-pros-cons-on-File-I-O-coding

James

[Patrice Terrier]

James--

For me there is no discussion low level API produces the smallest fastest code.

Did you check my Bufin function that is a replacement for GetLine, and is able to work with any kind of delimiter (UNIX or DOS)

Here is the matching c++ version.

Code Select Expand

string skBufin (IN WCHAR* szFilName, OUT long &nDone) {
    string sResult = "";
    static string sBuffer;
    static long nCR, nReading, nLenCr, nSaveCr;
    static DWORD nBufSize, nBufPos, nRemaining, nSizeFile, nLocation;
    static HANDLE hFile;
    long nErrCode, nSlop;

if (wcslen(szFilName)) {
   if (!nReading) {
      nReading = -1; nDone = 0;
      nLenCr = 0; nCR = 0; nLocation = 0;
      if (!FileExist(szFilName)) { goto BufEnd; }         // Nothing to do
      nErrCode = zFOpen(szFilName, 0, 2, hFile);
      if (nErrCode) {
          nDone = nErrCode; nReading = 0;
          return sResult;
      }
      nRemaining = zFlof(hFile);                        // Bytes count to be read
      nSizeFile = nRemaining;                           // Remember this
      if (nRemaining == 0) { goto BufEnd ; }            // Nothing to read
      nBufSize = 16384;
      sBuffer.assign(nBufSize, 32);                     // dito SPACE$.
   }
   while (nRemaining) {                                 // While more in the file
      if (nCR == 0) {                                   // if (no return was found
         if (nRemaining < nBufSize) {                   // read only what remains.
            nBufSize = nRemaining;                      // Resize the buffer.
            if (nBufSize < 1) { break; }                // Possible only if (EOF 26.
            sBuffer.assign(nBufSize, 32);               // dito SPACE$.
         }
         nErrCode = zFGetAt(hFile, nLocation, sBuffer); // Read a bloc.
         nLocation += nBufSize;                         // Here we are.
         nBufPos = 1;                                   // Start at the beginning
      }                                                 // of that bloc.
      do {                                              // Walk through buffer.
         nCR = instr(nBufPos, sBuffer, $cr);            // Look for a Return.
         if (nLenCr == 0) {                             // Do this to read files with
             nLenCr = 1;                                // or without LF after CR.
             if (asc(sBuffer, nCR + 1) == 10) { nLenCr = 2; }
         }
         if (nCR) {                                     // We found one.
             nSaveCr = nCR;                             // Save where
             sResult = mid$(sBuffer, nBufPos, nCR - nBufPos);
             nBufPos = nCR + nLenCr;                    // In case of LF skip it
             return sResult; }                          // all done for now.
         else {                                         // Back up in the file.
             // if we reached the end of file and no nCR
             // was found, return what remains in the string.
             if (nLocation >= nSizeFile) {
                 // Assign function and trap ^Z
                 sResult = rtrim$(mid$(sBuffer, nSaveCr + nLenCr), chr$(26));
                 nRemaining = nBufSize;
                 break;
             }
             nSlop = nBufSize - nSaveCr - (nLenCr - 1); // Calc buffer excess.
             nRemaining += nSlop;                       // Calc file excess.
             nLocation -= nSlop;
         }
      } while (nCR);                                    // While more in buffer.
      nRemaining -= nBufSize;
      nSaveCr = 0;
   }
}

BufEnd:
    nReading = 0; nDone = -1; zFClose(hFile);
    return sResult;
}


[James C. Fuller]

Patrice,
I know your focus is not text file line input but that was the purpose of my codeguru post.
Show me some numbers and code using the attached text file.
I still think for My c++ learning it is best to stick with c++ STL code.
I know you and Fred both dance to your own drummers but I prefer ease of use at this stage in my life.
I have been the bare bones route starting with the 8 bit 6502 and using asm code during horizontal and vertical interrupts of the crt.

Take the challenge.
Attached is a my million.txt file and 2kaud's examples

James

These are 2kaud's examples from codeguru
Example 1:

Code Select Expand


#include <cstdio>
#include <cstring>
#include <Windows.h>
#include <vector>
using namespace std;

int main()
{
FILE *pFile;
char s[500] = { 0 };
vector<string> vs(10000000);

    pFile = fopen("million.txt", "r");
    if (!pFile) {
        puts("cannot open file");
        return 1;
    }

unsigned int cnt = 0;
int ln = 0;
DWORD tim = GetTickCount();

    while (fgets(s, 500, pFile)) {
        if (s[(ln = strlen(s) - 1)] == '\n') {
            s[ln] = 0;
            ++cnt;
            vs.push_back(s);
        }
    }

    printf("Read %i. clib took %i\n", cnt, GetTickCount() - tim);
}


Example 2:

Code Select Expand


#include <iostream>
#include <fstream>
#include <string>
#include <vector>
#include <Windows.h>
using namespace std;

int main()
{
vector<string> vs(10000000);
string  sLine;
ifstream  f("million.txt");

    if (!f) {
        cout << "Cannot open file" << endl;
        return 1;
    }

unsigned int cnt = 0;
DWORD tim = GetTickCount();

    while (getline(f, sLine)) {
        ++cnt;
        vs.push_back(sLine);
    }

    cout << "Read " << cnt << ". STL string took " << GetTickCount() - tim << endl;
}


Example 3:

Code Select Expand


#include <iostream>
#include <fstream>
#include <vector>
#include <Windows.h>
using namespace std;

int main()
{
vector<string> vs(10000000);
char   sLine[500] = { 0 };
ifstream  f("million.txt");

    if (!f) {
        cout << "Cannot open file" << endl;
        return 1;
    }

unsigned int cnt = 0;
DWORD tim = GetTickCount();

    while (f.getline(sLine, 500)) {
        ++cnt;
        vs.push_back(sLine);
    }

    cout << "Read " << cnt << ". STL char took " << GetTickCount() - tim << endl;
}


[Frederick J. Harris]

Hi James,

     I read through your link from CodeGuru and found it interesting.  Some real knowledgible helpful folks there.  Right at this moment I don't have time to spend on it though because I'm working on another of your suggestions, and that is converting the DDT Address sample to RegisterClassEx()/CreateWindowEx() type SDK coding.  I believe you were going to do it with straight resource script dialog code.  How are you coming with that?  I never messed with printer code, and that address sample does that, so I've got to pull out the Petzold now and learn that too, I guess.

     From looking at your original timings on the text line reads, it looked like stdio gets() was going quicker than iostream stuff.  But like they pointed out there, the iostream library is doing a lot more and doing it in conjunction with the Std. Lib's String Class.

     I found it interesting that you kept directly asking in that thread if use of stdio was somehow discouraged or in poor form among C++ coders.  And I noted that no one directly answered your question.  In my opinion there's nothing wrong with using anything in the C Standard Library in C++.  In fact, some of the posters there even pointed out that iostream was implemented using stdio. 

     I've never been persecuted in any of my posts in C++ forums for using stdio.  I regularly post only at www.cplusplus.com though, and the Windows forum there is mostly beginner stuff.  But that's OK with me because I like to help beginners.  I used to post a lot at daniweb but kind of gave up there.  Likewise at cprogramming.com.  I tried hard to join CodeGuru.com but my registration got messed up and I'm permanently locked out of there and couldn't get anyone to straighten it out for me. 

     But if some C++ coder somewhere gave you the line that C is defunct in some way and is in poor form to use I'd relegate such a comment to the same trash heap I'd put statements by SDK'ers that DDT is useless and shouldn't be used, or the contrary by DDT'ers that SDK was against Bob's plan and shouldn't be used by PowerBASIC users because he created DDT for visual interfaces.

     Afterall, C is upfront included in C++; that was one of the language's main initial design considerations.  C++ is the superset which includes C as a subset.  I think that's factual information.

     My reason for preferring C isms over C++ isms for file i/o relate to the fact that philosophically, I'm a 'minimalist'.  I try to reduce everything to its simplest possible form, which still retains the capacity to perform the desired task.  Now take iostream.  That library does one heck of a lot of complicated stuff.  Take serialization.  It has the capacity to take a complex series of complex classes involving deep inheritance hierarchys and serialize them to a disk file.  And I imagine return them back to memory.  That can be pretty complicated.  And its why outputing "Hello, World!" to the screen using iostream takes 500K or 1 mb whereas stdio can do it in 5k.  So just in terms of granularity of code I'd prefer the more limited approach of stdio.  Just my thoughts!