file.h

Go to the documentation of this file.

/**********************************************************//**
 **
 ** @file util/file.h
 **
 ** Copyright (C) 2016  Xpace, LLC.  All rights reserved
 **
 ** www.xpace.net
 **
 **************************************************************/


#if !defined(XPACE_FILE_H)
#define XPACE_FILE_H

#if !defined NOMINMAX
#define NOMINMAX
#endif

#if _WIN32 || _WIN64
// for HANDLE
# include <windows.h>
# pragma warning(push)
# pragma warning(disable : 4251)
#else
# include <cstdio>
#endif
#include <algorithm>
#include <cstring>
#include <cassert>

#include "base/types.h"
#include "base/exception.h"

namespace Xpace
{
  class XPACE_EXPORT File
  {
  public:
    typedef uint64 Position;
    typedef int64  Distance;
    static const Position errorPosition; 

    /// constructor - Create an empty file
    File
      ();

    /// creates an independent copy
    /// rhs must be read-only; result is read-only
    /// @rhs copy this
    File
      (const File& rhs);

    /// assign an independent copy
    /// rhs must be read-only; result is read-only
    /// @rhs copy this
    File& operator=
      (const File& rhs);

    /// thrown if either of the above two fails
    /// e.g., if rhs is not read-only
    class Cant_Copy : public Exception
    {
    public:
      Cant_Copy
        (const File& f) :
          Exception("Can't copy file \"%1\".")
      {
        addParam(f.getName());
      }
    };

    /// destructor - closes if open
    virtual ~File
      ();

    enum OpenMode
    {
      ReadSequential,     ///< open existing for (primarily) sequential reading
      ReadRandom,         ///< open existing for (primarily) random reading
      WriteSequential,    ///< open existing or new for (primarily) sequential writing
      WriteSequentialNew, ///< create for (primarily) sequential writing
      WriteRandom,        ///< open existing or new for (primarily) random writing
      WriteRandomNew,     ///< create for (primarily) random writing
    };

    enum Type
    {
      Raw,                ///< an untyped file from an external source  
      Any,                ///< match any type (except raw)
      DataStore,          
      DataRefs,    
      DataMap,     
      IndexList,   
      IndexMap    
    };

    /// @param name full path name
    /// @param mode file open mode
    /// @param type the file's type; an attempt to open an existing file of the wrong type will fail
    /// @param bufSize: if 0, use defaults
    /// @return true iff successful
    virtual bool open
      (const String name,
       OpenMode mode,
       Type type = Any,
       size_t bufSize = 0);

    /// Open a special file, which is not
    /// seekable, such as a pipe
    /// @param handle OS handle
    /// @param bufSize: if 0, use defaults
    virtual void fromHandle
#if _WIN32
      (HANDLE handle,
#else
      (FILE* handle,
#endif
       size_t bufSize = 0);

    /// @return true iff the file is not open
    virtual bool operator!
      ()
      const;

    /// @return the file's name 
    virtual String getName
      ()
      const;

//    /// @return the file's Type
//    virtual Type getType
//      ()
//      const;

    /// @return the file's internal buffer size
    virtual size_t getBufSize
      ()
      const;

    /// Make sure all changes are written
    virtual bool flush
      ();

    /// Close the file, flushing if necessary
    virtual bool close
      ();

    /// delete the file, if it exists
    /// file does not have to be open
    /// @return true iff file existed and successfully deleted
    virtual bool remove
      ();

    /// Read from the file
    /// @param dest destination buffer
    /// @param bytes number of bytes requested
    /// @return actual bytes read; read will stop at end-of-file
    size_t read
      (void* dest,
       size_t bytes);

    /// Read a T from the file
    /// @param dest fillin destination T
    /// @return true iff successful
    template <typename T>
    bool read
      (T* dest);

    /// Read a T from the file
    /// @return the T
    /// @throw File_Cant_Read iff unsuccessful
    template <typename T>
    T read
      ();

    /// Read thehe remainder of the as a 16-bit char string
    /// @param raw read remainder of file as one string, 
    ///   otherwise read string written with writeString
    /// @param maxLen read no more than this many chars
    /// @return read String
    String readString
      (bool raw = false,
       size_t maxLen = 0x100000);

    /// Read the remainder of the file as an 8-bit char string
    /// @param maxLen read no more than this many chars
    /// @return read String
    String readString8
      (size_t maxLen = 0x100000);

    /// Write some data to the file
    /// @param src source buffer
    /// @param bytes number of bytes requested
    /// @return number of bytes written
    size_t write
      (const void* src,
       size_t bytes);

    /// Write a T to the file
    /// @param src source T
    /// @return true if successful
    template <typename T>
    bool write
      (const T& src);

    /// Write a String to the file
    /// @param str String to be written
    /// @param raw if false, write for readString(), if true just write chars
    /// @return true if successful
    bool writeString
      (const String& str,
       bool raw = false);

    /// Get by direct access to the file buffer 
    ///   not guaranteed to be valid after successive calls
    /// @param dest ptr to data file buffer
    /// @param count number of Ts requested
    /// @return actual Ts available
    template <typename T>
    size_t get
      (T** dest,
       size_t bytes);

    /// Get bytes until a delimiter
    /// @param dest ptr to data file buffer
    /// @param delim the delimiter
    /// @return bytes between dest and the delimiter
    template <typename T>
    size_t getUntil
      (T** dest,
       T delim);

    /// Get a T by direct access to the file buffer
    ///   return not guaranteed to be valid after successive calls
    /// @return ptr to T on success, 0 if fail
    template <typename T>
    T* get
      ();
    
    /// Get file length
    /// @return offset of first byte past end-of-file
    virtual Position getLength
      ()
      const;

    /// Get length of a file by name
    /// @param name full path name
    /// @return filelength; or positionError if file doesn't exist
    virtual Position getLength
      (const String name);
    
    /// Get current position
    /// @return current offset in file; -1 if file not open
    virtual Position getPos
      ()
      const;

    /// Move current position
    /// @param pos seek to this position
    /// @return true if successful
    virtual bool seek
      (Position pos);

    /// Move current position
    /// @param dist move this amount
    /// @return true if successful
    bool seekRel
      (Distance dist);

    /// @return how much can be read before a physical read
    size_t bufRemains
      ()
    const;

    /// copy from this file to another
    /// @param targ copy to this file
    /// @param size copy this many bytes
    /// @return true if successful
    bool copy
      (File* targ,
       File::Position size);

    // describes a piece of a file
    struct Segment
    {
      Segment(File* file, 
              Position offset, 
              Position size) :
        file(file),
        offset(offset),
        size(size)
      {
      }
      File* file;        
      const Position offset;    ///< start offset of this ref list
      const Position size;      ///< length of this ref list
    };

  protected:
    // defaults
    enum { SEQ_BUF_SIZE = 65536, RAND_BUF_SIZE = 16384 };

    String name;
    OpenMode mode;

    Type   type;             ///< the file's type
    bool   writable;         ///< opened for write
    bool   bufDirty;         ///< is buffer modified
    bool   seekable;         ///< is not a special file
    uint64 bufOffset;        ///< offset of buffer in file
    size_t bufSize;          ///< actual buffer size
    byte*  buffer;           ///< buffer
    byte*  current;          ///< current position in buffer
    byte*  end;              ///< end of data in buffer

    virtual bool seekFile
      (uint64);
    virtual size_t fillBuffer
      ();

    // low-level implementations
    // called only when buffer exhausted
    virtual size_t fileRead
      (byte* dest,
       size_t bytes);
    virtual size_t fileWrite
      (const byte* src,
       size_t bytes);

  #if _WIN32 || _WIN64
    HANDLE hd;
  #else
    FILE* hd;
  #endif

    enum { OS_BUF_SIZE = 4096 };

  private:
    std::vector<byte> g_buffer; ///< buffer for put()/get() 
    std::vector<byte> temp;
  };

  class File_Cant_Open : public Exception
  {
  public :
    File_Cant_Open
      (String fname,
       File::OpenMode mode) :
        Exception("Can't open file \"%1\" for %2.")
    {
      addParam(fname);
      switch (mode)
      {
      case File::ReadSequential : 
        addParam("sequential read");
        break;
      case File::ReadRandom :
        addParam("random read");
        break;
      case File::WriteSequential :
        addParam("sequential write");
        break;
      case File::WriteSequentialNew :
        addParam("sequential write new)");
        break;
      case File::WriteRandom :
        addParam("random write");
        break;
      case File::WriteRandomNew :
        addParam("random write (new)");
        break;
      }
    }
  };

  class File_Cant_Read : public Exception
  {
  public :
    File_Cant_Read
      (const File& f) :
        Exception("Can't read file \"%1\" at position \"%2\".")
    {
      addParam(f.getName());
      addParam(String().setNum(f.getPos()));
    }
  };

  class File_Cant_Write : public Exception
  {
  public :
    File_Cant_Write
      (const File& f) :
        Exception("Can't write file \"%1\" at position \"%2\".")
    {
      addParam(f.getName());
      addParam(String().setNum(f.getPos()));
    }
  };

  class File_Corrupt : public Exception
  {
  public :
    File_Corrupt
      (const File& f) :
        Exception("File \"%1\" corrupt at position \"%2\".")
    {
      addParam(f.getName());
      addParam(String().setNum(f.getPos()));
    }
  };

  class File_Cant_Close : public Exception
  {
  public :
    File_Cant_Close
      (const File& f) :
        Exception("Can't close file \"%1\".")
    {
      addParam(f.getName());
    }
  };

  // ============================================================ 
  // ============================================================ 
  // ============================================================

  inline 
  bool File::operator!
    ()
    const
  {
  #if defined _WIN32
    return (hd == INVALID_HANDLE_VALUE);
  #else
    return (hd == 0);
  #endif
  }

  inline
  String File::getName
    ()
    const
  {
    return name;
  }

  inline 
  size_t File::getBufSize
    ()
    const
  {
    return bufSize;
  }

  inline 
  size_t File::read
    (void* dest,
     size_t bytes)
  {
    size_t bytes_to_read = std::min(static_cast<size_t>(end - current), bytes);
    memcpy(dest, current, bytes_to_read);
    if (bytes_to_read == bytes)
    {
      current += bytes;
      return bytes;
    }
    return bytes_to_read
           + fileRead(reinterpret_cast<byte*>(dest) + bytes_to_read,
                      bytes - bytes_to_read);
  }

  template <typename T>
  inline 
  bool File::read
    (T* dest)
  {
    return (read(reinterpret_cast<byte*>(dest), sizeof(T)) == sizeof(T));
  }

  template <typename T>
  inline 
  T File::read
    ()
  {
    T t;
    if (!read(&t))
      throw File_Cant_Read(*this);
    return t;
  }

  template <typename T>
  inline 
  size_t File::get
    (T** dest,
     size_t count)
  {
    size_t bytes(count * sizeof(T));
    size_t bytes_to_get = std::min(static_cast<size_t>(end - current), bytes);
    if (bytes_to_get == bytes)
    {
      *dest = reinterpret_cast<T*>(current);
      current += bytes;
      return count;
    }

    if ((current == end) && (bytes <= bufSize))
    {
      if (end != buffer)
        bufOffset += bufSize;
      *dest = reinterpret_cast<T*>(buffer);
      size_t ret(std::min(bytes, fillBuffer()));
      current = buffer + ret;
      return ret / sizeof(T);
    }

    if (bytes > g_buffer.size()) 
      g_buffer.resize(bytes);

    byte* d;
    bytes = read(d = &g_buffer[0], bytes);
    *dest = reinterpret_cast<T*>(d);
    return bytes / sizeof(T);
  }

  // @todo: worry about Ts that straddle buffers
  template <typename T>
  inline 
  size_t File::getUntil
    (T** dest,
     T delim)
  {
    byte* start(current);
    T* t(reinterpret_cast<T*>(current));
    
    temp.clear();

    while (1)
    {
      if (t >= reinterpret_cast<T*>(end))
      {
        uint copy_size(end - start);
        if (copy_size)
        {
          temp.resize(copy_size);
          memcpy(&temp[0], start, copy_size);
        }
        else
          start = buffer;

        bufOffset += end - buffer;
        if (!fillBuffer())
        {
          *dest = 0;
          return 0;
        }
        t = reinterpret_cast<T*>(buffer);
      }

      if (*t == delim)
        break;
      ++t;
    }

    current = reinterpret_cast<byte*>(t);

    size_t ret;
    if (!temp.empty())
    {
      ret = temp.size() + current - buffer;
      if (ret > g_buffer.size())
        g_buffer.resize(ret);
      memcpy(&g_buffer[0], &temp[0], temp.size());
      if (current - buffer)
        memcpy(&g_buffer[temp.size()], buffer, current - buffer);
      *dest = reinterpret_cast<T*>(&g_buffer[0]);
    }
    else
    {
      *dest = reinterpret_cast<T*>(start);
      ret = current - start;
    }

    // advance past delim
    current += sizeof(T);

    return ret / sizeof(T);
  }

  template <typename T>
  inline 
  T* File::get
    ()
  {
    T* t;
    return (get(reinterpret_cast<byte**>(&t), sizeof(T)) == sizeof(T))
      ? t : 0;
  }

  inline 
  size_t File::write
    (const void* src,
     size_t bytes)
  {
    if (!writable)
      return 0;

    size_t bytes_to_copy = std::min(bufSize - static_cast<size_t>(current - buffer), bytes);
    memcpy(current, src, bytes_to_copy);
    bufDirty = true;

    if (bytes_to_copy == bytes)
      current += bytes;
    else
      bytes = bytes_to_copy
             + fileWrite(reinterpret_cast<const byte*>(src) + bytes_to_copy,
                        bytes - bytes_to_copy);

    end = std::max(end, current);

    return bytes;
  }

  template <typename T>
  inline 
  bool File::write
    (const T& src)
  {
    return (write(&src, sizeof(T)) == sizeof(T));
  }

  inline 
  bool File::seekRel
    (Distance dist)
  {
    if (dist < 0) 
    {
      if (static_cast<int>(buffer - current) <= dist)
      {
        current += dist;
        return true;
      }
    }
    else
    {
      if (end - current >= dist)
      {
        current += dist;
        return true;
      }
    }
    return seek(getPos() + dist);
  }

  inline
  size_t File::bufRemains
    ()
  const
  {
    return end - current;
  }

} // namespace

#if _WIN32 || _WIN64
# pragma warning(pop)
#endif

#endif