Talk:System.Classes.TStrings.LoadFromFile

When loading extremely large files with LoadFromFile, Windows memory can become fragmented. For example, loading a 265MB file might work once but not twice in a row.

Here is some code that can be used to avoid the EOutOfMemory exception.

type
  TStringsIO = class
  private
    class procedure AddText(Strings: TStrings; const Text: string);
  public
    class procedure LoadFromFile(Strings: TStrings; const FileName: string); overload;
    class procedure LoadFromFile(Strings: TStrings; const FileName: string;
      Encoding: TEncoding); overload;
    class procedure LoadFromStream(Strings: TStrings; Stream: TStream); overload;
    class procedure LoadFromStream(Strings: TStrings; Stream: TStream;
      Encoding: TEncoding); overload;
  end;

class procedure TStringsIO.AddText(Strings: TStrings; const Text: string);
var
  S: TStrings;
begin
  S := TStringList.Create;
  try
    S.Text := Text;
    Strings.AddStrings(S);
  finally
    S.Free;
  end;
end;

class procedure TStringsIO.LoadFromFile(Strings: TStrings;
  const FileName: string);
begin
  LoadFromFile(Strings, FileName, nil);
end;

class procedure TStringsIO.LoadFromFile(Strings: TStrings;
  const FileName: string; Encoding: TEncoding);
var
  Stream: TStream;
begin
  Stream := TFileStream.Create(FileName, fmOpenRead or fmShareDenyWrite);
  try
    LoadFromStream(Strings, Stream, Encoding);
  finally
    Stream.Free;
  end;
end;

class procedure TStringsIO.LoadFromStream(Strings: TStrings; Stream: TStream);
begin
  LoadFromStream(Strings, Stream, nil);
end;

class procedure TStringsIO.LoadFromStream(Strings: TStrings; Stream: TStream;
  Encoding: TEncoding);
const
  MaxBufferSize = 128 * 1024;
  HalfBufferSize = MaxBufferSize div 2;
var
  Size: Int64;
  Buffer: TBytes;
  BufferSize: Integer;
  ReadSize: Integer;
  ExtractIndex: Integer;
  ExtractSize: Integer;
  UnprocessedSize: Integer;
begin
  Strings.Clear;
  SetLength(Buffer, MaxBufferSize);
  BufferSize := 0;

  while True do
  begin
    Size := Stream.Size - Stream.Position;
    if Size <= 0 then
      Break;

    // fill the buffer
    ReadSize := Min(Size, MaxBufferSize - BufferSize);
    Stream.Read(Buffer[BufferSize], ReadSize);
    Inc(BufferSize, ReadSize);

    // detect an encoding and skip BOM
    if Encoding = nil then
      ExtractIndex := TEncoding.GetBufferEncoding(Buffer, Encoding)
    else
      ExtractIndex := 0;

    ExtractSize := BufferSize;
    if ReadSize < Size then
    begin
      // postpone processing of the last incomplete line
      while (ExtractSize > HalfBufferSize) and
        (Buffer[ExtractSize - 1] <> 10) do
        Dec(ExtractSize);
    end;

    // parse and add complete lines
    if ExtractSize > 0 then
      AddText(Strings, Encoding.GetString(Buffer, ExtractIndex,
        ExtractSize - ExtractIndex));

    // move unprocessed buffer
    UnprocessedSize := BufferSize - ExtractSize;
    if UnprocessedSize > 0 then
      Move(Buffer[ExtractSize], Buffer[0], UnprocessedSize);
    BufferSize := UnprocessedSize;
  end;
end;

Usage

TStringsIO.LoadFromFile(MyStringList, 'd:\bigfile.txt');