Uploading millions of entries to a string list can be very slow

Question

How can I load millions of entries from tadotable into a string list very quickly?

procedure TForm1.SlowLoadingIntoStringList(StringList: TStringList);
begin
  StringList.Clear;
  with SourceTable do
  begin
    Open;
    DisableControls;
    try
      while not EOF do
    begin
      StringList.Add(FieldByName('OriginalData').AsString);
      Next;
    end;
   finally
   EnableControls;
   Close;
  end;
end;

Answer 1

in your loop you will get a field. Search in a field from a loop

procedure TForm1.SlowLoadingIntoStringList(StringList: TStringList); 
var
  oField: TField;
begin
  StringList.Clear;   
  with SourceTable do   
  begin     
    Open;     
    DisableControls;     
    try       
      oField:= FieldByName('OriginalData');
      if oField<>Nil then
      begin
        while not EOF do
        begin       
          StringList.Add(oField.AsString);       
          Next;     
        end;   
      end; 
    finally    
      EnableControls;    
      Close;   
    end; 
  end;  
end;

Answer 2

Unfortunately, you cannot do this quickly. This is essentially a slow operation that requires large amounts of processor time and memory bandwidth to achieve. You can put more hardware on it, but I suspect you should reconsider your task instead.

Answer 3

: 1/

SELECT * FROM MYTABLE;

SELECT OriginalData FROM MYTABLE;

.

2/ , TStringList, .

3/ , :

• FieldByName
• OleDB

Answer 4

?

  // Turn off the sort for now
  StringList.Sorted := False;
  // Preallocate the space
  StringList.Capacity := recordCount;
  // Now add the data with Append()
  ...
  // Now turn the sort back on
  StringList.Sorted := True;

Answer 5

? ?

, , ...

, .

, ( ), blob (, , nvarchar (max)), , , , ( , ).

Text TStringList.

, 1000 .

.

Answer 6

@Ravaut123, :

, - - , on rowchanges, , .
, disablecontrols, .

...
SQLatable:= 'SELECT SingleField FROM atable ORDER BY indexedfield ASC';
AQuery:= TAdoQuery.Create(Form1);
AQuery.Connection:= ....
AQuery.SQL.Text:= SQLatable;  

, , , , . , .

function TForm1.LoadingAllIntoStringList(AQuery: TAdoQuery): TStringList;  
var 
  Field1: TField; 
begin 
  Result:= nil;
  try
    if not(AQuery.Active) then begin
      AQuery.Open;
    end else begin
      AQuery.First;
    end;
    AQuery.DisableControls;
    AQuery.Filtered:= false;                    //Filter in the SQL `where` clause
    AQuery.FetchAll;                            //Preload all data into memory
    Result:= TStringlist.Create;
  except
    {ignore error, will return nil}
  end;
  try
    Result.Sorted:= false;                      //Make sure you don't enable sorting
    Result.Capacity:= AQuery.RecordCount;       //Preallocate the needed space     
    Field1:= AQuery.FieldByName('SingleField'); //Never use `fieldbyname` in a loop!
    while not AQuery.EOF do begin
      Result.Add(Field1.AsString);
      AQuery.Next;
    end; {while} 
    AQuery.EnableControls;
  except
    FreeAndNil(Result);
  end;   

, , SQL. , .
CSV , DB .
MySQL :

SELECT X FROM table1 INTO OUTFILE 'c:/filename_of_csv_file.txt'

CSV.
.