Theoretisches:
Copying large files on Windows

Slow Large File Copy Issues


Man kann auch XCOPY verwenden, z.B.
xcopy /J - Kopiert ohne E/A-Puffer. Für große Dateien empfohlen.

Hier mal meine Unit die ich schon ewig verwende und ein wenig angepasst habe und irgendwo mal gefunden habe.

zusammenfalten · markieren

Delphi-Quellcode:

unit uFastFileCopy;

interface

uses
 Windows, SysUtils;

type
 TFastCopyFileMode = (fcfmCreate, fcfmAppend);
 TFastCopyFileNormalCallback = procedure(const FileName: TFileName; const CurrentSize, TotalSize: Int64; var CanContinue: Boolean);
 TFastCopyFileMethodCallback = procedure(const FileName: TFileName; const CurrentSize, TotalSize: Int64; var CanContinue: Boolean) of object;

const
 BufferSize_Default: Cardinal = 4096 * 3 * 2;

 // Simplest definition
function FastCopyFile(const ASourceFileName, ADestinationFileName: TFileName): Boolean; overload;

// Definition with CopyMode and without any callbacks and default buffer
function FastCopyFile(const ASourceFileName, ADestinationFileName: TFileName; CopyMode: TFastCopyFileMode): Boolean; overload;

// Definition with normal procedure callback and default buffer
function FastCopyFile(const ASourceFileName, ADestinationFileName: TFileName; CopyMode: TFastCopyFileMode; Callback: TFastCopyFileNormalCallback): Boolean; overload;

// Definition with normal procedure callback and custom buffer
function FastCopyFile(const ASourceFileName, ADestinationFileName: TFileName; CopyMode: TFastCopyFileMode; Callback: TFastCopyFileNormalCallback; BufferSize: Cardinal)
 : Boolean; overload;

// Definition with object method callback and custom buffer
function FastCopyFile(const ASourceFileName, ADestinationFileName: TFileName; CopyMode: TFastCopyFileMode; Callback: TFastCopyFileMethodCallback; BufferSize: Cardinal)
 : Boolean; overload;

implementation

{Dummy Callback: Method Version}
type
 TDummyCallBackClient = class(TObject)
 private
  procedure DummyCallback(const FileName: TFileName; const CurrentSize, TotalSize: Int64; var CanContinue: Boolean);
 end;

procedure TDummyCallBackClient.DummyCallback(const FileName: TFileName; const CurrentSize, TotalSize: Int64; var CanContinue: Boolean);
begin
 // Nothing
 CanContinue := True;
end;

{Dummy Callback: Classical Procedure Version}
procedure DummyCallback(const FileName: TFileName; const CurrentSize, TotalSize: Int64; var CanContinue: Boolean);
begin
 // Nothing
 CanContinue := True;
end;

{CreateFileW API abstract layer}
function OpenLongFileName(ALongFileName: string; DesiredAccess, ShareMode, CreationDisposition: LongWord): THandle;
var
 // IsUNC: Boolean;
 FileName: PWideChar;

begin
 // Translate relative paths to absolute ones
 ALongFileName := ExpandFileName(ALongFileName);

 // Check if already an UNC path
 // IsUNC := Copy(ALongFileName, 1, 2) = '\\';
 // if not IsUNC then
 // ALongFileName := '\\?\' + ALongFileName;

 // Preparing the FileName for the CreateFileW API call
 FileName := PWideChar(WideString(ALongFileName));

 // Calling the API
 Result := CreateFileW(FileName, DesiredAccess, ShareMode, nil, CreationDisposition, FILE_ATTRIBUTE_NORMAL, 0);
end;

{FastCopyFile implementation}
function FastCopyFile(const ASourceFileName, ADestinationFileName: TFileName; CopyMode: TFastCopyFileMode; Callback: TFastCopyFileNormalCallback; Callback2: TFastCopyFileMethodCallback;
 BufferSize: Cardinal): Boolean; overload;
var
 Buffer: TArray<Byte>;
 ASourceFile, ADestinationFile: THandle;
 FileSize, TotalBytesWritten: Int64;
 BytesRead, BytesWritten, BytesWritten2, CreationDisposition: LongWord;
 CanContinue, CanContinueFlag: Boolean;

begin
 FileSize := 0;
 TotalBytesWritten := 0;
 CanContinue := True;
 SetLength(Buffer, BufferSize);

 // Manage the Creation Disposition flag
 CreationDisposition := CREATE_ALWAYS;
 if CopyMode = fcfmAppend then
  CreationDisposition := OPEN_ALWAYS;

 // Opening the source file in read mode
 ASourceFile := OpenLongFileName(ASourceFileName, GENERIC_READ, 0, OPEN_EXISTING);
 if ASourceFile <> 0 then
  try
   FileSize := FileSeek(ASourceFile, Int64(0), FILE_END);
   FileSeek(ASourceFile, Int64(0), FILE_BEGIN);

   SysUtils.ForceDirectories(ExtractFilePath(ADestinationFileName));

   // Opening the destination file in write mode (in create/append state)
   ADestinationFile := OpenLongFileName(ADestinationFileName, GENERIC_WRITE, FILE_SHARE_READ, CreationDisposition);

   if ADestinationFile <> 0 then
    try
     // If append mode, jump to the file end
     if CopyMode = fcfmAppend then
      FileSeek(ADestinationFile, Int64(0), FILE_END);

     // For each blocks in the source file
     while CanContinue and (FileSeek(ASourceFile, Int64(0), FILE_CURRENT) < FileSize) do
      begin

       // Reading from source
       if (ReadFile(ASourceFile, Buffer[0], BufferSize, BytesRead, nil)) and (BytesRead <> 0) then
        begin
         // Writing to destination
         WriteFile(ADestinationFile, Buffer[0], BytesRead, BytesWritten, nil);

         // Read/Write secure code block (e.g. for WiFi connections)
         if BytesWritten < BytesRead then
          begin
           WriteFile(ADestinationFile, Buffer[BytesWritten], BytesRead - BytesWritten, BytesWritten2, nil);
           Inc(BytesWritten, BytesWritten2);
           if BytesWritten < BytesRead then
            RaiseLastOSError;
          end;

         // Notifying the caller for the current state
         Inc(TotalBytesWritten, BytesWritten);
         CanContinueFlag := True;
         if Assigned(Callback) then
          Callback(ASourceFileName, TotalBytesWritten, FileSize, CanContinueFlag);
         CanContinue := CanContinue and CanContinueFlag;
         if Assigned(Callback2) then
          Callback2(ASourceFileName, TotalBytesWritten, FileSize, CanContinueFlag);
         CanContinue := CanContinue and CanContinueFlag;
        end;

      end;

    finally
     CloseHandle(ADestinationFile);
    end;

  finally
   CloseHandle(ASourceFile);
  end;

 // Check if cancelled or not
 if not CanContinue then
  if FileExists(ADestinationFileName) then
   DeleteFile(ADestinationFileName);

 // Results (checking CanContinue flag isn't needed)
 Result := (FileSize <> 0) and (FileSize = TotalBytesWritten);
end;

{FastCopyFile simple definition}
function FastCopyFile(const ASourceFileName, ADestinationFileName: TFileName): Boolean; overload;
begin
 Result := FastCopyFile(ASourceFileName, ADestinationFileName, fcfmCreate);
end;

{FastCopyFile definition without any callbacks and default buffer}
function FastCopyFile(const ASourceFileName, ADestinationFileName: TFileName; CopyMode: TFastCopyFileMode): Boolean; overload;
begin
 Result := FastCopyFile(ASourceFileName, ADestinationFileName, CopyMode, DummyCallback, BufferSize_Default);
end;

{Definition with normal procedure callback and default buffer}
function FastCopyFile(const ASourceFileName, ADestinationFileName: TFileName; CopyMode: TFastCopyFileMode; Callback: TFastCopyFileNormalCallback): Boolean; overload;
begin
 Result := FastCopyFile(ASourceFileName, ADestinationFileName, CopyMode, DummyCallback, BufferSize_Default);
end;

{FastCopyFile definition with normal procedure callback and custom buffer}
function FastCopyFile(const ASourceFileName, ADestinationFileName: TFileName; CopyMode: TFastCopyFileMode; Callback: TFastCopyFileNormalCallback; BufferSize: Cardinal)
 : Boolean; overload;
var
 DummyObj: TDummyCallBackClient;

begin
 DummyObj := TDummyCallBackClient.Create;
 try
  Result := FastCopyFile(ASourceFileName, ADestinationFileName, CopyMode, Callback, DummyObj.DummyCallback, BufferSize);
 finally
  DummyObj.Free;
 end;
end;

{FastCopyFile definition with object method callback and custom buffer}
function FastCopyFile(const ASourceFileName, ADestinationFileName: TFileName; CopyMode: TFastCopyFileMode; Callback: TFastCopyFileMethodCallback; BufferSize: Cardinal)
 : Boolean; overload;
begin
 Result := FastCopyFile(ASourceFileName, ADestinationFileName, CopyMode, DummyCallback, Callback, BufferSize);
end;

end.

Aufruf zum Beispiel mit Callback

zusammenfalten · markieren

Delphi-Quellcode:

procedure FileCopyCallBack(const FileName: TFileName; const CurrentSize, TotalSize: Int64; var CanContinue: Boolean);
begin
 CanContinue := PruefeX_Y_Z;
end;

if FileSize < 4096 * 50 then
 BufferSize := iFileSize
else
 BufferSize := 4096 * 50;

if FastCopyFile(sSource, sDest, TFastCopyFileMode.fcfmCreate, FileCopyCallBack, BufferSize) then
 ....

Geht sicher noch schneller aber ich finde es gut soweit.

Original hier: http://stackoverflow.com/questions/4...fast-file-copy

Ich würde ja immer Blockread Blockwrite nehmen... Oder eben Robocopy

Mavarik

Nur um es mal erwähnt zu haben.

Statt mit Threads kann man auch im selben Thread asynchron arbeiten.
https://msdn.microsoft.com/de-de/lib.../aa365683.aspx

Kann sein, dass Windows/Dateisystemtreiber immer asynchron arbeiten und bei uns halt immer auf das ende warten, weil fast jeder synchron arbeitet.

Vielen Dank für eure Hinweise.

Ich denke, auch xcopy, robocopy und Kollegen kochen nur mit Wasser

Ich habe jetzt etwas mehr Intelligenz in die Read/Write Threads verlagert und kann jetzt mit dem Code im Main-Thread leben.

zusammenfalten · markieren

Delphi-Quellcode:

      while ( rest > 0 ) and ( Len > 0 ) do begin
         if ( read1.xBuf1Ready = 1 ) and ( write1.xBuf1Ready = 0 ) and ( write2.xBuf1Ready = 0 ) then begin
            len := read1.xLen1Read;
            rest := rest - len;
            write1.xLen1Read := Len;
            write2.xLen1Read := Len;
            write1.xBuf1Ready := 1; // Start write 1 Buffer 1
            write2.xBuf1Ready := 1; // Start write 2 Buffer 1
            read1.xBuf1Ready := 2; // Buf1 wird geschrieben
            dbg( 'RUN rest ' + intToStr( ( Rest div 1024 div 1024 ) ) + 'MB Buffer ' + intToStr( ( Len div 1024 div 1024 ) ) + 'MB');
         end;
         if ( read1.xBuf2Ready = 1 ) and ( write1.xBuf2Ready = 0 ) and ( write2.xBuf2Ready = 0 ) then begin
            len := read1.xLen2Read;
            rest := rest - len;
            write1.xLen2Read := Len;
            write2.xLen2Read := Len;
            write1.xBuf2Ready := 1; // Start write 1 Buffer 2
            write2.xBuf2Ready := 1; // Start write 2 Buffer 2
            read1.xBuf2Ready := 2; // Buf2 wird geschrieben
            dbg( 'RUN rest ' + intToStr( ( Rest div 1024 div 1024 ) ) + 'MB Buffer ' + intToStr( ( Len div 1024 div 1024 ) ) + 'MB');
         end;
         if ( read1.xBuf1Ready = 2 ) and ( write1.xBuf1Ready = 0 ) and ( write2.xBuf1Ready = 0 ) then read1.xBuf1Ready := 0; // Freigabe Read Buffer 1
         if ( read1.xBuf2Ready = 2 ) and ( write1.xBuf2Ready = 0 ) and ( write2.xBuf2Ready = 0 ) then read1.xBuf2Ready := 0; // Freigabe Read Buffer 2
         sleep(1);
      end; // While end

Tipp: FileSplitter

Ich hatte damals ein bissl mehr rumgeforscht, ein paar Dinge ausprobiert und versucht das kopieren zu optimieren.

Tipp: Übergroß braucht man den Cache nicht zu machen, denn ALLE teilen das wieder auf (Dateisystemtreiber haben intern nicht unendlich viel Speicher und auch Netzwerkprotokolle haben für ihre einzelnen Datenpakete maximalgrenzen) ... aber es ist "oft" von Vorteil, wenn man ein "ganzes" Vielfaches der einzelnen Pakete verwendet.

Vorallem sooo extrem große Dateien durch den WindowsFileCache zu jagen ist kontraproduktiv. (und standardmäßig machen das alle höheren Dateioperationen)