PowerShell: Perform Final Sync Between 2 Directories

Current Version

# Final-Sync.ps1
# This function performs CRC checks on each file at the source.
# For every mismatch of file hashing signatures at the destination, the source file will be copied over to the destination.
# A log file will also be set to record progress

$sourceDirectory="E:\ShomeFolder"
$destinationDirectory="\\FiletSherver999\ShomeFolder"
$logFile="c:\final-sync-log.txt"
$log="/LOG+:$logFile"
$quickEmcopySwitches="/noprivchecks /de /s /purge /r:0 /w:0 $log";

# This system backup privilege escalation should be ran only once per session
function addSystemPrivilege{
param(
[String[]]$privileges=@("SeBackupPrivilege","SeRestorePrivilege")
)

function includeSystemPrivileges{
$win32api = @'
using System;
using System.Runtime.InteropServices;

namespace SystemPrivilege.Win32API
{
[StructLayout(LayoutKind.Sequential)]
public struct LUID
{
public UInt32 LowPart;
public Int32 HighPart;
}

[StructLayout(LayoutKind.Sequential)]
public struct LUID_AND_ATTRIBUTES
{
public LUID Luid;
public UInt32 Attributes;
}

[StructLayout(LayoutKind.Sequential)]
public struct TOKEN_PRIVILEGES
{
public UInt32 PrivilegeCount;
public LUID Luid;
public UInt32 Attributes;
}

public class Privileges
{
public const UInt32 DELETE = 0x00010000;
public const UInt32 READ_CONTROL = 0x00020000;
public const UInt32 WRITE_DAC = 0x00040000;
public const UInt32 WRITE_OWNER = 0x00080000;
public const UInt32 SYNCHRONIZE = 0x00100000;
public const UInt32 STANDARD_RIGHTS_ALL = (
READ_CONTROL |
WRITE_OWNER |
WRITE_DAC |
DELETE |
SYNCHRONIZE
);
public const UInt32 STANDARD_RIGHTS_REQUIRED = 0x000F0000u;
public const UInt32 STANDARD_RIGHTS_READ = 0x00020000u;

public const UInt32 SE_PRIVILEGE_ENABLED_BY_DEFAULT = 0x00000001u;
public const UInt32 SE_PRIVILEGE_ENABLED = 0x00000002u;
public const UInt32 SE_PRIVILEGE_REMOVED = 0x00000004u;
public const UInt32 SE_PRIVILEGE_USED_FOR_ACCESS = 0x80000000u;

public const UInt32 TOKEN_QUERY = 0x00000008;
public const UInt32 TOKEN_ADJUST_PRIVILEGES = 0x00000020;

public const UInt32 TOKEN_ASSIGN_PRIMARY = 0x00000001u;
public const UInt32 TOKEN_DUPLICATE = 0x00000002u;
public const UInt32 TOKEN_IMPERSONATE = 0x00000004u;
public const UInt32 TOKEN_QUERY_SOURCE = 0x00000010u;
public const UInt32 TOKEN_ADJUST_GROUPS = 0x00000040u;
public const UInt32 TOKEN_ADJUST_DEFAULT = 0x00000080u;
public const UInt32 TOKEN_ADJUST_SESSIONID = 0x00000100u;
public const UInt32 TOKEN_READ = (
STANDARD_RIGHTS_READ |
TOKEN_QUERY
);
public const UInt32 TOKEN_ALL_ACCESS = (
STANDARD_RIGHTS_REQUIRED |
TOKEN_ASSIGN_PRIMARY |
TOKEN_DUPLICATE |
TOKEN_IMPERSONATE |
TOKEN_QUERY |
TOKEN_QUERY_SOURCE |
TOKEN_ADJUST_PRIVILEGES |
TOKEN_ADJUST_GROUPS |
TOKEN_ADJUST_DEFAULT |
TOKEN_ADJUST_SESSIONID
);

[DllImport("kernel32.dll", SetLastError = true, ExactSpelling = true)]
public static extern IntPtr GetCurrentProcess();

[DllImport("kernel32.dll", SetLastError = true, ExactSpelling = true)]
public static extern IntPtr GetCurrentThread();

[DllImport("advapi32.dll", SetLastError = true, CharSet = CharSet.Auto)]
public static extern bool LookupPrivilegeValue(string lpSystemName, string lpName, out LUID lpLuid);

[DllImport("advapi32.dll", SetLastError = true, CharSet = CharSet.Auto)]
public static extern bool AdjustTokenPrivileges(IntPtr TokenHandle, bool DisableAllPrivileges, ref TOKEN_PRIVILEGES NewState, UInt32 BufferLengthInBytes, IntPtr PreviousStateNull, IntPtr ReturnLengthInBytesNull);

[DllImport("advapi32.dll", SetLastError = true, CharSet = CharSet.Auto)]
public static extern bool OpenProcessToken(IntPtr ProcessHandle, UInt32 DesiredAccess, out IntPtr TokenHandle);

[DllImport("advapi32.dll", SetLastError = true, CharSet = CharSet.Auto)]
public static extern bool OpenThreadToken(IntPtr ThreadHandle, UInt32 DesiredAccess, bool OpenAsSelf, out IntPtr TokenHandle);

[DllImport("ntdll.dll", EntryPoint = "RtlAdjustPrivilege")]
public static extern int RtlAdjustPrivilege(
UInt32 Privilege,
bool Enable,
bool CurrentThread,
ref bool Enabled
);

[DllImport("Kernel32.dll", SetLastError = true)]
public static extern bool CloseHandle(IntPtr handle);

//
//

private static LUID LookupPrivilege(string privilegeName)
{
LUID privilegeValue = new LUID();

bool res = LookupPrivilegeValue(null, privilegeName, out privilegeValue);

if (!res)
{
throw new Exception("Error: LookupPrivilegeValue()");
}

return privilegeValue;
}

//
//

public static void AdjustPrivilege(string privilegeName, bool enable)
{
IntPtr accessToken = IntPtr.Zero;
bool res = false;

try
{
LUID privilegeValue = LookupPrivilege(privilegeName);

res = OpenThreadToken(GetCurrentThread(), TOKEN_ADJUST_PRIVILEGES | TOKEN_QUERY, false, out accessToken);

if (!res)
{
res = OpenProcessToken(GetCurrentProcess(), TOKEN_ADJUST_PRIVILEGES | TOKEN_QUERY, out accessToken);

if (!res)
{
throw new Exception("Error: OpenProcessToken()");
}
}

TOKEN_PRIVILEGES tokenPrivileges = new TOKEN_PRIVILEGES();
tokenPrivileges.PrivilegeCount = 1;
tokenPrivileges.Luid = privilegeValue;

if (enable)
{
tokenPrivileges.Attributes = SE_PRIVILEGE_ENABLED;
}
else
{
tokenPrivileges.Attributes = 0;
}

res = AdjustTokenPrivileges(accessToken, false, ref tokenPrivileges, (uint)System.Runtime.InteropServices.Marshal.SizeOf(tokenPrivileges), IntPtr.Zero, IntPtr.Zero);

if (!res)
{
throw new Exception("Error: AdjustTokenPrivileges()");
}
}

finally
{
if (accessToken != IntPtr.Zero)
{
CloseHandle(accessToken);
accessToken = IntPtr.Zero;
}
}
}
}
}
'@

if ([object]::Equals(('SystemPrivilege.Win32API.Privileges' -as [type]), $null)) {
Add-Type -TypeDefinition $win32api
}
}
includeSystemPrivileges;


$privileges|%{[SystemPrivilege.Win32API.Privileges]::AdjustPrivilege($_, $true)}

# Validation
whoami /priv|?{$_ -match "SeBackupPrivilege|SeRestorePrivilege"}
}
addSystemPrivilege;

function finalSync{
param(
$sourceDirectory,
$destinationDirectory,
$protocol="crc",
$logFile
)
$message="=========================Starting to compare using protocol $protocol on $env:computername at $(get-date -Format "yyyy-MM-dd-hh:mm:ss")=========================`r`n";
Add-Content $logFile $message;

if ($protocol -like "crc"){
function expandZipfile($file, $destination){
$shell = new-object -com shell.application
$zip = $shell.NameSpace($file)

foreach($item in $zip.items()){
$shell.Namespace($destination).copyhere($item)
}
}

$crcInstalled=(Get-Command getcrc.exe -ErrorAction SilentlyContinue) # Deterministic check on whether emcopy is already available on this system
if (!($crcInstalled)){
$tempDir="C:\Temp";
$extractionDir="C:\Windows"
$sourceFile = "https://kimconnect.com/wp-content/uploads/2020/01/getcrc.zip";
$destinationFile = "$tempDir\getcrc.zip";
try{[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12}catch{}
New-Item -ItemType Directory -Force -Path $tempDir
New-Item -ItemType Directory -Force -Path $extractionDir
$webclient = New-Object System.Net.WebClient;
$WebClient.DownloadFile($sourceFile,$destinationFile);
expandZipfile $destinationFile -Destination $extractionDir
}
}

$accumulatedTime=0;
$timer=[System.Diagnostics.Stopwatch]::StartNew();
$sourceItems=Get-ChildItem $sourceDirectory -Recurse -Force -ea SilentlyContinue; # Include hidden files
$sourceFiles=$sourceItems|?{!$_.PSisContainer}; # Exclude folders
$sourceFilePaths=$sourceFiles|%{$_.FullName};
$filesCount=$sourceFilePaths.Count;
$rootCount=$sourceDirectory.Length;
$commonDenominatingPaths=$sourceFilePaths|%{$_.substring($rootCount,$_.length-$rootCount)};
$destinationFilePaths=$commonDenominatingPaths | %{if([string]$_[0] -eq "\"){
[string]$destinationDirectory+[string]$_;
}else{
[string]$destinationDirectory+"\"+[string]$_;
}
}

$filesCountTime+=$timer.Elapsed.TotalMinutes;
$message="$filesCount files detected at $sourceDirectory..."
write-host $message;
Add-Content $logFile $message;

function getHash{
param($file,$protocol="crc")

switch ($protocol){
"crc"{
return \\snapshots\FileServerClusters\Software\crc32.exe '$file'
}
"md5"{return $($md5 = (New-Object -TypeName System.Security.Cryptography.MD5CryptoServiceProvider)
try{
$stream = [System.IO.File]::Open("$file",[System.IO.Filemode]::Open, [System.IO.FileAccess]::Read);
$hash = [System.BitConverter]::ToString($md5.ComputeHash($stream));
$stream.Dispose();
$stream.Close();
return $hash;
}
catch{
write-host "Cannot get MD5 hash of $file";
return $hash});
}
}
}

$timer.Reset(); # Windows 2008 timer doesn't have the restart() method
$timer.Start();
for ($i=0;$i -lt $filesCount;$i++){
# Compare each source file with its counterpart incrementally
$sourceFile=$sourceFilePaths[$i];
$destinationFile=$destinationFilePaths[$i];
$sourceHash=getHash -file $sourceFile -protocol $protocol;
$destinationHash=getHash -file $destinationFile -protocol $protocol;
$match=$sourceHash -eq $destinationHash
if(!$match){
$message="$sourceFile has been copied to $destinationFile due to a mismatch.`r`n";
write-host $message;
$sourceFolder=split-path $sourceFile -Parent
$destinationFolder=split-path $destinationFile -Parent
$fileName=split-path $sourceFile -Leaf
$nullOutput = robocopy $sourceFolder $destinationFolder $fileName /is 2>&1;
Add-Content $logFile $message;
}
# exclude this feature to improve speed slightly
# Output a periodic progress report every 5 minutes
if($timer.Elapsed.TotalMinutes -ge 5){
$accumulatedTime+=$timer.Elapsed.TotalMinutes;
$percent=($i/$filesCount).tostring("P")
$message="$([math]::round($accumulatedTime,2)) minutes: $($i+1) of $filesCount processed `($percent`).";
Add-Content $logFile $message;
$timer.Reset(); # Windows 2008 timer doesn't have the restart() method
$timer.Start();
}
#
}
$totalTime+=$accumulatedTime+$filesCountTime;
$message="$sourceDirectory vs $destinationDirectory`r`n=========================File compare is completed on $env:computername. TotalElapsed: $([math]::round($totalTime,2)) minutes=========================";
Add-Content $logFile $message;
}

finalSync -sourceDirectory $source -destinationDirectory $destination -protocol "crc" -logFile $logFile
# Final-Sync.ps1
# This function performs CRC checks on each file at the source.
# For every mismatch of file hashing signatures at the destination, the source file will be copied over to the destination.
# A log file will also be set to record progress
# Word of advise: don't run this from a slow computer against 100 million files.

function finalSync{
param(
$sourceDirectory,
$destinationDirectory,
$logFile="C:\final-sync-log.txt"
)

$sourceItems=Get-ChildItem $sourceDirectory -Recurse -Force; # Include hidden files
$sourceFiles=$sourceItems|?{!$_.PSisContainer}; # Exclude folders
$sourceFilePaths=$sourceFiles|%{$_.FullName};
$filesCount=$sourceFilePaths.Count;
$rootCount=$sourceDirectory.Length;
$commonDenominatingPaths=$sourceFilePaths|%{$_.substring($rootCount,$_.length-$rootCount)};
$destinationFilePaths=$commonDenominatingPaths | %{if([string]$_[0] -eq "\"){
[string]$destinationDirectory+[string]$_;
}else{
[string]$destinationDirectory+"\"+[string]$_;
}
}
# Get-FileHash (PowerShell 4.0+) replacement for Windows 2008. Forward-compatible.
function getMd5{
param(
$file,
$md5 = (New-Object -TypeName System.Security.Cryptography.MD5CryptoServiceProvider)
)

# $hash = [System.BitConverter]::ToString($md5.ComputeHash([System.IO.File]::ReadAllBytes($file))); # This line has the issue of 2GB file size limit
try{
$stream = [System.IO.File]::Open("$file",[System.IO.Filemode]::Open, [System.IO.FileAccess]::Read);
$hash = [System.BitConverter]::ToString($md5.ComputeHash($stream));
$stream.Dispose();
$stream.Close();
return $hash;
}
catch{
return $null;
}
}

$message="Starting the file compare progress...";
Add-Content $logFile $message;
$timer=[System.Diagnostics.Stopwatch]::StartNew();
$md5 = New-Object -TypeName System.Security.Cryptography.MD5CryptoServiceProvider;
for ($i=0;$i -lt $filesCount;$i++){
# Compare each source file with its counterpart incrementally
$sourceFile=$sourceFilePaths[$i];
$destinationFile=$destinationFilePaths[$i];
$sourceHash=getMd5 -file $sourceFile -md5 $md5;
$destinationHash=getMd5 -file $destinationFile -md5 $md5;
$match=$sourceHash -eq $destinationHash
if(!$match){
$message="$sourceFile has been copied to $destinationFile`r`n";
write-host $message;
$sourceFolder=split-path $sourceFile -Parent
$destinationFolder=split-path $destinationFile -Parent
$fileName=split-path $sourceFile -Leaf
$nullOutput = robocopy $sourceFolder $destinationFolder $fileName /is 2>&1;
Add-Content $logFile $message;
}

# Output a periodic progress report every 10 minutes
if($timer.Elapsed.TotalMinutes -ge 10){
$accumulatedTime+=$timer.Elapsed.TotalMinutes;
$percent=($i/$filesCount).tostring("P")
$message="$($i+1) of $filesCount processed in $([math]::round($accumulatedTime,2)) `($percent`) minutes.";
Add-Content $logFile $message;
$timer.Restart();
}
}
}

Output:

<# Sample Usage:
PS C:\Windows\system32> finalSync -sourceDirectory c:\temp -destinationDirectory c:\tempcopy -logFile C:\log.txt
C:\temp\helloworld.py has been copied to c:\tempcopy\helloworld.py
#>

Leave a Reply

Your email address will not be published. Required fields are marked *