PowerShell: Synchronize Directories with File Signature Comparisons

<# fileCompare_v0.01.ps1
- Requires that the source and destination paths are hosted on Windows machines, not Linux NAS
- Script is backward compatible with Windows 2008 (PowerShell 2.0)
- This program leverages the CPU(s) of the destination server as well as the source's to speed up the file compare process
- I/O intensive tasks would benefit more on multi-threading than CPU intensive tasks
- Obtaining file hashes is a CPU intensive task; hence, multi-threading isn't used
- XXHash is the chosen utility to perform signature collection of file contents
- Jacksum CR8 is the secondary utility for signature hashing (requires Java, which is included in the installation sequence)
- TLS 1.2 download maybe an issue for some machines. Search KimConnect for a method to resolve such issues.
- Future development: detect PowerShell version and optimize execution basing on the given PowerShell versions.
#>

$arr=@();
$arr+=[PSCustomObject]@{Clustername='';From='C:\Temp';To='C:\TempCopy'};

################################## Excuting Program as an Administrator ####################################
# Get the ID and security principal of the current user account
$myWindowsID=[System.Security.Principal.WindowsIdentity]::GetCurrent()
$myWindowsPrincipal=new-object System.Security.Principal.WindowsPrincipal($myWindowsID)

# Get the security principal for the Administrator role
$adminRole=[System.Security.Principal.WindowsBuiltInRole]::Administrator

# Check to see if we are currently running "as Administrator"
if ($myWindowsPrincipal.IsInRole($adminRole))
{
# We are running "as Administrator" - so change the title and background color to indicate this
$Host.UI.RawUI.WindowTitle = $myInvocation.MyCommand.Definition + "(Elevated)"
#$Host.UI.RawUI.BackgroundColor = "Black"
clear-host
}else{
# We are not running "as Administrator" - so relaunch as administrator

# Create a new process object that starts PowerShell
$newProcess = new-object System.Diagnostics.ProcessStartInfo "PowerShell";

# Specify the current script path and name as a parameter
$newProcess.Arguments = $myInvocation.MyCommand.Definition;

# Indicate that the process should be elevated
$newProcess.Verb = "runas";

# Start the new process
[System.Diagnostics.Process]::Start($newProcess);

# Exit from the current, unelevated, process
exit
}

Write-Host -NoNewLine "Running as Administrator..."
################################## Excuting Program as an Administrator ####################################

function addSystemPrivilege{
param(
[String[]]$privileges=@("SeBackupPrivilege","SeRestorePrivilege")
)
$win32api = @'
using System;
using System.Runtime.InteropServices;

namespace SystemPrivilege.Win32API
{
[StructLayout(LayoutKind.Sequential)]
public struct LUID
{
public UInt32 LowPart;
public Int32 HighPart;
}

[StructLayout(LayoutKind.Sequential)]
public struct LUID_AND_ATTRIBUTES
{
public LUID Luid;
public UInt32 Attributes;
}

[StructLayout(LayoutKind.Sequential)]
public struct TOKEN_PRIVILEGES
{
public UInt32 PrivilegeCount;
public LUID Luid;
public UInt32 Attributes;
}

public class Privileges
{
public const UInt32 DELETE = 0x00010000;
public const UInt32 READ_CONTROL = 0x00020000;
public const UInt32 WRITE_DAC = 0x00040000;
public const UInt32 WRITE_OWNER = 0x00080000;
public const UInt32 SYNCHRONIZE = 0x00100000;
public const UInt32 STANDARD_RIGHTS_ALL = (
READ_CONTROL |
WRITE_OWNER |
WRITE_DAC |
DELETE |
SYNCHRONIZE
);
public const UInt32 STANDARD_RIGHTS_REQUIRED = 0x000F0000u;
public const UInt32 STANDARD_RIGHTS_READ = 0x00020000u;

public const UInt32 SE_PRIVILEGE_ENABLED_BY_DEFAULT = 0x00000001u;
public const UInt32 SE_PRIVILEGE_ENABLED = 0x00000002u;
public const UInt32 SE_PRIVILEGE_REMOVED = 0x00000004u;
public const UInt32 SE_PRIVILEGE_USED_FOR_ACCESS = 0x80000000u;

public const UInt32 TOKEN_QUERY = 0x00000008;
public const UInt32 TOKEN_ADJUST_PRIVILEGES = 0x00000020;

public const UInt32 TOKEN_ASSIGN_PRIMARY = 0x00000001u;
public const UInt32 TOKEN_DUPLICATE = 0x00000002u;
public const UInt32 TOKEN_IMPERSONATE = 0x00000004u;
public const UInt32 TOKEN_QUERY_SOURCE = 0x00000010u;
public const UInt32 TOKEN_ADJUST_GROUPS = 0x00000040u;
public const UInt32 TOKEN_ADJUST_DEFAULT = 0x00000080u;
public const UInt32 TOKEN_ADJUST_SESSIONID = 0x00000100u;
public const UInt32 TOKEN_READ = (
STANDARD_RIGHTS_READ |
TOKEN_QUERY
);
public const UInt32 TOKEN_ALL_ACCESS = (
STANDARD_RIGHTS_REQUIRED |
TOKEN_ASSIGN_PRIMARY |
TOKEN_DUPLICATE |
TOKEN_IMPERSONATE |
TOKEN_QUERY |
TOKEN_QUERY_SOURCE |
TOKEN_ADJUST_PRIVILEGES |
TOKEN_ADJUST_GROUPS |
TOKEN_ADJUST_DEFAULT |
TOKEN_ADJUST_SESSIONID
);

[DllImport("kernel32.dll", SetLastError = true, ExactSpelling = true)]
public static extern IntPtr GetCurrentProcess();

[DllImport("kernel32.dll", SetLastError = true, ExactSpelling = true)]
public static extern IntPtr GetCurrentThread();

[DllImport("advapi32.dll", SetLastError = true, CharSet = CharSet.Auto)]
public static extern bool LookupPrivilegeValue(string lpSystemName, string lpName, out LUID lpLuid);

[DllImport("advapi32.dll", SetLastError = true, CharSet = CharSet.Auto)]
public static extern bool AdjustTokenPrivileges(IntPtr TokenHandle, bool DisableAllPrivileges, ref TOKEN_PRIVILEGES NewState, UInt32 BufferLengthInBytes, IntPtr PreviousStateNull, IntPtr ReturnLengthInBytesNull);

[DllImport("advapi32.dll", SetLastError = true, CharSet = CharSet.Auto)]
public static extern bool OpenProcessToken(IntPtr ProcessHandle, UInt32 DesiredAccess, out IntPtr TokenHandle);

[DllImport("advapi32.dll", SetLastError = true, CharSet = CharSet.Auto)]
public static extern bool OpenThreadToken(IntPtr ThreadHandle, UInt32 DesiredAccess, bool OpenAsSelf, out IntPtr TokenHandle);

[DllImport("ntdll.dll", EntryPoint = "RtlAdjustPrivilege")]
public static extern int RtlAdjustPrivilege(
UInt32 Privilege,
bool Enable,
bool CurrentThread,
ref bool Enabled
);

[DllImport("Kernel32.dll", SetLastError = true)]
public static extern bool CloseHandle(IntPtr handle);

//
//

private static LUID LookupPrivilege(string privilegeName)
{
LUID privilegeValue = new LUID();

bool res = LookupPrivilegeValue(null, privilegeName, out privilegeValue);

if (!res)
{
throw new Exception("Error: LookupPrivilegeValue()");
}

return privilegeValue;
}

//
//

public static void AdjustPrivilege(string privilegeName, bool enable)
{
IntPtr accessToken = IntPtr.Zero;
bool res = false;

try
{
LUID privilegeValue = LookupPrivilege(privilegeName);

res = OpenThreadToken(GetCurrentThread(), TOKEN_ADJUST_PRIVILEGES | TOKEN_QUERY, false, out accessToken);

if (!res)
{
res = OpenProcessToken(GetCurrentProcess(), TOKEN_ADJUST_PRIVILEGES | TOKEN_QUERY, out accessToken);

if (!res)
{
throw new Exception("Error: OpenProcessToken()");
}
}

TOKEN_PRIVILEGES tokenPrivileges = new TOKEN_PRIVILEGES();
tokenPrivileges.PrivilegeCount = 1;
tokenPrivileges.Luid = privilegeValue;

if (enable)
{
tokenPrivileges.Attributes = SE_PRIVILEGE_ENABLED;
}
else
{
tokenPrivileges.Attributes = 0;
}

res = AdjustTokenPrivileges(accessToken, false, ref tokenPrivileges, (uint)System.Runtime.InteropServices.Marshal.SizeOf(tokenPrivileges), IntPtr.Zero, IntPtr.Zero);

if (!res)
{
throw new Exception("Error: AdjustTokenPrivileges()");
}
}

finally
{
if (accessToken != IntPtr.Zero)
{
CloseHandle(accessToken);
accessToken = IntPtr.Zero;
}
}
}
}
}
'@

if ([object]::Equals(('SystemPrivilege.Win32API.Privileges' -as [type]), $null)) {
Add-Type -TypeDefinition $win32api
}

$privileges|%{[SystemPrivilege.Win32API.Privileges]::AdjustPrivilege($_, $true)}

# Validation
$validation=whoami /priv|?{$_ -match "SeBackupPrivilege|SeRestorePrivilege"};
write-host $validation;
}

function installXxHash{
$xxHashIsInstalled=(Get-Command xxhsum.exe -ErrorAction SilentlyContinue);
if (!($xxHashIsInstalled)){
# Set default protocol to TLS 1.2 and bypass certificate trust issues
$classTrustAllCerts = @"
using System.Net;
using System.Security.Cryptography.X509Certificates;
public class TrustAllCertsPolicy : ICertificatePolicy {
public bool CheckValidationResult(ServicePoint srvPoint, X509Certificate certificate,WebRequest request, int certificateProblem) {
return true;
}
}
"@
try{
Add-Type -TypeDefinition $classTrustAllCerts
$trustAllCertsPolicy=New-Object TrustAllCertsPolicy
[System.Net.ServicePointManager]::CertificatePolicy = $trustAllCertsPolicy
}catch{}
try{
[System.Net.ServicePointManager]::SecurityProtocol = [System.Net.SecurityProtocolType]::Tls12;
}catch{
[System.Net.ServicePointManager]::SecurityProtocol = [Enum]::ToObject([System.Net.SecurityProtocolType], 3072);
}

$tempDir="C:\Temp";
$extractionDir="C:\Windows"
$source = "https://github.com/Cyan4973/xxHash/releases/download/v0.7.2/xxhsum_win64_0_7_2_sse2.zip";
$destinationFile = "$tempDir\xxhash.zip";
New-Item -ItemType Directory -Force -Path $tempDir;
New-Item -ItemType Directory -Force -Path $extractionDir;
$webclient = New-Object System.Net.WebClient;
$WebClient.DownloadFile($source,$destinationFile);
try{
Expand-Archive -LiteralPath $destinationFile -DestinationPath $extractionDir;
}catch{
$shell = New-Object -ComObject Shell.Application
$zipFile = $shell.NameSpace($destinationFile);
$destinationFolder = $shell.NameSpace($extractionDir);

$copyFlags = 0x00
$copyFlags += 0x04 # Hide progress dialogs
$copyFlags += 0x10 # Overwrite existing files

$destinationFolder.CopyHere($zipFile.Items(), $copyFlags)
}
# Validation
$xxHashIsInstalled=(Get-Command xxhsum.exe -ErrorAction SilentlyContinue);
if ($xxHashIsInstalled){
write-host "xxhsum.exe has been added into environmental path of $extractionDir successfully."
}else{
write-host "There was an error preventing xxhsum.exe from being installed onto this system.";
break;
}
}else{
write-host "xxHash is already available in this system.`n";
}
}

function installJackSum{
if (!(get-command jacksum -ea SilentlyContinue)){
write-host "Jacksum is not available on this system. Now attempting to install...";
# Install Chocolatey
if (!(Get-Command choco.exe -ErrorAction SilentlyContinue)) {
try{
Set-ExecutionPolicy Bypass -Scope Process -Force; iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1'))
}catch{
write-host "Unable to install Choco. Try enabling TLS 1.2 and retry this program";
break;
}
}
# Install Jacksum
try{
choco install jacksum -y;
}catch{
write-host "Unable to install Jacksum. Program cannot continue without that program.";
break;
}
}

$path=$env:path;
$defaultJavaPath="C:\Program Files (x86)\Common Files\Oracle\Java\javapath";
if (!($path -match [regex]::escape($defaultJavaPath))){
$env:path+=";$defaultJavaPath";
}
}
<# Test: Jacksum is slower than xxHash; hence, it is not used. Here's the test result
PS C:\Windows\system32> measure-command{jacksum -a crc8 -x c:\temp\testfile.txt}
Days : 0
Hours : 0
Minutes : 0
Seconds : 0
Milliseconds : 152
Ticks : 1520838
TotalDays : 1.76022916666667E-06
TotalHours : 4.22455E-05
TotalMinutes : 0.00253473
TotalSeconds : 0.1520838
TotalMilliseconds : 152.0838

PS C:\Windows\system32> measure-command{xxhsum -q c:\temp\testfile.txt}
Days : 0
Hours : 0
Minutes : 0
Seconds : 0
Milliseconds : 14
Ticks : 143584
TotalDays : 1.66185185185185E-07
TotalHours : 3.98844444444444E-06
TotalMinutes : 0.000239306666666667
TotalSeconds : 0.0143584
TotalMilliseconds : 14.3584
#>

# This function is not used until after "cutover"
function getFileHashes{
param ($directory)

# Be advised that this method of reading all file contents is extremely slow;
$message="Now obtaining hash signature of every file inside directory $directory...";
write-host $message;

# Fastest way to list all files in a directory
$getFiles = @"
cmd.exe /C dir '$directory' /S /B /W /A:-D
"@
$files = Invoke-Expression -Command:$getFiles; # Tested speed on an iSCSI volume: 629.36 files per second or 2,265,715 files per hour
$fileCount=$files.Count;
$filesArray=@();
if($fileCount){
for ($j=0;$j -lt $fileCount;$j++){
$file=$files[$j];
$hash=$([void]((xxhsum -q $file ) -match "^(.*?\s)");$matches[1]);
$filesArray+=[PSCustomObject]@{File=$file;Hash=$hash};
}
<#
# Run a sample of 1% of the files to get a speed estimate
$sampleTimer=[System.Diagnostics.Stopwatch]::StartNew();
[int]$tenPercentSample=$fileCount/10
for ($i=0;$i -lt $tenPercentSample;$i++){
$file=$files[$i];
$hash=$([void]((xxhsum.exe $file) -match "^(.*?\s)");$matches[1]);
$filesArray+=[PSCustomObject]@{File=$file;Hash=$hash};
}
$timeElapsedHours=[math]::round($sampleTimer.Elapsed.TotalHours,2);
$sampleSpeed=[math]::round($tenPercentSample/$sampleTimer.Elapsed.TotalHours,2);
$sampleTimer.stop();
$message="=======================================$tenPercentSample files have been processed in $timeElapsedHours hours: speed $sampleSpeed files/hour=======================================`r`n"
write-host $message;
Add-Content $logFile $message;

# Finish the rest of the files
for ($j=$tenPercentSample;$j -lt $fileCount;$j++){
$file=$files[$j];
$hash=$([void]((xxhsum $file) -match "^(.*?\s)");$matches[1]);
$filesArray+=[PSCustomObject]@{File=$file;Hash=$hash};
}
#>
}
return $filesArray;
}

# copyMismatchedFiles -sourcePath $source -destinationPath $destination -sourceFiles $fileDifferences;
function copyMismatchedFiles{
param(
$sourcePath,
$destinationPath,
$sourceFiles
)
foreach ($file in $sourceFiles){
$correspondingDestinationFile=$destinationPath+$file.Substring($source.Length);
<# This doesn't work with robocopy XP010
$sourceParent=split-path $file -parent;
$destinationParent=Split-Path $correspondingDestinationFile -parent;
$fileName=Split-path $file -Leaf;
$copyCommand="robocopy.exe '$sourceParent' '$destinationParent' '$fileName' /XO /njh /njs /ndl /nc /ns /np 2>&1";
#>
$leftTimeStamp=[datetime](Get-ItemProperty -Path $file -Name LastWriteTime -ErrorAction SilentlyContinue).lastwritetime;
$rightTimeStamp=try{[datetime](Get-ItemProperty -Path $correspondingDestinationFile -Name LastWriteTime -ErrorAction SilentlyContinue).lastwritetime}catch{}
if ($leftTimeStamp -gt $rightTimeStamp){
$copyCommand="copy '$file' '$correspondingDestinationFile'";
invoke-expression $copyCommand;
$message="Copied: $file => $correspondingDestinationFile"
write-host $message;
}else{
$message="Source timestamp $leftTimeStamp is older than Destination timestamp $rightTimeStamp => Skipping $file"
write-host $message;
}
}
}

function fileCompare{
param($sourceAndDestinationArray)
foreach ($item in $sourceAndDestinationArray){
# Begin processing
$timer=[System.Diagnostics.Stopwatch]::StartNew();
$source=$item.From;
$destination=$item.To;

$sourceIsLocal=$source -match "^[A-Za-z]\:";
if ($sourceIsLocal){
$sourceServer=$env:computername;
}else{
$sourceServer=$([void]($destination -match "^\\\\(.*?)\\");$matches[1];);
$sourceWinRmIsAvailable=Test-WSMan -ComputerName $sourceServer -ErrorAction silentlycontinue
if (!($sourceWinRmIsAvailable)){$sourceServer=$env:computername}
}

$destinationIsLocal=$destination -match "^[A-Za-z]\:";
if ($destinationIsLocal){
$destinationServer=$env:computername;
}else{
$destinationServer=$([void]($destination -match "^\\\\(.*?)\\");$matches[1];);
$remoteWinRmIsAvailable=Test-WSMan -ComputerName $destinationServer -ErrorAction silentlycontinue
if (!($remoteWinRmIsAvailable)){$destinationServer=$env:computername}
}

$job1=invoke-command -computername $sourceServer -ScriptBlock{
param($addSystemPrivilege,$installXxHash,$getFileHashes,$directory)
[ScriptBlock]::Create($addSystemPrivilege).invoke();
[ScriptBlock]::Create($installXxHash).invoke();
$executionTime=(measure-command{$fileArray=[ScriptBlock]::Create($getFileHashes).invoke($directory)}).TotalHours
return @($executionTime,$fileArray)
} -args ${function:addSystemPrivilege},${function:installXxHash},${function:getFileHashes},$source -AsJob

$job2=invoke-command -computername $destinationServer -ScriptBlock{
param($addSystemPrivilege,$installXxHash,$getFileHashes,$directory)
[ScriptBlock]::Create($addSystemPrivilege).invoke();
[ScriptBlock]::Create($installXxHash).invoke();
$executionTime=(measure-command{$fileArray=[ScriptBlock]::Create($getFileHashes).invoke($directory);}).TotalHours
return @($executionTime,$fileArray)
} -args ${function:addSystemPrivilege},${function:installXxHash},${function:getFileHashes},$destination -AsJob

#$job1=Start-Job $processPath -ArgumentList $source
#$job2=Start-Job $processPath -ArgumentList $destination
$job1ID=$job1.ID;
$job2ID=$job2.ID;

# Wait for both jobs to complete
Wait-Job -Id $job1.id,$job2.id
#While ((Get-Job -id $job1ID -State "Running") -OR (Get-Job -id $job2ID -State "Running")){Start-Sleep 10}

# Getting the information back from the jobs
$sourceArray=Receive-Job -Id $job1.id #| Select-Object -Property * -ExcludeProperty PSComputerName,RunspaceID
$destinationArray=Receive-Job -Id $job2.id #| Select-Object -Property * -ExcludeProperty PSComputerName,RunspaceID
remove-job -Id $job1.id;
remove-job -Id $job2.id;

#$fileDifferences=(Compare-Object -ReferenceObject $sourceArray -DifferenceObject $destinationArray -Property Hash -PassThru).File #This notation doesn't work on PS2.0
# This code is to be backward compatible with Windows 2008
$job1Duration=$sourceArray[0]
$sourceFiles=$sourceArray[1]|%{$_.File}
$sourceHashes=$sourceArray[1]|%{$_.Hash}

$job2Duration=$destinationArray[0]
$destinationFiles=$destinationArray[1]|%{$_.File}
$destinationHashes=$destinationArray[1]|%{$_.Hash}

# Clear variables to save memory
Clear-Variable -Name sourceArray;
Clear-Variable -Name destinationArray;

$sourceFileCount=$sourceFiles.Count;
$destinationFileCount=$destinationFiles.Count;
#$job1Duration=($job1.PSEndTime-$job1.PSBeginTime).TotalHours; # These were not available on Windows 2008
#$job2Duration=($job2.PSEndTime-$job2.PSBeginTime).TotalHours;
$sourceSpeed=[math]::round($sourceFileCount/$job1Duration,2);
$destinationSpeed=[math]::round($destinationFileCount/$job2Duration,2);
$message="Source: $sourceFileCount files processed in $([math]::round($job1Duration,2)) hours v.s. Destination: $destinationFileCount files processed in $([math]::round($job2Duration)) hours`r`n";
$message+="Source files hashing collection speed: $sourceSpeed files/hour vs Destination files hashing collection speed: $destinationSpeed files/hour`r`n";
write-host $message;

# Comparing arrays in PowerShell 2.0 compatible coding
$fileDifferences=@();
if ($destinationFileCount){
for ($i=0;$i -lt $sourceFiles.count;$i++){
$sourceFile=$sourceFiles[$i]
$sourceHash=$sourceHashes[$i]
$x=$source.length
$commonDenominator=$sourceFile.SubString($x,$sourceFile.length-$x);
$translatedFileName=$destination+$commonDenominator;
#$matchedIndex=$destinationFiles.IndexOf($translatedFileName); # This doesn't work on legacy systems
$matchedIndex=[array]::indexof($destinationFiles,$translatedFileName) # load the .NET function as workaround for older Windows
if($matchedIndex -eq -1){
$fileDifferences+=,$sourceFile;
}else{
$hashMismatch=$sourceHash -ne $destinationHashes[$matchedIndex];
if ($hashMismatch){$fileDifferences+=,$sourceFile;}
}
}
}else{
$destinationFileCount=0;
if ($sourceFileCount){
$fileDifferences=$sourceFiles;
}else{
write-host "Source directory is empty. Program now aborts.";
break;
}
}
#>

if($fileDifferences){
$copyMessage="These source files don't match their destination counterparts:`r`n$($fileDifferences|out-string)`r`n";
$copyMessage+="Please wait while the program copies those files to their destinations.";
write-host $copyMessage;
copyMismatchedFiles -sourcePath $source -destinationPath $destination -sourceFiles $fileDifferences;
}

$time=[math]::round($timer.Elapsed.TotalHours,2);
$timer.Stop();
$timer.Reset();
$closingMessage="`r`nOverall Time: $time hours"
write-host $closingMessage;
}
}

fileCompare;

Leave a Reply

Your email address will not be published. Required fields are marked *