PowerShell: Use RoboCopy & Volume Shadow Copy to Mirror Terrabytes of Data

version 0.2
<#
.Description
Current Features:
1. Check for any errors on the Sources or Destinations and generate a report of any extra spaces in UNC
2. Scan for any open files from the source directory recursively and generate a report of any locked files during the copying operations
3. Execute robocopy to mirror each source to its corresponding destination with a time stamp variance allowance of 2 seconds for speed and resiliency

Features under development:
4. Turn on Volume Shadow Copy (VSS) at source
5. Remote into each source machine to perform file copying operations
6. Copy from VSS snapshots rather than from live systems to capture locked files

Recommendations:
1. Execute in the context of a Domain Administrator
2. Run this script on the source server to eliminate extra hops
#>

$sourcesAndDestinations=@(
"C:\Users\kim\Desktop\test\test1 C:\Users\kim\Desktop\test\test2",
"C:\Users\kim\Desktop\test\test2 C:\Users\kim\Desktop\test\test3"
)

#$switches="/MIR /SEC /Z /R:0 /W:0 /XO /XJD /XJF /FFT /MT:32 /TBD /V /NP /NDL /NJS /NS /NC "
$switches="/MIR /SEC /R:0 /W:0 /XO /XJD /XJF /FFT /MT:32 /TBD /NP "
$dateStamp = Get-Date -Format "yyyy-MM-dd-hhmmss"
$scriptName=$MyInvocation.MyCommand.Path
$scriptPath=Split-Path -Path $scriptName
$logPath="$scriptPath\logs"
$logFile="$logPath\robocopy-log-$dateStamp.txt"
$log="/LOG+:$logFile"
$lockedFilesReport="$logPath\_locked-files-log-$dateStamp.txt"

$source="";
$destination="";
$pathErrorsLog="$logPath`\_path-errors-log-$dateStamp.txt"

function validateDirectory($dirToValidate){
if(Test-Path -Path $dirToValidate){return $True}
else{return $False;}
}

function createDirectory($dir){
# Create folder if it doesn't exist
if(!(validateDirectory $dir)){
New-Item -path $dir -type directory
}
}

function createFiles{
createDirectory $testPath
# Generate dummy files
for ($i=0; $i -lt 100; $i++){
fsutil file createnew "$testPath\$i.txt" 2000
}
}

function simulateLockedFile ($lockit){
<#
# Simulate error by creating file in current directory with some contents; then, open file in locking mode
$testDir="$scriptPath\testlock"
createDirectory $testDir;
createDirectory $logPath;
$testLockFile="$testDir\testlock.txt"
Add-Content ($testLockFile) "locky here!"
#>
"Simulating a locking of $lockit ..."
$GLOBAL:lockedFile=[System.IO.File]::Open(($lockit), 'Open', 'Read', 'None')

<#
# Try Robocopying
try{
isFileLocked $testDir
#invoke-expression "robocopy $testDir\ '$testDir\test' $switches";
}
catch{
$file.close();
"Error: $_"
Add-Content $lockedFilesLog "$_";
}
#>
}

function isFileLocked ($file) {
Process {
If ([System.IO.File]::Exists($file)) {
Try {
$FileStream = [System.IO.File]::Open($file,'Open','Write')
$FileStream.Close()
$FileStream.Dispose()
$IsLocked = $False
} Catch [System.UnauthorizedAccessException] {
$IsLocked = 'AccessDenied'
} Catch {
$IsLocked = $True
}
return $IsLocked
}
}
}

function reportLockedFiles($unc){
$files=Get-ChildItem -Path $unc -Recurse | where { ! $_.PSIsContainer }| % {$_.FullName}
#$x=$files[0]
#simulateLockedFile "$x"
$lockedFiles=$files | % { if(isFileLocked $_){$_; } }
if ($lockedFiles){
createDirectory $logPath;
Add-Content $lockedFilesReport $lockedFiles;
}
#$lockedFile.Close();
}


function logPathError($pathError){
Add-Content $pathErrorsLog "$pathError";
}

function validateSourceAndDestination($block){
$spacesCount=($block.Split(' ')).Count-1
if ($spacesCount -eq 1){
$GLOBAL:source,$GLOBAL:destination=$block.split(' ');
$sourceTest=validateDirectory $source
$destinationTest=validateDirectory $destination
if ($sourceTest -and $destinationTest){
return $True;
}
else {
if (!($sourceTest)){
logPathError "Source: $source";
return $False;
}
if (!($destinationTest)){
$createDestinationPath=(Read-Host -Prompt "Destination: $destination does not exist.`nType 'y' or 'yes' to create.");
if ($createDestinationPath -like 'yes' -or $createDestinationPath -like 'y'){
createDirectory $destination;
return $True;
}
else{
logPathError "Destination: $destination";
return $False
}
}
return $False;
}
}
else {
logPathError $block;
return $False;
}
}

function displayTime($t){
if ($t -lt 86400){
$display=([timespan]::fromseconds($t)).ToString()
$dotsCount=($display.Split('.')).Count-1
if ($dotsCount -eq 1){}
}
}

function startRobocopy{
Add-Content $logFile "`n------------------------------Job Started: $dateStamp------------------------------";
$totalTime=0;
foreach ($item in $sourcesAndDestinations){
$stopWatch= [System.Diagnostics.Stopwatch]::StartNew()
if (validateSourceAndDestination $item){
try{
invoke-expression "robocopy $item $switches $log";
#invoke-expression "robocopy $item $switches $log | Out-Null";
}
catch{
# Record any errors into log and continue to next item
continue;
}
"Checking locked files in $source..."
reportLockedFiles $source
}
$elapsedSeconds=$stopWatch.Elapsed.TotalSeconds;
$elapsedDisplay=([timespan]::fromseconds($elapsedSeconds)).ToString().Split('.')[0]
Add-Content $logFile "Block Time Elapsed: $elapsedDisplay";
$totalTime+=$elapsedSeconds;
}
$timeDisplay = ([timespan]::fromseconds($totalTime)).ToString()
#$timeDisplay = $timeString.substring(0, $timeString.lastIndexOf("."));
Add-Content $logFile "`n------------------------------Total Time Elapsed: $timeDisplay------------------------------";
}

startRobocopy;
"Process completed.";
pause;
Version 0.11
$sourcesAndDestinations=@(
"C:\Users\Kim\Desktop\test\test1 C:\Users\Kim\Desktop\test\test2",
"C:\Users\Kim\Desktop\test\test2 C:\Users\Kim\Desktop\test\test3"
)

$switchesMirrorDirectories="/MIR /SEC /B /R:1 /W:2 /XO /FFT /NFL /NDL /NJH /NJS "
$dateStamp = Get-Date -Format "yyyy-MM-dd-hh-mm"
$scriptName=$MyInvocation.MyCommand.Path
$scriptPath=Split-Path -Path $scriptName
$logPath="$scriptPath\logs"
$logFile="$logPath`\robocopy-log-$dateStamp.txt"
$log="/LOG+:$logFile"

$source="";
$destination="";
$pathErrorsLog="$logPath`\_path-errors-log-$dateStamp.txt"

function validateDirectory($dirToValidate){
if(Test-Path -Path $dirToValidate){return $True}
else{return $False;}
}

function createDirectory($dir){
# Create folder if it doesn't exist
if(!(validateDirectory $dir)){
New-Item -path $dir -type directory
}
}

function createFiles{
createDirectory $testPath
# Generate dummy files
for ($i=0; $i -lt 100; $i++){
fsutil file createnew "$testPath\$i.txt" 2000
}
}

function logPathError($pathError){
Add-Content $pathErrorsLog "$pathError";
}

function validateSourceAndDestination($block){
$spacesCount=($block.Split(' ')).Count-1
if ($spacesCount -eq 1){
$source,$destination=$block.split(' ');
$sourceTest=validateDirectory $source
$destinationTest=validateDirectory $destination
if ($sourceTest -and $destinationTest){
return $True;
}
else {
if (!($sourceTest)){
logPathError "Source: $source";
return $False;
}
if (!($destinationTest)){
$createDestinationPath=(Read-Host -Prompt "Destination: $destination does not exist.`nType 'y' or 'yes' to create.");
if ($createDestinationPath -like 'yes' -or $createDestinationPath -like 'y'){
createDirectory $destination;
return $True;
}
else{
logPathError "Destination: $destination";
return $False
}
}
return $False;
}
}
else {
logPathError $block;
return $False;
}
}

function startRobocopy{
$totalTime=0;
foreach ($item in $sourcesAndDestinations){
$stopWatch= [System.Diagnostics.Stopwatch]::StartNew()
if (validateSourceAndDestination $item){
try{
invoke-expression "robocopy $item $switches $log";
}
catch{
# Record any errors into log and continue to next item
continue;
}
}
$elapsedSeconds=$stopWatch.Elapsed.TotalSeconds;
Add-Content $logFile "Elapsed: $elapsedSeconds";
$totalTime+=$elapsedSeconds;
}
$timeDisplay = ([timespan]::fromseconds($totalTime)).ToString()
Add-Content $logFile "Total Time elapsed: $timeDisplay";
}

function simulateLockedFile{
# Simulate error by creating file in current directory with some contents; then, open file in locking mode
$testDir="$scriptPath\testlock"
createDirectory $testDir;
$testLockFile="$testDir\testlock.txt"
Add-Content ($testLockFile) "locky here!"
$file=[System.IO.File]::Open(($testLockFile), 'Open', 'Read', 'None')

# Try Robocopying
try{
invoke-expression "robocopy $testDir '$testDir/testcopy' $switches";
}
catch{
$file.close();
break;
}
}

createDirectory $logPath
startRobocopy;
simulateLockedFile;
Sample Output
PS C:\Users\Kim\Desktop\test> C:\Users\Kim\Desktop\test\robocopy-test.ps1

Log File : C:\Users\Kim\Desktop\test\logs\robocopy-log-2019-07-04-010132.txt
Checking locked files in C:\Users\kim\Desktop\test\test1...

Log File : C:\Users\Kim\Desktop\test\logs\robocopy-log-2019-07-04-010132.txt
Checking locked files in C:\Users\kim\Desktop\test\test2...
Process completed.
Press Enter to continue...:

Leave a Reply

Your email address will not be published. Required fields are marked *