PowerShell: File Copying Operation

Version 0.16
<#
.Description File_Copy_Script Version: 0.16

Purpose: this PowerShell Script is to efficiently mirror large batches of files using Emcopy in conjunction with Volume Shadow Services

Current Features:
1. Check for any errors on the Sources or Destinations and generate a report of any extra spaces in UNC
2. Create a snapshot of the source volume using Shadow Copy to capture any locked files
3. Execute robocopy to mirror the source to its corresponding destination with a time stamp variance allowance of 2 seconds for speed and resiliency
4. Sample the copied files to 'spot check' any time stamp variances
5. Execute in the context of an Administrator

Features planned for development:
6. Enable Volume Shadow Copy (VSS) at Source machines if it has been disabled, and reverse the action when done with copying
7. Trigger Remote Powershell to launch execution from a middle server (a "jump box" that is not a source nor destination)
if the provided Source is detected as a Universal Naming Convention (UNC) path instead of a local file system (LFS) path

Limitations:
1. This iteration requires that script is triggered from a local Windows machine with Internet access (no proxies) to download dependencies
2. Source must be LFS and Destination could either be LFS or UNC

Troubleshooting:

# Create snapshot
$targetVolume="M:\"
$shadowCopyClass=[WMICLASS]"root\cimv2:win32_shadowcopy";
$thisSnapshot = $shadowCopyClass.Create($targetVolume, "ClientAccessible");
$thisShadow = Get-WmiObject Win32_ShadowCopy | Where-Object { $_.ID -eq $thisSnapshot.ShadowID };
$thisShadowPath = $thisShadow.DeviceObject + "\";
C:
cmd /c mklink /d "C:\shadowcopy" $thisShadowPath

# Remove snapshot
$thisShadow.Delete()
$thisSnapshot.Dispose()
(Get-Item "C:\shadowcopy").Delete()

# Delete all shadows
vssadmin delete shadows /all /Quiet
#>

# Specify Sources (LFS) and Destinations (UNC)
# Using brackets notation to create a two dimensional array, overcoming PowerShell's limitations
$arr=@{}
$arr["from"] = @{}; $arr["to"] = @{}
$arr["from"] = @("C:\Users\Kim\Desktop\Tech_Documentation"); $arr["to"]=@("C:\Users\Kim\Desktop\Test")
$arr["from"] += "C:\Users\Kim\Desktop\Test"; $arr["to"]+="C:\Users\Kim\Desktop\Test1"

# Init global variables
$GLOBAL:totalTime=0;
$GLOBAL:totalStorage=0;
$GLOBAL:sampleSize=1000;
$GLOBAL:shadowMount="C:\shadowcopy"
$GLOBAL:toCreateShadow=$True
$GLOBAL:previousDriveLetter="none"
$GLOBAL:thisDriveLetter="none"
$GLOBAL:nextDriveLetter="none"
$clusterSizeOfNetworkShare=16384

# Initialize log files
$dateStamp = Get-Date -Format "yyyy-MM-dd-hhmmss"
$scriptName=$MyInvocation.MyCommand.Path
$scriptPath=Split-Path -Path $scriptName
$logPath="$scriptPath\filecopy_logs"
$logFile="$logPath\filecopy-log-$dateStamp.txt"
$log=" /LOG+:$logFile"
$lockedFilesReport="$logPath\_locked-files-log-$dateStamp.txt"
$pathErrorsLog="$logPath`\_path-errors-log-$dateStamp.txt"

# Emcopy switches
$switches="/o /secforce /de /sd /c /r:0 /w:0 /th 128 /s /purge /sdd /stream"
<# Switch explanations
/s copies sub directories
/purge removes files and directories from the destination that do not exist in the source.
/sdd forces the target directories dates to be synchronized with the source directory.
/de Compares both file size and last modification time when deciding to update a file, updates it if either have been changed.
/cm md5 - checks the file content after copying using and md5 comparison of the source and destination.
/o copies the files owner, without this the account used for the copy will be the owner
/secforce overwrites the destination security settings with the source security settings (no merging, actual update of security settings at the destination)
/sd preserves security, the file isn't copied if an error occurs during security settings.
/th 128 - Uses 128 threads, default is 64
/r:0 retries zero times
/w:0 is the wait time in seconds between retries
/c will allow the process to continue after the retries
/log:filename option allows to redirect the console messages to a new file.
/log+:filename option appends the new messages to an existing file.
/stream option enables the copie of files and directories datastreams. Without that option only the main datastream of files are copied.
#>

################################## Excuting Program as an Administrator ####################################
# Get the ID and security principal of the current user account
$myWindowsID=[System.Security.Principal.WindowsIdentity]::GetCurrent()
$myWindowsPrincipal=new-object System.Security.Principal.WindowsPrincipal($myWindowsID)

# Get the security principal for the Administrator role
$adminRole=[System.Security.Principal.WindowsBuiltInRole]::Administrator

# Check to see if we are currently running "as Administrator"
if ($myWindowsPrincipal.IsInRole($adminRole))
{
# We are running "as Administrator" - so change the title and background color to indicate this
$Host.UI.RawUI.WindowTitle = $myInvocation.MyCommand.Definition + "(Elevated)"
$Host.UI.RawUI.BackgroundColor = "Black"
clear-host
}
else
{
# We are not running "as Administrator" - so relaunch as administrator

# Create a new process object that starts PowerShell
$newProcess = new-object System.Diagnostics.ProcessStartInfo "PowerShell";

# Specify the current script path and name as a parameter
$newProcess.Arguments = $myInvocation.MyCommand.Definition;

# Indicate that the process should be elevated
$newProcess.Verb = "runas";

# Start the new process
[System.Diagnostics.Process]::Start($newProcess);

# Exit from the current, unelevated, process
exit
}

Write-Host -NoNewLine "Running as Administrator..."
################################## Excuting Program as an Administrator ####################################

################################## Commence Programming Sequence ####################################
New-Item -ItemType Directory -Force -Path $logpath;
"Executing copying tasks..."

function createShadow(){
[cmdletbinding()]
param(
[string]$targetVolume="C:\"
)
if (!($targetVolume -like "*\")){$targetVolume+="\"}

if ($toCreateShadow){
$shadowCopyClass=[WMICLASS]"root\cimv2:win32_shadowcopy"
$thisSnapshot = $shadowCopyClass.Create($targetVolume, "ClientAccessible")
$thisShadow = Get-WmiObject Win32_ShadowCopy | Where-Object { $_.ID -eq $thisSnapshot.ShadowID }
$thisShadowPath = $thisShadow.DeviceObject + "\"
C:
cmd /c mklink /d $shadowMount $thisShadowPath
"Shadow of $targetVolume has been made and it's accessible at this local file system (LFS): $shadowMount."
#copyLockedFiles; # this function is to be developed: retrieve lock files list, copy each item on list
#deleteShadow $thisShadow $thisShadowMount

# Export variables
$GLOBAL:shadow=$thisShadow;
$GLOBAL:snapshot=$thisSnapshot;

}else{
"Not creating a duplicate shadow in this iteration.";
}
}

function deleteShadow(){
if ($nextDriveLetter -eq $thisDriveLetter){
"Not removing snapshot because next one is on the same drive letter.";
}else{
# Remove symlink
(Get-Item $shadowMount).Delete()

# delete single instance of volume snapshots
$shadow.Delete()
$snapshot.Dispose()

# Delete all instances of volume snapshots
#Get-WmiObject Win32_ShadowCopy | % {$_.delete()}

"Shadow link $shadowMount has been removed."
}
}

function logPathError($pathError){
Add-Content $pathErrorsLog "$pathError";
}

function validateDirectory($dirToValidate){
if(Test-Path -Path $dirToValidate){return $True}
else{return $False;}
}

function createDirectory($dir){
# Create folder if it doesn't exist
if(!(validateDirectory $dir)){
New-Item -path $dir -type directory
}
}

function validateSourceAndDestination($thisblock){
$spacesCount=($thisblock.Split(' ')).Count-1
if ($spacesCount -eq 1){
$GLOBAL:source,$GLOBAL:destination=$thisblock.split(' ');
$sourceTest=validateDirectory $source
$destinationTest=validateDirectory $destination
if ($sourceTest -and $destinationTest){
return $True;
}
else {
if (!($sourceTest)){
logPathError "Source: $source";
return $False;
}
if (!($destinationTest)){
$createDestinationPath=(Read-Host -Prompt "Destination: $destination does not exist.`nType 'y' or 'yes' to create.");
if ($createDestinationPath -like 'yes' -or $createDestinationPath -like 'y'){
createDirectory $destination;
return $True;
}
else{
logPathError "Destination: $destination";
return $False
}
}
return $False;
}
}
else {
logPathError $thisblock;
return $False;
}
}

function translateSource{
param(
[string]$uncPath
)
$uri = new-object System.Uri($uncPath)
$thisLocalPath=$uri.LocalPath
#$thisHost=$uri.Host
$GLOBAL:sourceVolume="$((Get-Item $thisLocalPath).PSDrive.Name)`:"
$GLOBAL:translatedSource=$thisLocalPath -replace "$sourceVolume", $shadowMount
}

function sampleTimeStamp{
# Enable Remote to remote symlink following if it's not already set
$r2rEnabled=fsutil behavior query SymlinkEvaluation | select-string -Pattern "Remote to remote symbolic links are enabled."
if (!($r2rEnabled)){fsutil behavior set SymlinkEvaluation R2R:1;}

#$sourceFiles=Get-ChildItem $source -recurse | ? { !($_.PsIsContainer -and $_.FullName -notmatch 'archive' -and $_.FullName.Length -lt 260) } | get-random -count $sampleSize | % {$_.FullName}
$sourceFiles=Get-ChildItem $source -recurse | get-random -count $sampleSize | % {$_.FullName}
$commonDenominatingPaths=$sourceFiles | %{$_.replace($source,'')}
$destinationFiles=$commonDenominatingPaths | %{"$destination"+"$_";}
$badStamps=0;

if ($sourceFiles[0].Length -gt 1){
"Checking a sample of $sampleSize for any time stamp inaccuracy..."
for ($i=0;$i -lt $sourceFiles.length;$i++){
$sourceFile=$sourceFiles[$i];
$destinationFile=$destinationFiles[$i];
if ($destinationFile.Length -lt 260){
$sourceTimeStamp=(gi $sourceFile).LastWriteTime;
$destinationTimeStamp=(gi $destinationFile).LastWriteTime;
if ($sourceTimeStamp -eq $destinationTimeStamp){
#$output+="`r`n$destinationFile is GOOD";
} else {
$output+="`r`n$destinationFile timestamp of $destinationTimeStamp DOES NOT MATCH its source $sourceFile timestamp of $sourceTimeStamp";
$badStamps++;
}
} else {
$output+="`r`nException: $destinationFile length is $($destinationFile.Length)";
}
}
$output="`r`n`r`n------------$((($sourceFiles.Length-$badStamps)/$sourceFiles.Length).tostring('P')) of the files in a sample of $($sourceFiles.Length) are having accurate time stamps--------------`n"+$output;
}else{$output="`r`n`r`n------------Insufficient number of files to process.------------`r`n";}

$output;
Add-Content $logFile $output;
}

function checkDiskFree{
<#
Excerpt from http://technet.microsoft.com/en-us/library/ee692290(WS.10).aspx:

"For volumes less than 500 megabytes, the minimum is 50 megabytes of free space.
For volumes more than 500 megabytes, the minimum is 320 megabytes of free space.
It is recommended that least 1 gigabyte of free disk space on each volume if the volume size is more than 1 gigabyte."

#>

# Import variables
$thisNode="localhost"
$thisVolume=$sourceVolume

# Obtain disk information
$diskObject = Get-WmiObject Win32_LogicalDisk -ComputerName $thisNode -Filter "DeviceID='$thisVolume'"
$diskFree=[Math]::Round($diskObject.FreeSpace / 1MB)
$diskSize=[Math]::Round($diskObject.Size / 1MB)

switch ($diskSize){
{$diskSize -ge 1024} {if ($diskFree -gt 1024){$feasible=$True;}else{$feasible=$False;};;break;}
{$diskSize -ge 500} {if ($diskFree -gt 320){$feasible=$True;}else{$feasible=$False;};;break;}
{$diskSize -lt 500} {if ($diskFree -gt 50){$feasible=$True;}else{$feasible=$False;};break;}
}

return $feasible
}

function expandZipfile($file, $destination){
$shell = new-object -com shell.application
$zip = $shell.NameSpace($file)

foreach($item in $zip.items()){
$shell.Namespace($destination).copyhere($item)
}
}

function installEmcopy{
$emcopyIsInstalled=(Get-Command emcopy.exe -ErrorAction SilentlyContinue) # Deterministic check on whether emcopy is already available on this system
if (!($emcopyIsInstalled)){
$tempDir="C:\Temp";
$extractionDir="C:\Windows"
$source = "https://kimconnect.com/wp-content/uploads/2019/08/emcopy.zip";
$destinationFile = "$tempDir\emcopy.zip";
[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
New-Item -ItemType Directory -Force -Path $tempDir
New-Item -ItemType Directory -Force -Path $extractionDir
$webclient = New-Object System.Net.WebClient;
$WebClient.DownloadFile($source,$destinationFile);
expandZipfile $destinationFile -Destination $extractionDir
}else{
"EMCOPY is currently available in this system.`n";
}
}

function setACL{
get-acl -Path $source | Set-Acl -Path $destination # This command requires PowerShell 5
}

function startEmcopy{
param(
[string]$sourceAndDestination
)
"Emcopy has started..."
try{
invoke-expression "emcopy.exe $sourceAndDestination $switches $log";
}
catch{
# Record any errors into log and continue to next item
continue;
}
"`r`nEmcopy process has finished. Log is now generated at: $log";
}

Function startCopy($from,$to){
$GLOBAL:source=$from;
$GLOBAL:destination=$to;
$GLOBAL:block="$source $destination"
$time=0;
$stopWatch= [System.Diagnostics.Stopwatch]::StartNew()

if (validateSourceAndDestination $block){
translateSource $source;
#"Shadow Copy source: $tranlatedSource`nDestination: $destination";
$translatedBlock="$translatedSource $destination";
if (checkDiskFree){
createShadow $sourceVolume;
installEmcopy;
Try{
startEmcopy $translatedBlock;
}
catch{
continue;
}
deleteShadow;
sampleTimeStamp;
"Log for this activity has been generated at $logFile"
}else{"Not enough disk space to create a VSS snapshot.`r`Skipping this iteration..."}
}

# Getting storage estimate
#$storage=(Get-ChildItem $source -Recurse | Measure-Object -Property Length -Sum -ErrorAction Stop).Sum / 1GB;
#$storageDestination=(Get-ChildItem $destination -Recurse | Measure-Object -Property Length -Sum -ErrorAction Stop).Sum / 1GB;
$storage=(getSizeOnDisk -path $source) / 1GB;
$storageDestination=(getSizeOnDisk -path $destination) / 1GB;
$percentDifference=[math]::Round((($storageDestination-$storage)/$storage)*100),2)
$GLOBAL:totalStorage+=$storage;
$totalStorage;

# Stop the timer
$time=$stopWatch.Elapsed.TotalSeconds;
#$elapsedDisplay=([timespan]::fromseconds($elapsedSeconds)).ToString().Split('.')[0]
$GLOBAL:totalTime+=$time;
$totalTime;

# Add total time display to log
$timeDisplay = ([timespan]::fromseconds($time)).ToString()
Add-Content $logFile "`r`n------------------------------Time Elapsed: $timeDisplay------------------------------";

# Record overall speed to log
$speed=($storage/$time)*3600
Add-Content $logFile "`r`n---------------------------Speed: $([math]::Round($speed,4)) GiB per hour---------------------------";

# Compare storage results
Add-Content $logFile "`r`n---------------------------$(if($percentDifference -ge 0){"$percentDifference`% loss";}else{"$([math]::abs($percentDifference))`% gain";})`: Source $([math]::Round($storage,4)) GiB vs Destination $([math]::Round($storageDestination,4)) GiB---------------------------";
}

function getSizeOnDisk{
param (
[string]$path='.'
)

$source = @"
using System;
using System.Runtime.InteropServices;
using System.ComponentModel;
using System.IO;

namespace Win32
{

public class Disk {

[DllImport("kernel32.dll")]
static extern uint GetCompressedFileSizeW([In, MarshalAs(UnmanagedType.LPWStr)] string lpFileName,
[Out, MarshalAs(UnmanagedType.U4)] out uint lpFileSizeHigh);

public static ulong GetSizeOnDisk(string filename)
{
uint HighOrderSize;
uint LowOrderSize;
ulong size;

FileInfo file = new FileInfo(filename);
LowOrderSize = GetCompressedFileSizeW(file.FullName, out HighOrderSize);

if (HighOrderSize == 0 && LowOrderSize == 0xffffffff)
{
throw new Win32Exception(Marshal.GetLastWin32Error());
}
else {
size = ((ulong)HighOrderSize << 32) + LowOrderSize;
return size;
}
}
}
}

"@

Add-Type -TypeDefinition $source

$totalSize=0;
$count=0;
$driveLetter=(Get-Item $path).PSDrive.Root
$clusterSize=(Get-WmiObject -Class Win32_Volume | Where-Object {$_.Name -eq $driveLetter}).BlockSize
if(!($clusterSize)){$clusterSize=$clusterSizeOfNetworkShare}

Get-ChildItem $path -Recurse -Force | where { !$_.PSisContainer }|% {
$size = [Win32.Disk]::GetSizeOnDisk($_.FullName)
#$AFZ=[MidpointRounding]::AwayFromZero
#$roundedSize=[math]::Round($size/$ClusterSize+0.5,$AFZ)*$ClusterSize
if($size -gt $clusterSize){$roundedSize=[math]::ceiling($size/$clusterSize)*$clusterSize}else{$roundedSize=$clusterSize;}
$count=$count+1
#"$count`. $($_.FullName): $size vs $roundedSize"
$totalSize+=$roundedSize
}

#$foldersCount=(Get-ChildItem $path -Recurse -Force | where { $_.PSisContainer }).Count
#write-output "$count files $foldersCount folders total size: $totalSize"
return $totalSize;
}

function proceed{
$initInfo="=============================================Job Started: $dateStamp=============================================`r`n";
$initInfo+="Powershell version detected: $($PSVersionTable.PSVersion.Major)`.$($PSVersionTable.PSVersion.Minor)`r`n"
$initInfo+="Processing the following operations:`r";
$initInfo+=for ($i=0;$i -lt $arr.from.count; $i++){"$($arr.from[$i]) => $($arr.to[$i])`r";}
$initInfo;
Add-Content $logFile $initInfo;

# Process the copying operations
for ($i=0;$i -lt $arr.from.count; $i++){
$processDisplay="=======================================Pass $($i+1)`: $($arr.from[$i]) => $($arr.to[$i])=======================================`r";
Add-Content $logFile $processDisplay;
$GLOBAL:thisDriveLetter=Split-Path -Path $arr.from[$i] -Qualifier
$GLOBAL:nextDriveLetter=try{Split-Path -Path $arr.from[$i+1] -Qualifier}catch{$false;}
"Previous drive $previousDriveLetter vs current $thisDriveLetter vs next $nextDriveLetter"
if($previousDriveLetter -ne $thisDriveLetter){$GLOBAL:toCreateShadow=$True}else{$GLOBAL:toCreateShadow=$False}
startCopy $arr.from[$i] $arr.to[$i];
$GLOBAL:previousDriveLetter=$thisDriveLetter;
# cmd /c pause | out-null;
}

# Record overall speed to log
$aggregateSpeed=($totalStorage/$totalTime)*3600
$summary="`r`n==============================Program has completed.==============================`r";
$summary+="`r`n==============================$([math]::Round($totalStorage,4)) GiB has been processed in $([math]::Round($totalTime/3600,4)) hours(s)==============================`r"
$summary+="`r`n==============================Jobs Completed with Total Aggregate Speed: $([math]::Round($aggregateSpeed,4)) GiB per hour=============================="
$summary;
Add-Content $logFile $summary;
}

function checkInputs{
"Checking inputs for bad paths..."
$GLOBAL:proceed=$True
for ($i=0;$i -lt $arr.from.count; $i++){
$from=$arr.from[$i]
$to=$arr.to[$i]
if(!(test-path $from)){$from;$GLOBAL:proceed=$False}
if(!(test-path $to)){$to;$GLOBAL:proceed=$False}
}
}

checkInputs;
$proceed
if($proceed){proceed;}else{"Paths validation has failed. Program will not proceed."}
################################## Main Programming Sequence ####################################
pause;
#cmd /c pause | out-null;
Version 0.15
<# .Description File_Copy_Script Version: 0.15

Purpose: this PowerShell Script is to efficiently mirror large batches of files using Emcopy in conjunction with Volume Shadow Services

Current Features:
1. Check for any errors on the Sources or Destinations and generate a report of any extra spaces in UNC
2. Create a snapshot of the source volume using Shadow Copy to capture any locked files
3. Execute emcopy to mirror the source to its corresponding destination with a time stamp variance allowance of 2 seconds for speed and resiliency
4. Sample the copied files to 'spot check' any time stamp variances
5. Execute in the context of an Administrator

Features planned for development:
6. Enable Volume Shadow Copy (VSS) at Source machines if it has been disabled, and reverse the action when done with copying
7. Trigger Remote Powershell to launch execution from a middle server (a "jump box" that is not a source nor destination)
if the provided Source is detected as a Universal Naming Convention (UNC) path instead of a local file system (LFS) path

Limitations:
1. This iteration requires that script is triggered from a local Windows machine with Internet access (no proxies)
2. Source must be LFS and Destination could either be LFS or UNC
#>

# Specify Source and Destination
$source="C:\Users\Test\Desktop\Clients" # Must be a LFS path
$destination="C:\Test\Kim\Desktop\Test"
$block="$source $destination"

# Emcopy switches
$switches="/o /secforce /de /sd /c /r:0 /w:0 /th 128 /s /purge /sdd /stream"
<# Switch explanations
/s copies sub directories
/purge removes files and directories from the destination that do not exist in the source.
/sdd forces the target directories dates to be synchronized with the source directory.
/de Compares both file size and last modification time when deciding to update a file, updates it if either have been changed.
/cm md5 - checks the file content after copying using and md5 comparison of the source and destination.
/o copies the files owner, without this the account used for the copy will be the owner
/secforce overwrites the destination security settings with the source security settings (no merging, actual update of security settings at the destination)
/sd preserves security, the file isn't copied if an error occurs during security settings.
/th 128 - Uses 128 threads, default is 64
/r:0 retries zero times
/w:0 is the wait time in seconds between retries
/c will allow the process to continue after the retries
/log:filename option allows to redirect the console messages to a new file.
/log+:filename option appends the new messages to an existing file.
/stream option enables the copie of files and directories datastreams. Without that option only the main datastream of files are copied.
#>

# Initialize log files
$dateStamp = Get-Date -Format "yyyy-MM-dd-hhmmss"
$scriptName=$MyInvocation.MyCommand.Path
$scriptPath=Split-Path -Path $scriptName
$logPath="$scriptPath\emcopy_logs"
$logFile="$logPath\emcopy-log-$dateStamp.txt"
$log=" /LOG+:$logFile"
$lockedFilesReport="$logPath\_locked-files-log-$dateStamp.txt"
$pathErrorsLog="$logPath`\_path-errors-log-$dateStamp.txt"

# Init other variables
$sampleSize=1000;
$GLOBAL:shadowMount="C:\shadowcopy"

################################## Excuting Program as an Administrator ####################################
# Get the ID and security principal of the current user account
$myWindowsID=[System.Security.Principal.WindowsIdentity]::GetCurrent()
$myWindowsPrincipal=new-object System.Security.Principal.WindowsPrincipal($myWindowsID)

# Get the security principal for the Administrator role
$adminRole=[System.Security.Principal.WindowsBuiltInRole]::Administrator

# Check to see if we are currently running "as Administrator"
if ($myWindowsPrincipal.IsInRole($adminRole))
{
# We are running "as Administrator" - so change the title and background color to indicate this
$Host.UI.RawUI.WindowTitle = $myInvocation.MyCommand.Definition + "(Elevated)"
$Host.UI.RawUI.BackgroundColor = "White"
clear-host
}
else
{
# We are not running "as Administrator" - so relaunch as administrator

# Create a new process object that starts PowerShell
$newProcess = new-object System.Diagnostics.ProcessStartInfo "PowerShell";

# Specify the current script path and name as a parameter
$newProcess.Arguments = $myInvocation.MyCommand.Definition;

# Indicate that the process should be elevated
$newProcess.Verb = "runas";

# Start the new process
[System.Diagnostics.Process]::Start($newProcess);

# Exit from the current, unelevated, process
exit
}

Write-Host -NoNewLine "Running as Administrator..."
################################## Excuting Program as an Administrator ####################################

################################## Commence Programming Sequence ####################################
New-Item -ItemType Directory -Force -Path $logpath;
"Executing copying tasks..."

function createShadow(){
[cmdletbinding()]
param(
[string]$targetVolume="C:\"
)
if (!($targetVolume -like "*\")){$targetVolume+="\"}
$shadowCopyClass=[WMICLASS]"root\cimv2:win32_shadowcopy"
$thisSnapshot = $shadowCopyClass.Create($targetVolume, "ClientAccessible")
$thisShadow = Get-WmiObject Win32_ShadowCopy | Where-Object { $_.ID -eq $thisSnapshot.ShadowID }
$thisShadowPath = $thisShadow.DeviceObject + "\"
C:
cmd /c mklink /d $shadowMount $thisShadowPath
"Shadow of $targetVolume has been made and it's accessible at this local file system (LFS): $shadowMount."
#copyLockedFiles; # this function is to be developed: retrieve lock files list, copy each item on list
#deleteShadow $thisShadow $thisShadowMount

# Export variables
$GLOBAL:shadow=$thisShadow;
}

function deleteShadow(){
# Remove symlink
(Get-Item $shadowMount).Delete()

# delete single instance of volume snapshots
$shadow.Delete()

# Delete all instances of volume snapshots
#Get-WmiObject Win32_ShadowCopy | % {$_.delete()}

"Shadow link $shadowMount has been removed."
}

function logPathError($pathError){
Add-Content $pathErrorsLog "$pathError";
}

function validateDirectory($dirToValidate){
if(Test-Path -Path $dirToValidate){return $True}
else{return $False;}
}

function createDirectory($dir){
# Create folder if it doesn't exist
if(!(validateDirectory $dir)){
New-Item -path $dir -type directory
}
}

function validateSourceAndDestination($thisblock){
$spacesCount=($thisblock.Split(' ')).Count-1
if ($spacesCount -eq 1){
$GLOBAL:source,$GLOBAL:destination=$thisblock.split(' ');
$sourceTest=validateDirectory $source
$destinationTest=validateDirectory $destination
if ($sourceTest -and $destinationTest){
return $True;
}
else {
if (!($sourceTest)){
logPathError "Source: $source";
return $False;
}
if (!($destinationTest)){
$createDestinationPath=(Read-Host -Prompt "Destination: $destination does not exist.`nType 'y' or 'yes' to create.");
if ($createDestinationPath -like 'yes' -or $createDestinationPath -like 'y'){
createDirectory $destination;
return $True;
}
else{
logPathError "Destination: $destination";
return $False
}
}
return $False;
}
}
else {
logPathError $thisblock;
return $False;
}
}

function translateSource{
param(
[string]$uncPath
)
$uri = new-object System.Uri($uncPath)
$thisLocalPath=$uri.LocalPath
#$thisHost=$uri.Host
$GLOBAL:sourceVolume="$((Get-Item $thisLocalPath).PSDrive.Name)`:"
$GLOBAL:translatedSource=$thisLocalPath -replace "$sourceVolume", $shadowMount
}

function sampleTimeStamp{
# Enable Remote to remote symlink following if it's not already set
$r2rEnabled=fsutil behavior query SymlinkEvaluation | select-string -Pattern "Remote to remote symbolic links are enabled."
if (!($r2rEnabled)){fsutil behavior set SymlinkEvaluation R2R:1;}

#$sourceFiles=Get-ChildItem $source -recurse | ? { !($_.PsIsContainer -and $_.FullName -notmatch 'archive' -and $_.FullName.Length -lt 260) } | get-random -count $sampleSize | % {$_.FullName}
$sourceFiles=Get-ChildItem $source -recurse | get-random -count $sampleSize | % {$_.FullName}
$commonDenominatingPaths=$sourceFiles | %{$_.replace($source,'')}
$destinationFiles=$commonDenominatingPaths | %{"$destination"+"$_";}
$badStamps=0;

"Checking a sample of $sampleSize for any time stamp inaccuracy..."

for ($i=0;$i -lt $sourceFiles.length;$i++){
$sourceFile=$sourceFiles[$i];
$destinationFile=$destinationFiles[$i];
if ($destinationFile.Length -lt 260){
$sourceTimeStamp=(gi $sourceFile).LastWriteTime;
$destinationTimeStamp=(gi $destinationFile).LastWriteTime;
if ($sourceTimeStamp -eq $destinationTimeStamp){
#$output+="`r`n$destinationFile is GOOD";
} else {
$output+="`r`n$destinationFile timestamp of $destinationTimeStamp DOES NOT MATCH its source $sourceFile timestamp of $sourceTimeStamp";
$badStamps++;
}
} else {
$output+="`r`nException: $destinationFile length is $($destinationFile.Length)";
}
}
$output="`r`n`r`n------------$((($sourceFiles.Length-$badStamps)/$sourceFiles.Length).tostring('P')) of the files in a sample of $($sourceFiles.Length) are having accurate time stamps--------------`n"+$output;
Add-Content $logFile $output;
}

function checkDiskFree{
<#
Excerpt from http://technet.microsoft.com/en-us/library/ee692290(WS.10).aspx:

"For volumes less than 500 megabytes, the minimum is 50 megabytes of free space.
For volumes more than 500 megabytes, the minimum is 320 megabytes of free space.
It is recommended that least 1 gigabyte of free disk space on each volume if the volume size is more than 1 gigabyte."

#>

# Import variables
$thisNode="localhost"
$thisVolume=$sourceVolume

# Obtain disk information
$diskObject = Get-WmiObject Win32_LogicalDisk -ComputerName $thisNode -Filter "DeviceID='$thisVolume'"
$diskFree=[Math]::Round($diskObject.FreeSpace / 1MB)
$diskSize=[Math]::Round($diskObject.Size / 1MB)

switch ($diskSize){
{$diskSize -ge 1024} {if ($diskFree -gt 1024){$feasible=$True;}else{$feasible=$False;};;break;}
{$diskSize -ge 500} {if ($diskFree -gt 320){$feasible=$True;}else{$feasible=$False;};;break;}
{$diskSize -lt 500} {if ($diskFree -gt 50){$feasible=$True;}else{$feasible=$False;};break;}
}

return $feasible
}

function expandZipfile($file, $destination){
$shell = new-object -com shell.application
$zip = $shell.NameSpace($file)

foreach($item in $zip.items()){
$shell.Namespace($destination).copyhere($item)
}
}

function installEmcopy{
$emcopyIsInstalled=(Get-Command emcopy.exe -ErrorAction SilentlyContinue) # Deterministic check on whether emcopy is already available on this system
if (!($emcopyIsInstalled)){
$tempDir="C:\Temp";
$extractionDir="C:\Windows"
$source = "https://kimconnect.com/wp-content/uploads/2019/08/emcopy.zip";
$destinationFile = "$tempDir\emcopy.zip";
[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
New-Item -ItemType Directory -Force -Path $tempDir
New-Item -ItemType Directory -Force -Path $extractionDir
$webclient = New-Object System.Net.WebClient;
$WebClient.DownloadFile($source,$destinationFile);
expandZipfile $destinationFile -Destination $extractionDir
}else{
"EMCOPY is currently available in this system.`n";
}
}

function setACL{
get-acl -Path $source | Set-Acl -Path $destination # This command requires PowerShell 5
}

function startEmcopy{
param(
[string]$sourceAndDestination
)
"Emcopy has started..."
try{
invoke-expression "emcopy.exe $sourceAndDestination $switches $log";
}
catch{
# Record any errors into log and continue to next item
continue;
}
"`r`nEmcopy process has finished. Log is now generated at: $log";
}

Function proceed{
# Start the timer
Add-Content $logFile "`n`n`n---------------------------------Job Started: $dateStamp---------------------------------";
$totalTime=0;
$stopWatch= [System.Diagnostics.Stopwatch]::StartNew()
"Powershell version detected: $($PSVersionTable.PSVersion.Major)`.$($PSVersionTable.PSVersion.Minor)"
if (validateSourceAndDestination $block){
translateSource $source;
#"Shadow Copy source: $tranlatedSource`nDestination: $destination";
$translatedBlock="$translatedSource $destination";
if (checkDiskFree){
createShadow $sourceVolume;
installEmcopy;
startEmcopy $translatedBlock;
deleteShadow;
sampleTimeStamp;
"Program is completed."
"Log for this activity has been generated at $logFile"
}else{"Not enough disk space to create a VSS snapshot.`r`Program is aborted."}
}

# Getting storage estimate
$storage=(Get-ChildItem $source -Recurse | Measure-Object -Property Length -Sum -ErrorAction Stop).Sum / 1GB;
$storageDestination=(Get-ChildItem $destination -Recurse | Measure-Object -Property Length -Sum -ErrorAction Stop).Sum / 1GB;

# Stop the timer
#$elapsedSeconds=$stopWatch.Elapsed.TotalSeconds;
$totalTime=$stopWatch.Elapsed.TotalSeconds;
#$elapsedDisplay=([timespan]::fromseconds($elapsedSeconds)).ToString().Split('.')[0]
#$totalTime+=$elapsedSeconds;

# Add total time display to log
$timeDisplay = ([timespan]::fromseconds($totalTime)).ToString()
Add-Content $logFile "`r`n------------------------------Total Time Elapsed: $timeDisplay------------------------------";

# Record overall speed to log
$speed=$storage/($totalTime/360)
Add-Content $logFile "`r`n---------------------------Aggregate speed: $speed GB per hour---------------------------";

# Compare storage results
Add-Content $logFile "`r`n---------------------------Comparing: Source $storage GB vs Destination $storageDestination GB---------------------------";
}

proceed;
################################## Main Programming Sequence ####################################

# cmd /c pause | out-null;
Version 0.14
<#
.Description: File_Copy_Script Version: 0.14

Purpose: this PowerShell Script is to efficiently mirror large batches of files using Robocopy in conjunction with Volume Shadow Services

Current Features:
1. Check for any errors on the Sources or Destinations and generate a report of any extra spaces in UNC
2. Create a snapshot of the source volume using Shadow Copy to capture any locked files
3. Execute robocopy to mirror the source to its corresponding destination with a time stamp variance allowance of 2 seconds for speed and resiliency
4. Sample the copied files to 'spot check' any time stamp variances

Features planned for development:
5. Execute in the context of a Domain Administrator
6. Enable Volume Shadow Copy (VSS) at Source machines if it has been disabled, and reverse the action when done with copying
7. Trigger Remote Powershell to launch execution from a middle server (a "jump box" that is not a source nor destination)
if the provided Source is detected as a Universal Naming Convention (UNC) path instead of a local file system (LFS) path

Limitations:
- Require robocopy version XP016 or higher
- Only works if the source is LFS and destination is UNC
#>


$source="C:\Users\yamama\Desktop\test" # LFS path, only
$destination="\\FILESHERVER01\test" # UNC path, only
$block="$source $destination"

$switches="/MIR /SEC /DCOPY:T /R:0 /W:0 /XO /FFT /TBD /NP "
$dateStamp = Get-Date -Format "yyyy-MM-dd-hhmmss"
$scriptName=$MyInvocation.MyCommand.Path
$scriptPath=Split-Path -Path $scriptName
$logPath="$scriptPath\robocopy_logs"
$logFile="$logPath\robocopy-log-$dateStamp.txt"
$log="/LOG+:$logFile"
$lockedFilesReport="$logPath\_locked-files-log-$dateStamp.txt"
$pathErrorsLog="$logPath`\_path-errors-log-$dateStamp.txt"
$sampleSize=1000;
$GLOBAL:shadowMount="C:\shadowcopy"

# Get the ID and security principal of the current user account
$myWindowsID=[System.Security.Principal.WindowsIdentity]::GetCurrent()
$myWindowsPrincipal=new-object System.Security.Principal.WindowsPrincipal($myWindowsID)

# Get the security principal for the Administrator role
$adminRole=[System.Security.Principal.WindowsBuiltInRole]::Administrator

# Check to see if we are currently running "as Administrator"
if ($myWindowsPrincipal.IsInRole($adminRole))
{
# We are running "as Administrator" - so change the title and background color to indicate this
$Host.UI.RawUI.WindowTitle = $myInvocation.MyCommand.Definition + "(Elevated)"
$Host.UI.RawUI.BackgroundColor = "Black"
clear-host
}
else
{
# We are not running "as Administrator" - so relaunch as administrator

# Create a new process object that starts PowerShell
$newProcess = new-object System.Diagnostics.ProcessStartInfo "PowerShell";

# Specify the current script path and name as a parameter
$newProcess.Arguments = $myInvocation.MyCommand.Definition;

# Indicate that the process should be elevated
$newProcess.Verb = "runas";

# Start the new process
[System.Diagnostics.Process]::Start($newProcess);

# Exit from the current, unelevated, process
exit
}

# Run your code that needs to be elevated here
Write-Host -NoNewLine "Now Running as Administrator..."
#$null = $Host.UI.RawUI.ReadKey("NoEcho,IncludeKeyDown")

New-Item -ItemType Directory -Force -Path $logpath;
"Executing copying tasks..."

function createShadow(){
[cmdletbinding()]
param(
[string]$targetVolume="C:\"
)
if (!($targetVolume -like "*\")){$targetVolume+="\"}
$shadowCopyClass=[WMICLASS]"root\cimv2:win32_shadowcopy"
$thisSnapshot = $shadowCopyClass.Create($targetVolume, "ClientAccessible")
$thisShadow = Get-WmiObject Win32_ShadowCopy | Where-Object { $_.ID -eq $thisSnapshot.ShadowID }
$thisShadowPath = $thisShadow.DeviceObject + "\"
cmd /c mklink /d $shadowMount $thisShadowPath
"Shadow of $targetVolume has been made and it's accessible at this local file system (LFS): $shadowMount."
#copyLockedFiles; # this function is to be developed: retrieve lock files list, copy each item on list
#deleteShadow $thisShadow $thisShadowMount

# Export variables
$GLOBAL:shadow=$thisShadow;
}

function deleteShadow(){
# Remove symlink
(Get-Item $shadowMount).Delete()

# delete single instance of volume snapshots
$shadow.Delete()

# Delete all instances of volume snapshots
#Get-WmiObject Win32_ShadowCopy | % {$_.delete()}

"Shadow link $shadowMount has been removed."
}

function logPathError($pathError){
Add-Content $pathErrorsLog "$pathError";
}

function validateDirectory($dirToValidate){
if(Test-Path -Path $dirToValidate){return $True}
else{return $False;}
}

function createDirectory($dir){
# Create folder if it doesn't exist
if(!(validateDirectory $dir)){
New-Item -path $dir -type directory
}
}

function validateSourceAndDestination($thisblock){
$spacesCount=($thisblock.Split(' ')).Count-1
if ($spacesCount -eq 1){
$GLOBAL:source,$GLOBAL:destination=$thisblock.split(' ');
$sourceTest=validateDirectory $source
$destinationTest=validateDirectory $destination
if ($sourceTest -and $destinationTest){
return $True;
}
else {
if (!($sourceTest)){
logPathError "Source: $source";
return $False;
}
if (!($destinationTest)){
$createDestinationPath=(Read-Host -Prompt "Destination: $destination does not exist.`nType 'y' or 'yes' to create.");
if ($createDestinationPath -like 'yes' -or $createDestinationPath -like 'y'){
createDirectory $destination;
return $True;
}
else{
logPathError "Destination: $destination";
return $False
}
}
return $False;
}
}
else {
logPathError $thisblock;
return $False;
}
}

function translateSource{
param(
[string]$uncPath
)
$uri = new-object System.Uri($uncPath)
$thisLocalPath=$uri.LocalPath
#$thisHost=$uri.Host
$GLOBAL:sourceVolume="$((Get-Item $thisLocalPath).PSDrive.Name)`:"
$GLOBAL:translatedSource=$thisLocalPath -replace "$sourceVolume", $shadowMount
}

function reportLockedFiles($unc){
$files=Get-ChildItem -Path $unc -Recurse | where { ! $_.PSIsContainer }| % {$_.FullName}
#$x=$files[0]
#simulateLockedFile "$x"
$lockedFiles=$files | % { if(isFileLocked $_){$_; } }
if ($lockedFiles){
createDirectory $logPath;
Add-Content $lockedFilesReport $lockedFiles;
}
#$lockedFile.Close();
}

function startRobocopy{
param(
[string]$sourceAndDestination
)
Add-Content $logFile "`n`n`n---------------------------------Job Started: $dateStamp---------------------------------";
$totalTime=0;
"Robocopy has started..."
$stopWatch= [System.Diagnostics.Stopwatch]::StartNew()
try{
invoke-expression "robocopy $sourceAndDestination $switches $log | Out-Null";
#invoke-expression "robocopy $item $switches $log | Out-Null";
}
catch{
# Record any errors into log and continue to next item
continue;
}
#"Checking locked files in $source..."
#reportLockedFiles $source
$elapsedSeconds=$stopWatch.Elapsed.TotalSeconds;
$elapsedDisplay=([timespan]::fromseconds($elapsedSeconds)).ToString().Split('.')[0]
#Add-Content $logFile "Block Time Elapsed: $elapsedDisplay";
$totalTime+=$elapsedSeconds;
"`r`nRobocopy is done."
$timeDisplay = ([timespan]::fromseconds($totalTime)).ToString()
#$timeDisplay = $timeString.substring(0, $timeString.lastIndexOf("."));
Add-Content $logFile "`r`n------------------------------Total Time Elapsed: $timeDisplay------------------------------";
}

function sampleTimeStamp{
$sourceFiles=Get-ChildItem $source -recurse | ? { !($_.PsIsContainer -and $_.FullName -notmatch 'archive') } | get-random -count $sampleSize | % {$_.FullName}
$commonDenominatingPaths=$sourceFiles | %{$_.replace($source,'')}
$destinationFiles=$commonDenominatingPaths | %{"$destination"+"$_";}
$badStamps=0;

"Checking a sample of $sampleSize for any time stamp inaccuracy..."

for ($i=0;$i -lt $sourceFiles.length;$i++){
$sourceFile=$sourceFiles[$i];
$destinationFile=$destinationFiles[$i];
$sourceTimeStamp=(gi $sourceFile).LastWriteTime;
$destinationTimeStamp=(gi $destinationFile).LastWriteTime;
if ($sourceTimeStamp -eq $destinationTimeStamp){
#$output+="`r`n$destinationFile is GOOD";
}
else {
$output+="`r`n$destinationFile is BAD";
$badStamps++;
}
}
$output="`r`n`r`n------------$((($sourceFiles.Length-$badStamps)/$sourceFiles.Length).tostring('P')) of the files in a sample of $($sourceFiles.Length) are having accurate time stamps--------------`n"+$output;
Add-Content $logFile $output;
}

function checkDiskFree{
<#
Excerpt from http://technet.microsoft.com/en-us/library/ee692290(WS.10).aspx:

"For volumes less than 500 megabytes, the minimum is 50 megabytes of free space.
For volumes more than 500 megabytes, the minimum is 320 megabytes of free space.
It is recommended that least 1 gigabyte of free disk space on each volume if the volume size is more than 1 gigabyte."

#>

# Import variables
$thisNode="localhost"
$thisVolume=$sourceVolume

# Obtain disk information
$diskObject = Get-WmiObject Win32_LogicalDisk -ComputerName $thisNode -Filter "DeviceID='$thisVolume'"
$diskFree=[Math]::Round($diskObject.FreeSpace / 1MB)
$diskSize=[Math]::Round($diskObject.Size / 1MB)

switch ($diskSize){
{$diskSize -ge 1024} {if ($diskFree -gt 1024){$feasible=$True;}else{$feasible=$False;};;break;}
{$diskSize -ge 500} {if ($diskFree -gt 320){$feasible=$True;}else{$feasible=$False;};;break;}
{$diskSize -lt 500} {if ($diskFree -gt 50){$feasible=$True;}else{$feasible=$False;};break;}
}

return $feasible
}

Function proceed{
if (validateSourceAndDestination $block){
translateSource $source;
#"Shadow Copy source: $tranlatedSource`nDestination: $destination";
$translatedBlock="$translatedSource $destination";
if (checkDiskFree){
createShadow $sourceVolume;
startRobocopy $translatedBlock;
deleteShadow;
sampleTimeStamp;
"Program is completed."
"Log for this activity has been generated at $logFile"
}else{"Not enough disk space to create a VSS snapshot.`r`Program is aborted."}
}
}

proceed;
pause;

Leave a Reply

Your email address will not be published. Required fields are marked *