PowerShell: Maintaining Processes on Remote Servers

# maintainProcess.ps1
# version 0.0.1

$computernames=@(
    'server1',
    'server2'
)
$processName='cmd'
$processPath='C:\WINDOWS\system32\cmd.exe'
$minutesToDefineCrashed=1 # this marker is only valid if the process is marked as unresponsive by the system
$runMinutes=1
$credentials=$null
$maxMinutesPerJob=10
$verbose=$true

function maintainProcess{
    param(
        $targetMachines=$env:computername,
        $processname='cmd',
        $processPath='C:\Windows\system32\cmd.exe',
        $minutesToDefineCrashed=1,
        $runMinutes=2,
        $credentials
        )
    $actionOnCrash={param($processName,$processPath) 
        stop-process -name $processName -force -ea Ignore
        if($processPath){$null=start-process -FilePath $processPath}else{$null=start-process -Name $processName}
    }    
    
    function restartProcess($actionOnCrash,$processname,$processPath,$psSession){
        invoke-command -session $psSession -scriptblock $actionOnCrash -Arg $processname,$processPath
        $processIsRunning=$null -ne (invoke-command -session $psSession -scriptblock {param($processname)get-process -name $processName -EA SilentlyContinue} -Args $processname)
        if($processIsRunning){
            write-host "$processName has successfully restarted on $targetMachine"
            return $true
        }else{
            write-host "$processName has NOT successfully restarted on $targetMachine"
            return $false
        }
    }
    
    $results=[hashtable]@{}
    foreach($targetMachine in $targetMachines){
        $overallClock=[System.Diagnostics.Stopwatch]::StartNew()
        $psSession=if($credentials){
            New-PSSession $targetMachine -Credential $credentials
        }else{
            New-PSSession $targetMachine
            }     
        if($psSession.State -eq 'Opened'){      
            $previousCpuConsumption=0
            write-host "Checking CPU consumption of $processName on $targetMachine"
            do{
                $iterationClock=[System.Diagnostics.Stopwatch]::StartNew()          
                $process=Invoke-Command -Session $psSession {param($processname) Get-Process $processname -EA SilentlyContinue} -Args $processName
                if($null -eq $process){
                    write-warning "$processname is NOT running on $targetMachine"
                    $result=restartProcess $actionOnCrash $processname $processPath $psSession                    
                }else{
                    $responding=$process.Responding
                    $currentCpuConsumption=$process.CPU
                    write-host $currentCpuConsumption
                    $cpuConsumptionChanged=$currentCpuConsumption -ne $previousCpuConsumption
                    $noActivities=$iterationClock.elapsed.totalminutes -ge $minutesToDefineCrashed
                    if($cpuConsumptionChanged){
                        $null=$iterationClock.reset
                        $previousCpuConsumption=$currentCpuConsumption
                        $result=$true
                    }elseif(!$responding -and $noActivities){
                        write-warning "$processName has CRASHED on $targetMachine as defined its RESPONDING flag equal False and there are no activities."
                        $result=restartProcess $actionOnCrash $processname $processPath $psSession
                    }
                    sleep -Seconds 10
                }
            } until ($overallClock.elapsed.totalminutes -ge $runMinutes)   
            Remove-PSSession $psSession
            $minutesElapsed=[math]::round($overallClock.Elapsed.TotalMinutes,2)
            write-host "Runtime of $minutesElapsed minutes has elapsed for $targetMachine"
            $results+=[hashtable]@{$targetMachine=$result}
        }else{
            write-warning "Unable to open a WinRM session to $targetMachine.`r`nPlease monitor it's progress manually."
            $results+=[hashtable]@{$targetMachine=$null}
        }
    }
    return $results
}

# maintainProcess $targetMachines $processname $processPath $minutesToDefineCrashed $runMinutes $credential
function maintainProcessParallel{
    param(
        $computerNames=$env:computername,
        $processname='cmd',
        $processPath='C:\Windows\system32\cmd.exe',
        $minutesToDefineCrashed=1, # this marker is only valid if the process is marked as unresponsive by the system
        $runMinutes=2,
        $credentials,
        $maxMinutesPerJob=10,
        $verbose=$true
    )

    $timer=[System.Diagnostics.Stopwatch]::StartNew()
    $jobResults=@()
    $lineBreak=60
    $dotCount=0
    $minute=0
    $processorsCount=(Get-CimInstance Win32_ComputerSystem).NumberOfLogicalProcessors
    $cpuLoad=(Get-WmiObject win32_processor|Measure-Object -property LoadPercentage -Average).Average
    $maxSimultaneousJobs=if($cpuLoad -gt 90){$processorsCount}else{($processorsCount*2)-1} # dynamically limiting concurrent jobs basing on available CPU cores
    write-host "CPU load detected as: $cpuLoad`%`r`nSetting concurrent jobs max count to be $maxSimultaneousJobs"
    $jobtimer = @{}
    foreach($computerName in $computerNames){
        $thisIterationCompleted=$false
        do {
            $jobsCount=(Get-Job -State 'Running').Count
            if ($jobsCount -lt $maxSimultaneousJobs){            
                if($verbose){write-host "Initiating job for $computerName"}
                $job=Start-Job -name $computerName -ScriptBlock {
                    param($maintainProcess,$computername,$processname,$processPath,$minutesToDefineCrashed,$runMinutes,$credentials)
                    [scriptblock]::create($maintainProcess).invoke($computername,$processname,$processPath,$minutesToDefineCrashed,$runMinutes,$credential)
                } -Args ${function:maintainProcess},$computername,$processname,$processPath,$minutesToDefineCrashed,$runMinutes,$credentials
                $jobtimer[$job.Id]=[System.Diagnostics.Stopwatch]::startnew()
                $thisIterationCompleted=$true
            }else{
                if($verbose){
                    if($dotCount++ -lt $lineBreak){
                        write-host '.' -NoNewline
                    }else{
                        $minute++
                        write-host "`r`n$minute`t:" -ForegroundColor Yellow -NoNewline
                        $dotCount=0
                        }
                }
                sleep -seconds 1
            }
            $expiredJobs=$jobtimer.GetEnumerator()|?{$_.value.elapsed.totalminutes -ge $maxMinutesPerJob}
            if($expiredJobs){
                $expiredJobs.Name|%{stop-job -name $_ -EA Ignore}
                $expiredJobs.Name|%{$jobTimer.Remove($_)}
            }            
        }until ($thisIterationCompleted)
    }
    $totalJobsCount=(get-job).count
    $processedCount=0
    while($processedCount -lt $totalJobsCount){
        $completedJobs=get-job|?{$_.State -eq 'Completed'}
        $stoppedJobs=get-job|?{$_.State -eq 'Stopped'}
        $expiredJobs=$jobtimer.GetEnumerator()|?{$_.value.elapsed.totalminutes -ge $maxMinutesPerJob}
        if($expiredJobs){
            $expiredJobs.Name|%{stop-job -name $_ -EA Ignore}
            $expiredJobs.Name|%{$jobTimer.Remove($_)}
        }
        if($completedJobs){
            foreach ($job in $completedJobs){
                $computer=$job.Name
                if($verbose){
                    write-host "`r`n===================================================`r`n$computer job COMPLETED with these messages:`r`n===================================================`r`n"
                }
                $jobResult=receive-job -id $job.id
                $jobResults+=,$jobResult
                remove-job -id $job.id -force
                $processedCount++
            }
        }
        if($stoppedJobs){
            foreach ($job in $stoppedJobs){
                $computer=$job.Name
                if($verbose){
                    write-host "`r`n===================================================`r`n$computer job STOPPED with these messages:`r`n===================================================`r`n" -ForegroundColor Red
                }
                $jobResult=receive-job -id $job.id
                # $jobResults+=,$jobResult
                $timeZoneName=[System.TimeZoneInfo]::Local.StandardName
                $abbreviatedZoneName=if($timeZoneName -match ' '){[regex]::replace($timeZoneName,'([A-Z])\w+\s*', '$1')}else{$timeZoneName}
                $timeStampFormat="yyyy-MM-dd HH:mm:ss $abbreviatedZoneName"
                $timeStamp=[System.TimeZoneInfo]::ConvertTimeBySystemTimeZoneId([datetime]::UtcNow,$timeZoneName).ToString($timeStampFormat)
                $jobResult=[pscustomobject]@{
                    timeStamp=$timeStamp
                    computerName=$computer
                    stoppedServices='serverTimeout'
                    fixedAutomatically=$False
                    }
                $jobResults+=,$jobResult
                remove-job -id $job.id -force
                $processedCount++
            }
        }

        # Safeguard against stuck jobs
        if($timer.elapsed.totalminutes -ge $maxMinutesPerJob){
            get-job|Remove-Job -Force
            write-warning "There were some errors in this iteration. Shell was aborted to mitigate potential persistency issues."
            exit
        }
    }    
    $minutesElapsed=[math]::round($timer.Elapsed.TotalMinutes,2)
    $timer.stop()
    write-host "$($computerNames.count) computers were processed in $minutesElapsed minutes."
    return $jobResults #|select -property * -excludeproperty RunspaceId
}

maintainProcessParallel $computerNames `
    $processname `
    $processPath `
    $minutesToDefineCrashed `
    $runMinutes `
    $credentials `
    $maxMinutesPerJob `
    $verbose

Leave a Reply

Your email address will not be published. Required fields are marked *