PowerShell: Maintain Linux Services via SSH Remoting

Version 2:

# maintainLinuxServices.ps1
# Version 0.0.2
# Description: this is a simple script to SSH into remote Linux machines to check for running statuses of certain services

$linuxServers=@()
$linuxServers+=[PSCustomObject]@{
    linuxServer='linux1'
    sshUser='rooty'
    sshPass='password'
    sshPort=2323
    serviceNames='httpd'
}
$linuxServers+=[PSCustomObject]@{
    linuxServer='linux2'
    sshUser='rooty'
    sshPass='password'
    sshPort=2882
    serviceNames='mongod','ntpd'
}
$results=@()

function maintainLinuxService{
	param(
		[string]$linuxServer,
		[string]$sshUser,
		[string]$sshPass,
		[int]$sshPort=22,
        [string[]]$serviceNames,
        [bool]$verbose=$true
    )
    $ErrorActionPreference='stop'

    # Set default variables
    $securedPass = ConvertTo-SecureString $sshPass -AsPlainText -Force
	$sshCredentials = New-Object System.Management.Automation.PSCredential($sshUser, $securedPass)
    if(!$serviceNames){$serviceNames='crond'}
    $failedServices=$False
    $inactiveServices=''

    $moduleCommand='New-SSHSession'
    $moduleName='Posh-SSH'
    if(!(get-command $moduleCommand -ea Ignore)){
        if($verbose){write-host "Installing $moduleName..."}
        [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
        if(!(get-packageprovider nuget)){
            $null=Install-PackageProvider -Name NuGet -MinimumVersion 2.8.5.201 -Force -ErrorAction SilentlyContinue
            # Preempt this Nuget installation prompt for user-input
            # NuGet provider is required to continue
            # PowerShellGet requires NuGet provider version '2.8.5.201' or newer to interact with NuGet-based repositories. The NuGet
            # provider must be available in 'C:\Program Files\PackageManagement\ProviderAssemblies' or
            # 'C:\Users\rambo\AppData\Local\PackageManagement\ProviderAssemblies'. You can also install the NuGet provider by running
            # 'Install-PackageProvider -Name NuGet -MinimumVersion 2.8.5.201 -Force'. Do you want PowerShellGet to install and
            # import the NuGet provider now?
            # [Y] Yes  [N] No  [S] Suspend  [?] Help (default is "Y"):  
        }        
        try{
            $null=Install-Module -Name $moduleName -Force -EA SilentlyContinue
        }catch{
            $null=Register-PSRepository -Default
            $null=Install-Module -Name $moduleName -Force -EA SilentlyContinue
            # Error caused by PSRepository being untrusted
            # PackageManagement\Install-Package : No match was found for the specified search criteria and module name 'Posh-SSH'. Try Get-PSRepository to see all available
            # registered module repositories.
            # At C:\Program Files\WindowsPowerShell\Modules\PowerShellGet\1.0.0.1\PSModule.psm1:1772 char:21
            # + ...          $null = PackageManagement\Install-Package @PSBoundParameters
            # +                      ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
            #     + CategoryInfo          : ObjectNotFound: (Microsoft.Power....InstallPackage:InstallPackage) [Install-Package], Exception
            #     + FullyQualifiedErrorId : NoMatchFoundForCriteria,Microsoft.PowerShell.PackageManagement.Cmdlets.InstallPackage
            # The fix:
            # Set-executionPolicy bypass
            # Register-PSRepository -Default
        }
        refreshenv
        if(!(get-command $moduleCommand -ea Ignore)){
            write-warning "$moduleName is still NOT available on $env:computername"
            return $False
        }
    }
    function createSshSession{
        $connected=$false
        $maxRetries=60
        do{
            try{
                $sshSession = New-SSHSession -ComputerName $linuxServer `
                                -Credential $sshCredentials -Port $sshPort `
                                -AcceptKey -KeepaliveInterval 5 -ConnectionTimeout 30 -OperationTimeout 0 
            }catch{
                write-warning $_                
            }
            if($sshSession.Connected){
                if($verbose){write-host "Connected to $linuxServer"}
                $connected=$True
            }elseif($maxRetries-- -eq 0){                    
                $connected=$True
            }else{
                sleep 2
            }
        }while(!$connected)
        return $sshSession
    }
    if(!($sshSession.Connected)){$sshSession=createSshSession}
    if($sshSession.Connected){
        $serviceStatuses=[hashtable]@{}        
        $serviceNames|ForEach-Object{
            $checkService="systemctl is-active -q --no-block --no-ask-password $_ && echo 1"            
            $exitCondition=$false
            $maxRetries=30 # this takes into account where systemctl returns false negatives for certain processes
            do{
                $x=Invoke-SSHCommand -Index $sshSession.sessionid -Command $checkService -EnsureConnection -ea SilentlyContinue
                # The following loop takes into account connection instability where false positives would result (e.g. ExitStatus : 3)
                if(($x.Output|out-string).trim() -eq '1'){
                    $exitCondition=$True
                }elseif($x.ExitStatus -ne 0){                    
                    if($maxRetries-- -eq 0){
                        $exitCondition=$True
                    }
                }else{
                    if(!($sshSession.Connected)){
                        $null=Remove-SSHSession -Index $sshSession.SessionId
                        $sshSession=createSshSession
                    }
                    sleep 1
                }
            }while (!$exitCondition)
            [bool]$serviceIsRunning=($x.Output|out-string).trim() -eq '1'
            # [System.Convert]::ToBoolean($serviceIsRunning) # casting to bool doesn't work for Array types that would be returned by invoke-sshcommand
            $serviceStatuses+=@{
                $_=$serviceIsRunning
            }            
        }
        $inactiveServices=($serviceStatuses.GetEnumerator()|?{$_.Value -eq $false}).Name        
        if($inactiveServices){
            $restartStatuses=[hashtable]@{}            
            if($inactiveServices.count -eq 1){
                $restartService="systemctl start $inactiveServices && systemctl is-active -q --no-block --no-ask-password $inactiveServices && echo 1"
                $exitCondition=$false
                do{
                    $x=Invoke-SSHCommand -Index $sshSession.sessionid -Command $restartService -EnsureConnection -ea SilentlyContinue
                    if($x.ExitStatus -eq 0){
                        $exitCondition=$True
                    }else{
                        if(!($sshSession.Connected)){
                            $null=Remove-SSHSession -Index $sshSession.SessionId
                            $sshSession=createSshSession
                        }
                        sleep 1
                    }
                }while (!$exitCondition)
                [bool]$serviceIsStarted=($x.Output|out-string).trim() -eq '1'
                #[bool]$serviceIsStarted=!(!(Invoke-SSHCommand -Index $sshSession.sessionid -Command $restartService -EnsureConnection).Output -eq 1)
                if(!$serviceIsStarted){
                    $restartStatuses=@{$inactiveServices=$serviceIsStarted}
                }
            }else{
                $inactiveServices.GetEnumerator()|%{
                    $restartService="systemctl start $_ && systemctl is-active -q --no-block --no-ask-password $_ && echo 1"
                    $exitCondition=$false
                    do{
                        $x=Invoke-SSHCommand -Index $sshSession.sessionid -Command $restartService -EnsureConnection -ea SilentlyContinue
                        if($x.ExitStatus -eq 0){
                            $exitCondition=$True
                        }else{
                            if(!($sshSession.Connected)){
                                $null=Remove-SSHSession -Index $sshSession.SessionId
                                $sshSession=createSshSession
                            }
                            sleep 1
                        }
                    }while (!$exitCondition)
                    [bool]$serviceIsStarted=($x.Output|out-string).trim() -eq '1'
                    #[bool]$serviceIsStarted=!(!(Invoke-SSHCommand -Index $sshSession.sessionid -Command $restartService -EnsureConnection).Output -eq 1)
                    if(!$serviceIsStarted){
                        $restartStatuses+=@{$_=$serviceIsStarted}
                    }
                }
            }
            $failedServices=($restartStatuses.GetEnumerator()|?{$_.Value -eq $false}).Name
        }elseif($verbose){
            write-host "$($serviceNames -join ', ') on $linuxServer are running as normal" -ForegroundColor Green
        }
        # This destructor is important for subsequent runs
        if($sshSession.Connected){$null=Remove-SSHSession -Index $sshSession.SessionId}
        $null=Get-SSHSession|Remove-SSHSession
        $timeZoneName=[System.TimeZoneInfo]::Local.StandardName
        $abbreviatedZoneName=if($timeZoneName -match ' '){[regex]::replace($timeZoneName,'([A-Z])\w+\s*', '$1')}else{$timeZoneName}
        $timeStampFormat="yyyy-MM-dd HH:mm:ss $abbreviatedZoneName"
        $timeStamp=[System.TimeZoneInfo]::ConvertTimeBySystemTimeZoneId([datetime]::UtcNow,$timeZoneName).ToString($timeStampFormat)        
        return [pscustomobject]@{
            timeStamp=$timeStamp
            computername=$linuxServer
            stoppedServices=if($inactiveServices){$inactiveServices}else{'None'}
            fixedAutomatically=if($failedServices){$false}else{$true}
            }
    }else{
        write-warning "Couldn't connecto to $linuxServer"
        $timeZoneName=[System.TimeZoneInfo]::Local.StandardName
        $abbreviatedZoneName=if($timeZoneName -match ' '){[regex]::replace($timeZoneName,'([A-Z])\w+\s*', '$1')}else{$timeZoneName}
        $timeStampFormat="yyyy-MM-dd HH:mm:ss $abbreviatedZoneName"
        $timeStamp=[System.TimeZoneInfo]::ConvertTimeBySystemTimeZoneId([datetime]::UtcNow,$timeZoneName).ToString($timeStampFormat)        
        return [pscustomobject]@{
            timeStamp=$timeStamp
            computername=$linuxServer
            stoppedServices='Unknown'
            fixedAutomatically='Unknown'
            }
    } 
}

$linuxResults=$linuxServers|%{maintainLinuxService $_.linuxServer $_.sshUser $_.sshPass $_.sshPort $_.serviceNames $true }
$linuxResults|select-object -Property timeStamp,computerName,stoppedServices,fixedAutomatically

Version 1:

# maintainLinuxServices.ps1
# Version 0.0.1

$linuxServer='IPADDRESSHERE'
$sshUser='USERNAME'
$sshPass='PASSWORD'
$sshPort=22
$serviceName='proftpd'
$checkService="systemctl is-active --quiet $serviceName && echo True"
$restartService="systemctl start $serviceName && systemctl is-active --quiet $serviceName && echo True"

function maintainLinuxService{
	param(
		$linuxServer,
		$sshUser,
		$sshPass,
		$sshPort=22,
		$serviceName,
		$checkService,
		$restartService
    )
    $ErrorActionPreference='stop'
    if(!$serviceName){$serviceName='crond'}
    if(!$checkService){$checkService="systemctl is-active --quiet $serviceName && echo True"}
    if(!$restartService){$restartService="systemctl start $serviceName && systemctl is-active --quiet $serviceName && echo True"}
    $allServicesRunning=$true
    
    $moduleCommand='New-SSHSession'
    $moduleName='Posh-SSH'
    if(!(get-command $moduleCommand -ea Ignore)){
        write-host "Installing $moduleName..."
        [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
        if(!(get-packageprovider nuget)){
            $null=Install-PackageProvider -Name NuGet -MinimumVersion 2.8.5.201 -Force -ErrorAction SilentlyContinue
            # Preempt this Nuget installation prompt for user-input
            # NuGet provider is required to continue
            # PowerShellGet requires NuGet provider version '2.8.5.201' or newer to interact with NuGet-based repositories. The NuGet
            # provider must be available in 'C:\Program Files\PackageManagement\ProviderAssemblies' or
            # 'C:\Users\rambo\AppData\Local\PackageManagement\ProviderAssemblies'. You can also install the NuGet provider by running
            # 'Install-PackageProvider -Name NuGet -MinimumVersion 2.8.5.201 -Force'. Do you want PowerShellGet to install and
            # import the NuGet provider now?
            # [Y] Yes  [N] No  [S] Suspend  [?] Help (default is "Y"):  
        }        
        try{
            $null=Install-Module -Name $moduleName -Force -EA SilentlyContinue
        }catch{
            $null=Register-PSRepository -Default
            $null=Install-Module -Name $moduleName -Force -EA SilentlyContinue
            # Error caused by PSRepository being untrusted
            # PackageManagement\Install-Package : No match was found for the specified search criteria and module name 'Posh-SSH'. Try Get-PSRepository to see all available
            # registered module repositories.
            # At C:\Program Files\WindowsPowerShell\Modules\PowerShellGet\1.0.0.1\PSModule.psm1:1772 char:21
            # + ...          $null = PackageManagement\Install-Package @PSBoundParameters
            # +                      ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
            #     + CategoryInfo          : ObjectNotFound: (Microsoft.Power....InstallPackage:InstallPackage) [Install-Package], Exception
            #     + FullyQualifiedErrorId : NoMatchFoundForCriteria,Microsoft.PowerShell.PackageManagement.Cmdlets.InstallPackage
            # The fix:
            # Set-executionPolicy bypass
            # Register-PSRepository -Default
        }
        refreshenv
        if(!(get-command $moduleCommand -ea Ignore)){
            write-warning "$moduleName is still NOT available on $env:computername"
            return $False
        }
    }	
	$securedPass = ConvertTo-SecureString $sshPass -AsPlainText -Force
	$sshCredentials = New-Object System.Management.Automation.PSCredential($sshUser, $securedPass)
    try{
        $sshSession = New-SSHSession -ComputerName $linuxServer -Credential $sshCredentials -Port $sshPort -AcceptKey -ConnectionTimeout 5
    }catch{
        write-warning $_
        return $false
    }
    $serviceStatus=[System.Convert]::ToBoolean((Invoke-SSHCommand -Index $sshSession.sessionid -Command $checkService).Output)
	if(!$serviceStatus){
		$allServicesRunning=[System.Convert]::ToBoolean((Invoke-SSHCommand -Index $sshSession.sessionid -Command $restartService).Output)
		if($allServicesRunning){
            write-host "$serviceName on $sshServer has restarted successfully" -foregroundcolor green
		}else{
			write-warning "$serviceName on $sshServer cannot be started"
		}
	}
	$null=Remove-SSHSession -Index $sshSession.SessionId
	return [pscustomobject]@{
		computername=$linuxServer
		stoppedServices=$serviceStatus
		allServicesNowRunning=$allServicesRunning
		}
}

maintainLinuxService $linuxServer $sshUser $sshPass $sshPort $serviceName $checkService $restartService

Leave a Reply

Your email address will not be published. Required fields are marked *