PowerShell: Invoke CRM Services Maintenance

Version 4:

# invokeCrmServersMaintenance.ps1
# Version 0.0.4
# This script is a processes watcher on Microsoft Dynamics CRM Servers with these routines:
# - Multi-threading to process multiple nodes concurrently to improve efficiency
# - Ensures that all automatically starting services are running
# - Handles of certain special services: 'MSCRMAsyncService$maintenance', 'OneSyncSvc_*'
# - Detects name resolution issues
# - Check for any custom scheduled tasks for signs of failures

# Obtain credentials being passed by automation engines such as Jenkins
$username=$env:username
$password=$env:password
$encryptedPassword=ConvertTo-SecureString $password -AsPlainText -Force
$credentials = New-Object -TypeName System.Management.Automation.PSCredential -ArgumentList $userName,$encryptedPassword;
 
# List of servers to check
$computerNames=@(
    'CRM-WEB01',
    'CRM-SQL01',
    'CRM-DEV01'
)
 
# Email relay parameters
$emailFrom='admin@kimconnect.com'
$emailTo='webAdmins@kimconnect.com'
$subject='CRM Server Issues'
$smtpRelayServer='smtp.office365.com'

# Other Variables
$skipServices=@('TrustedInstaller','BITS','gupdate','MapsBroker','RemoteRegistry','sppsvc','WbioSrvc','twdservice','MSCRMAsyncService$maintenance','CDPSvc')
$csvFile='C:\scripts\logs\serverIssues.csv'
$limitEmailRecords=100

function invokeCrmServersMaintenance{
    param(
        [string[]]$computerNames=$env:computername,
        [pscredential]$credentials,
        [string[]]$skipServices=@('TrustedInstaller','gupdate','MapsBroker','RemoteRegistry','sppsvc','WbioSrvc','twdservice','MSCRMAsyncService$maintenance'),
        [string]$desiredStatus='running'
    )
    $results=@()
    $emailRouterServiceName='MSCRMEmail'
    $asyncServiceName='MSCRMAsyncService$maintenance'
    $targetAsyncServices='MSCRMAsyncService$maintenance','MSCRMAsyncService'
    function checkService($serviceName,$status='Running'){
        # Sanitation
        #$systemInvalidChars =[Regex]::Escape(-join [System.Io.Path]::GetInvalidFileNameChars())
        #$regexInvalidChars = "[$systemInvalidChars]"
        #$serviceName=$serviceName -replace [regex]::Matches($serviceName, $regexInvalidChars, 'IgnoreCase').Value
        try{
            $service=get-service -name $serviceName -erroraction 'silentlycontinue'|select -first 1
            if($service.Status -eq $status){
                # write-host "$serviceName status of $status is matching the desired state." -foregroundcolor Green
                return 0
            }elseif($null -ne $service.Status){
                write-host "$env:computername`: $serviceName status $($service.Status) doesn`'t match the desired state" -foregroundcolor Red
                return 1
            }else{
                #write-verbose "$serviceName was not found" -foregroundcolor Red
                return -1          
            }
        }catch{
            #Write-verbose $_
            #write-verbose "$serviceName was not found" -foregroundcolor Red
            return -1
        }
    }

    function getFailedScheduledTasks{
        param(
            $excludedPaths='^\\Microsoft|Mozilla|Integrations\\',
            $excludedTaskResults=@(
                0, # success
                267009, # running
                267010, # disabled
                267011, # not yet ran
                267012, # There are no more runs scheduled for this task
                267014, # The last run of the task was terminated by the user
                267015, # Either the task has no triggers or the existing triggers are disabled or not set
                2147750687, # An instance of this task is already running
                3221225786, # The application terminated as a result of a CTRL+C
                1073807364, # 40010004 (hex). This means the system cannot open a file 4001 is the facility code. This can safely be ignored as it pertains to CreateExplorerShellUnelevatedTask 
                2147943517 # Firefox Default Browser Agent
                ),
            $excludedStates=@('Disabled','Running')
        )
        $windowsVersion=[Environment]::OSVersion.Version
        if($windowsVersion -ge [version]'6.2'){
          # This function requires Windows 8 / Server 2012 (build 9200) or higher
          # Get-ScheduledTask : The term 'Get-ScheduledTask' is not recognized as the name of a cmdlet, function, script file, or
          # operable program. Check the spelling of the name, or if a path was included, verify that the path is correct and try
          # again.
          # At line:1 char:14
          # + $customTasks=Get-ScheduledTask|?{ $_.State -ne "Disabled" -and $_.Tas ...
          # +              ~~~~~~~~~~~~~~~~~
          #     + CategoryInfo          : ObjectNotFound: (Get-ScheduledTask:String) [], CommandNotFoundException
          #     + FullyQualifiedErrorId : CommandNotFoundException
          # $knownStates=@('Unknown','Disabled','Queued','Ready','Running')
          # Task status codes: https://docs.microsoft.com/en-us/windows/win32/taskschd/task-scheduler-error-and-success-constants
          
          # Common Task Result Codes (int32):
          # 0 - The operation completed successfully.
          # 1 - Incorrect function called or unknown function called.
          # 10 - The environment is incorrect.
          # 267008 - Task is ready to run at its next scheduled time.
          # 267009 - Task is currently running.
          # 267010 - The task will not run at the scheduled times because it has been disabled.
          # 267011 - Task has not yet run.
          # 267012 - There are no more runs scheduled for this task.
          # 267013 - One or more of the properties that are needed to run this task on a schedule have not been set.
          # 267014 - The last run of the task was terminated by the user.
          # 267015 - Either the task has no triggers or the existing triggers are disabled or not set.
          # 2147750671 - Credentials became corrupted.
          # 2147750687 - An instance of this task is already running.
          # 2147943645 - The service is not available (is "Run only when an user is logged on" checked?).
          # 3221225786 - The application terminated as a result of a CTRL+C.
          # 3228369022 - Unknown software exception.          
          $customTasks=Get-ScheduledTask|?{ $_.State -notin $excludedStates -and $_.TaskPath -notmatch $excludedPaths}
          $failedTasks=$customTasks|Get-ScheduledTaskInfo|?{$_.LastTaskResult -notin $excludedTaskResults}
          if($failedTasks){
            return $failedTasks|Select-Object -Property * -ExcludeProperty PSComputerName,CimClass,CimInstanceProperties,CimSystemProperties
          }
        }
    }
      
    function startAllAutoServices{
        param(
            [string[]]$skipServices=@('TrustedInstaller','gupdate','MapsBroker','RemoteRegistry','sppsvc','WbioSrvc')
            )        
        $filterString=($skipServices|%{"AND name!='$_' "}) -join ''
        $stoppedServices=Get-WmiObject win32_service -Filter "startmode='auto' AND state!='running' $filterString"
        if($stoppedServices){
            write-warning "$env:computername`: these services were not running $($stoppedServices.Name).`r`nProgram now attempts to start them."
            $null=$stoppedServices|Invoke-WmiMethod -Name StartService
            $null=(get-service $stoppedServices.Name).waitforstatus('Running')
            # $timeoutSeconds=45
            # $timeSpan = New-Object Timespan 0,0,$timeoutSeconds
            # foreach($service in $stoppedServices){
            #     $service = Get-Service $service.Name
            #     $service.Start()
            #     $service.WaitForStatus([ServiceProcess.ServiceControllerStatus]::Running, $timeSpan)
            # }
            # Detect failed services
            $failedServices=Get-WmiObject win32_service -Filter "startmode = 'auto' AND state != 'running' $filterString"
            if($failedServices){
                write-warning "$env:computername`: failed to start these services $($failedServices.Name)"
            }else{
                write-host "$env:computername`: all auto-start services are running." -ForegroundColor Green
            }
            $excludedOneSync=$stoppedServices|?{$_.Name -notlike 'OneSyncSvc*'}
            $filteredServices=if($excludedOneSync){($excludedOneSync.Name -join ','|out-string).trim()}else{'None'}
            return [pscustomobject]@{
                stoppedServices=$filteredServices
                fixedAutomatically=if($failedServices){$false}else{$true}
            }
        }else{
            write-host "$env:computername`: all auto-start services are running." -ForegroundColor Green
            return [pscustomobject]@{
                stoppedServices='none'
                fixedAutomatically=$true
            }
        }        
    } 

    $fixAsyncService={
        $erroractionpreference='continue'
        $targetAsyncServices='MSCRMAsyncService$maintenance','MSCRMAsyncService'
        try{
            $crmTools=if(test-path "$env:programfiles\dynamics 365"){
                    "$env:programfiles\dynamics 365\Tools"
                }else{"$env:programfiles\Microsoft Dynamics CRM\Tools"}            
            $asyncServices=get-service|?{$_.name -like 'MSCRMAsyncService*'}
            $stoppedAsync=$asyncServices|?{$_.Status -eq 'Stopped'}
            if($stoppedAsync){
                write-host "Restarting services: $($asyncServices.Name)"
                $stoppedAsync|Start-Service
                write-host 'Renewing keys to CRM App server...'
                Start-Process -Wait -FilePath cmd -Verb RunAs -ArgumentList '/c',"`"$crmTools\Microsoft.Crm.Tools.WRPCKeyRenewal.exe`" /R"
            }
            #cmd /c "`"$crmTools\Microsoft.Crm.Tools.WRPCKeyRenewal.exe`" /R" # alternate method of calling cmd from Powershell with switches
            # cmd /c "`"C:\Windows\system32\PING.EXE`" -n 10 google.com"
            # comment-out because $lastexitcode does not register properly within WinRM sessions
            # if($LASTEXITCODE -ne 0){
            #     write-host 'Microsoft.Crm.Tools.WRPCKeyRenewal.exe command failed.'
            #     return $false
            #   }
        }catch{
            write-host $_
        }
        return !(get-service $stoppedAsync.Name|?{$_.status -eq 'Stopped'})
    }

    $fixEmailRouterService={        
        $erroractionpreference='stop'
        $emailRouterServiceName='MSCRMEmail'
        $expectedXmlLocation='C:\Program Files\Microsoft CRM Email\Service\Microsoft.Crm.Tools.EmailAgent.SystemState.xml'
        $alreadyRunning=(get-service $emailRouterServiceName).Status -eq 'Running'
        if(!$alreadyRunning){
            try{
                $emailRouterXmlFile=if(test-path $expectedXmlLocation){
                        $expectedXmlLocation
                    }else{
                        "${env:ProgramFiles(x86)}\Microsoft CRM Email\Service\Microsoft.Crm.Tools.EmailAgent.SystemState.xml"
                    }
                write-host "Attempting to start email router service normally..."
                start-service $emailRouterServiceName
                $isRunning=(get-service $emailRouterServiceName).Status -eq 'Running'
                if(!$isRunning){
                    write-host 'Fixing Microsoft Dynamics CRM Email Router Services...'            
                    $null=stop-service $emailRouterServiceName -force
                    $null=rename-item $emailRouterXmlFile "$originalFileName.bak" -force
                    start-service $emailRouterServiceName
                }
                $isRunning=(get-service $emailRouterServiceName).Status -eq 'Running'
                if($isRunning){
                    return $true
                }else{
                    return $false
                }                
            }catch{
                write-host $_
                return $false
            }
        }else{
            write-host "$emailRouterServiceName is already running" -ForegroundColor Green
        }
    }

    foreach ($computerName in $computerNames){
        $sessionTimeout=New-PSSessionOption -OpenTimeout 120000 # 2 minutes
        $sessionIncludePort=New-PSSessionOption -IncludePortInSPN -OpenTimeout 120000
        $session=if($credentials){
                try{
                    New-PSSession -ComputerName $computername -Credential $credentials -ea Stop -SessionOption $sessionTimeout
                }catch{
                    New-PSSession -ComputerName $computername -Credential $credentials -SessionOption $sessionIncludePort
                }
            }else{
                try{
                    New-PSSession -ComputerName $computername -ea Stop -SessionOption $sessionTimeout
                }catch{
                    New-PSSession -ComputerName $computername -SessionOption $sessionIncludePort
                }
            }        
        if($session.state -eq 'Opened'){
            $asyncServiceStatusCode=invoke-command -Session $session -ScriptBlock{
                param ($checkService,[array]$asyncServiceName)
                foreach ($x in $asyncServiceName){
                    $null=$statusCode
                    $statusCode=[scriptblock]::create($checkService).invoke($x)
                    if($statusCode){
                        return $statusCode
                    }
                }
                return $statusCode
                } -args ${function:checkService},$targetAsyncServices
            if($asyncServiceStatusCode -eq 1){
                    invoke-command -session $session -scriptblock $fixAsyncService
                }

            $emailRouterServiceStatusCode=invoke-command -Session $session -ScriptBlock{
                param ($checkService,$emailRouterServiceName)
                [scriptblock]::create($checkService).invoke($emailRouterServiceName)
                } -args ${function:checkService},'MSCRMEmail'
            if($emailRouterServiceStatusCode -eq 1){
                $null=invoke-command -session $session -scriptblock $fixEmailRouterService
                }

            $otherServicesResult=invoke-command -Session $session -ScriptBlock{
                param ($startAllAutoServices,$skipServices)
                [scriptblock]::create($startAllAutoServices).invoke($skipServices)
                } -args ${function:startAllAutoServices},$skipServices
            
            if($asyncServiceStatusCode -eq 1){
                if($otherServicesResult.stoppedServices -ne 'none'){
                    $otherServicesResult.stoppedServices+=','+$asyncServiceName
                }else{
                    $otherServicesResult.stoppedServices=$asyncServiceName
                }
            }
            if($emailRouterServiceStatusCode -eq 1){
                if($otherServicesResult.stoppedServices -ne 'none'){
                    $otherServicesResult.stoppedServices+=','+'MSCRMEmail'
                }else{
                    $otherServicesResult.stoppedServices='MSCRMEmail'
                }
            }
            $dnsOK=invoke-command -Session $session -ScriptBlock{
                # test-connection google.com -Count 1 -Quiet # This sometimes will freeze the session
                $pingResult=ping google.com -n 1
                return [bool]($pingResult -match '0% loss')
            }
            $failedScheduledTasks=invoke-command -Session $session -ScriptBlock{
                param($x)
                [scriptblock]::create($x).invoke()
            } -Args ${function:getFailedScheduledTasks}
            Remove-PSSession $session
            $timeZoneName=[System.TimeZoneInfo]::Local.StandardName
            $abbreviatedZoneName=if($timeZoneName -match ' '){[regex]::replace($timeZoneName,'([A-Z])\w+\s*', '$1')}else{$timeZoneName}
            $timeStampFormat="yyyy-MM-dd HH:mm:ss $abbreviatedZoneName"
            $timeStamp=[System.TimeZoneInfo]::ConvertTimeBySystemTimeZoneId([datetime]::UtcNow,$timeZoneName).ToString($timeStampFormat)
            $otherStoppedServices=$otherServicesResult.stoppedServices
            $otherStoppedServices=.{
                $x=$otherStoppedServices|?{$_ -ne 'none'}
                if(!$dnsOK){
                    $x+=,'dnsClient'
                }
                if($failedScheduledTasks){
                    $x+=,"FailedScheduleTasks: $($failedScheduledTasks.TaskName -join ',')"
                }
                return $x
            }
            $stoppedServices=($otherStoppedServices|out-string).trim()
            $results+=[pscustomobject]@{
                timeStamp=$timeStamp
                computerName=$computerName
                stoppedServices=if($stoppedServices){$stoppedServices}else{'None'}
                fixedAutomatically=if(!$failedScheduledTasks -and $dnsOK -and $otherServicesResult.fixedAutomatically){$True}else{$False}
                }
        }else{
            write-warning "$env:computername cannnot connect to $computername via WinRM"
            $timeZoneName=[System.TimeZoneInfo]::Local.StandardName
            $abbreviatedZoneName=if($timeZoneName -match ' '){[regex]::replace($timeZoneName,'([A-Z])\w+\s*', '$1')}else{$timeZoneName}
            $timeStampFormat="yyyy-MM-dd HH:mm:ss $abbreviatedZoneName"
            $timeStamp=[System.TimeZoneInfo]::ConvertTimeBySystemTimeZoneId([datetime]::UtcNow,$timeZoneName).ToString($timeStampFormat)
            $results+=[pscustomobject]@{
                timeStamp=$timeStamp
                computerName=$computerName
                stoppedServices='Unknown'
                fixedAutomatically='Unknown'
                }
        }
    }
    return $results    
}

function checkCrmServices($computerNames,$credentials,$skipServices,$verbose=$true){
    $timer=[System.Diagnostics.Stopwatch]::StartNew()
    $jobResults=@()
    $lineBreak=60
    $dotCount=0
    $minute=0
    $processorsCount=(Get-CimInstance Win32_ComputerSystem).NumberOfLogicalProcessors
    $cpuLoad=(Get-WmiObject win32_processor|Measure-Object -property LoadPercentage -Average).Average
    $maxSimultaneousJobs=if($cpuLoad -gt 90){$processorsCount}else{($processorsCount*2)-1} # dynamically limiting concurrent jobs basing on available CPU cores
    write-host "CPU load detected as: $cpuLoad`%`r`nSetting concurrent jobs max count to be $maxSimultaneousJobs"
    foreach($computerName in $computerNames){
        $thisIterationCompleted=$false
        do {
            $jobsCount=(Get-Job -State 'Running').Count
            if ($jobsCount -lt $maxSimultaneousJobs){            
                if($verbose){write-host "Initiating job for $computerName"}
                $null=Start-Job -name $computerName -ScriptBlock {
                    param($invokeCrmServersMaintenance,$computerName,$credentials,$skipServices)
                    [scriptblock]::create($invokeCrmServersMaintenance).invoke($computerName,$credentials,$skipServices)
                } -Args ${function:invokeCrmServersMaintenance},$computerName,$credentials,$skipServices
                $thisIterationCompleted=$true
            }else{
                if($verbose){
                    if($dotCount++ -lt $lineBreak){
                        write-host '.' -NoNewline
                    }else{
                        $minute++
                        write-host "`r`n$minute`t:" -ForegroundColor Yellow -NoNewline
                        $dotCount=0
                        }
                }
                sleep -seconds 1
            }
        }until ($thisIterationCompleted)
    }
    $totalJobsCount=(get-job).count
    $processedCount=0
    while($processedCount -lt $totalJobsCount){
        $completedJobs=get-job|?{$_.State -eq 'Completed'}
        if($completedJobs){
            foreach ($job in $completedJobs){
                $computer=$job.Name
                if($verbose){
                    write-host "`r`n===================================================`r`n$computer job completed with these messages:`r`n===================================================`r`n"
                }
                $jobResult=receive-job -id $job.id
                $jobResults+=,$jobResult
                remove-job -id $job.id -force
                $processedCount++
            }
        }
    }
    $minutesElapsed=[math]::round($timer.Elapsed.TotalMinutes,2)
    $timer.stop()
    write-host "$($computerNames.count) computers were processed in $minutesElapsed minutes."
    return $jobResults #|select -property * -excludeproperty RunspaceId
}

$results=checkCrmServices $computerNames $credentials $skipServices $false|select-object -Property timeStamp,computername,stoppedServices,fixedAutomatically

write-host $($results|ft|out-string).trim()
$problemsDetected=$results|?{$_.stoppedServices -ne 'none'}
if($problemsDetected){
    $previousProblems=import-csv -path $csvFile|select -last $problemsDetected.computerName.count|select -Property * -ExcludeProperty timeStamp
    if(!(test-path $csvFile)){
        if(!(test-path $(split-path $csvFile -parent))){mkdir $(split-path $csvFile -parent) -force}
        $header='"timeStamp","computerName","stoppedServices","fixedAutomatically"'
        Add-Content -Path $csvFile  -Value $header
    }
    $problemsDetected|Export-Csv -Path $csvFile -Append
    $problemsDetectedReformatted=$problemsDetected|ConvertTo-Csv|convertFrom-csv|select -Property * -ExcludeProperty timeStamp
    $sameProblemAsPrior=!(Compare-Object $problemsDetectedReformatted.PSObject.Properties $previousProblems.PSObject.Properties)
    if(!$sameProblemAsPrior){
        $css="
        <style>
        .h1 {
            font-size: 18px;
            height: 40px;
            padding-top: 80px;
            margin: auto;
            text-align: center;
        }
        .h5 {
            font-size: 22px;
            text-align: center;
        }
        .th {text-align: center;}
        .table {
            padding:7px;
            border:#4e95f4 1px solid;
            background-color: white;
            margin-left: auto;
            margin-right: auto;
            width: 100%
            }
        .colgroup {}
        .th { background: #0046c3; color: #fff; padding: 5px 10px; }
        .td { font-size: 11px; padding: 5px 20px; color: #000;
              width: 1px;
              white-space: pre;
            }
        .tr { background: #b8d1f3;}
        .tr:nth-child(even) {
            background: #dae5f4;
            width: 1%;
            white-space: nowrap
        }
        .tr:nth-child(odd) {
            background: #b8d1f3;
            width: 1%;
            white-space: nowrap
        }
        </style>
        "
        $currentReport=$problemsDetected | ConvertTo-Html -Fragment | Out-String
        $historicalReport=Import-CSV $csvFile|sort timeStamp -Descending|select-object -first $limitEmailRecords|ConvertTo-Html -fragment|Out-String
        $currentReportHtml=$currentReport -replace '\<(?<item>\w+)\>', '<${item} class=''${item}''>'
        $historicalReportHtml=$historicalReport -replace '\<(?<item>\w+)\>', '<${item} class=''${item}''>'
        $emailContent='<html><head>'+$css+"</head><body><h5 class='h5'>Current Server Errors</h5>"+$currentReportHtml+"<h5 class='h5'>Historical Server Errors</h5>"+$historicalReportHtml+'</body></html>'
        write-host "Updating the report html file: $csvFile.html"
        $null='<html><head>'+$css+"</head><body><h5 class='h5'>Server Errors</h5>"+$historicalReportHtml+'</body></html>' | Out-File "$csvFile.html"
    
        Send-MailMessage -From $emailFrom `
        -To $emailTo `
        -Subject $subject `
        -Body $emailContent `
        -BodyAsHtml `
        -SmtpServer $smtpRelayServer
    }else{
        write-host "Current issue is same as before. Email sending is skipped"
    }
}

Version 3:

# invokeCrmServersMaintenance.ps1
# Version 0.0.3
# This script is a processes watcher on Microsoft Dynamics CRM Servers with these routines:
# - Ensures that all automatically starting services are running
# - Handles of certain special services: 'MSCRMAsyncService$maintenance', 'OneSyncSvc_*'
# - Detects name resolution issues

# Obtain credentials being passed by automation engines such as Jenkins
$username=$env:username
$password=$env:password
$encryptedPassword=ConvertTo-SecureString $password -AsPlainText -Force
$credentials = New-Object -TypeName System.Management.Automation.PSCredential -ArgumentList $userName,$encryptedPassword;
 
# List of servers to check
$computerNames=@(
    'CRM-WEB01',
    'CRM-SQL01',
    'CRM-DEV01'
)
 
# Email relay parameters
$emailFrom='admin@kimconnect.com'
$emailTo='webAdmins@kimconnect.com'
$subject='CRM Server Issues'
$smtpRelayServer='smtp.office365.com'

# Services to skip
$skipServices=@('TrustedInstaller','BITS','gupdate','MapsBroker','RemoteRegistry','sppsvc','WbioSrvc','twdservice','MSCRMAsyncService$maintenance','CDPSvc')
$logFile='C:\scripts\logs\ServersMaintenanceIssues.csv'
$limitEmailRecords=100

function invokeCrmServersMaintenance{
    param(
        [string[]]$computerNames=$env:computername,
        [pscredential]$credentials,
        [string[]]$skipServices=@('TrustedInstaller','gupdate','MapsBroker','RemoteRegistry','sppsvc','WbioSrvc','twdservice','MSCRMAsyncService$maintenance'),
        [string]$desiredStatus='running'
    )
    $results=@()
    $emailRouterServiceName='MSCRMEmail'
    $asyncServiceName='MSCRMAsyncService$maintenance'
    function checkService($serviceName,$status='Running'){
        # Sanitation
        #$systemInvalidChars =[Regex]::Escape(-join [System.Io.Path]::GetInvalidFileNameChars())
        #$regexInvalidChars = "[$systemInvalidChars]"
        #$serviceName=$serviceName -replace [regex]::Matches($serviceName, $regexInvalidChars, 'IgnoreCase').Value
        try{
            $service=get-service -name $serviceName -erroraction 'silentlycontinue'|select -first 1
            if($service.Status -eq $status){
                write-host "$serviceName status of $status is matching the desired state." -foregroundcolor Green
                return 0
            }elseif($null -ne $service.Status){
                write-host "$env:computername`: $serviceName status $($service.Status) doesn`'t match the desired state" -foregroundcolor Red
                return 1
            }else{
                #write-verbose "$serviceName was not found" -foregroundcolor Red
                return -1          
            }
        }catch{
            #Write-verbose $_
            #write-verbose "$serviceName was not found" -foregroundcolor Red
            return -1
        }
    }

    function startAllAutoServices{
        param(
            [string[]]$skipServices=@('TrustedInstaller','gupdate','MapsBroker','RemoteRegistry','sppsvc','WbioSrvc')
            )        
        $filterString=($skipServices|%{"AND name!='$_' "}) -join ''
        $stoppedServices=Get-WmiObject win32_service -Filter "startmode='auto' AND state!='running' $filterString"
        if($stoppedServices){
            write-warning "$env:computername`: these services were not running $($stoppedServices.Name).`r`nProgram now attempts to start them."
            $null=$stoppedServices|Invoke-WmiMethod -Name StartService
            (get-service $stoppedServices.Name).waitforstatus('Running')
            # Detect failed services
            $failedServices=Get-WmiObject win32_service -Filter "startmode = 'auto' AND state != 'running' $filterString"
            if($failedServices){
                write-warning "$env:computername`: failed to start these services $($failedServices.Name)"
            }else{
                write-host "$env:computername`: all auto-start services are running." -ForegroundColor Green
            }
            $excludeOneSync=$stoppedServices|?{$_.Name -notlike 'OneSyncSvc*'}
            $filteredServices=if($excludeOneSync){$excludeOneSync}else{'None'}
            return [pscustomobject]@{
                stoppedServices=($filteredServices.Name -join ','|out-string).trim()
                fixedAutomatically=if($failedServices){$false}else{$true}
            }
        }else{
            write-host "$env:computername`: all auto-start services are running." -ForegroundColor Green
            return [pscustomobject]@{
                stoppedServices='none'
                fixedAutomatically=$true
            }
        }        
    } 

    $fixAsyncService={
        $erroractionpreference='stop'
        $asyncServiceName='MSCRMAsyncService$maintenance'
        try{
            $crmTools=if(test-path "$env:programfiles\dynamics 365"){
                    "$env:programfiles\dynamics 365\Tools"
                }else{"$env:programfiles\Microsoft Dynamics CRM\Tools"}            
            $asyncServices=get-service|?{$_.name -like 'MSCRMAsyncService*'}
            write-host "Restarting services: $($asyncServices.Name)"
            restart-service $asyncServices
            write-host 'Renewing keys to CRM App server...'
            Start-Process -Wait -FilePath cmd -Verb RunAs -ArgumentList '/c',"`"$crmTools\Microsoft.Crm.Tools.WRPCKeyRenewal.exe`" /R"
            #cmd /c "`"$crmTools\Microsoft.Crm.Tools.WRPCKeyRenewal.exe`" /R" # alternate method of calling cmd from Powershell with switches
            # cmd /c "`"C:\Windows\system32\PING.EXE`" -n 10 google.com"
            # comment-out because $lastexitcode does not register properly within WinRM sessions
            # if($LASTEXITCODE -ne 0){
            #     write-host 'Microsoft.Crm.Tools.WRPCKeyRenewal.exe command failed.'
            #     return $false
            # }else{
                sleep 1
                write-host 'Restarting Microsoft Dynamics CRM Asynchronous Service (maintenance)...'
                restart-service $asyncServiceName
                return $true
            #    }
        }catch{
            write-host $_
            return $false
        }
    }

    $fixEmailRouterService={        
        $erroractionpreference='stop'
        $emailRouterServiceName='MSCRMEmail'
        $expectedXmlLocation='C:\Program Files\Microsoft CRM Email\Service\Microsoft.Crm.Tools.EmailAgent.SystemState.xml'
        try{
            $emailRouterXmlFile=if(test-path $expectedXmlLocation){
                    $expectedXmlLocation
                }else{
                    "${env:ProgramFiles(x86)}\Microsoft CRM Email\Service\Microsoft.Crm.Tools.EmailAgent.SystemState.xml"
                }
            write-host "Attempting to start email router service normally..."
            start-service $emailRouterServiceName
            $isRunning=(get-service $emailRouterServiceName).Status -eq 'Running'
            if(!$isRunning){
                write-host 'Fixing Microsoft Dynamics CRM Email Router Services...'            
                stop-service $emailRouterServiceName -force
                $null=rename-item $emailRouterXmlFile "$originalFileName.bak" -force
                start-service $emailRouterServiceName
            }
            return $true
        }catch{
            write-host $_
            return $false
        }
    }

    foreach ($computerName in $computerNames){
        $session=if($credentials){
                try{
                    New-PSSession -ComputerName $computername -Credential $credentials -ea Stop
                }catch{
                    New-PSSession -ComputerName $computername -Credential $credentials -SessionOption $(new-pssessionoption -IncludePortInSPN)
                }
            }else{
                try{
                    New-PSSession -ComputerName $computername -ea Stop
                }catch{
                    New-PSSession -ComputerName $computername -SessionOption $(new-pssessionoption -IncludePortInSPN)
                }
            }        
        if($session.state -eq 'Opened'){
            $asyncServiceStatusCode=invoke-command -Session $session -ScriptBlock{
                param ($checkService,$asyncServiceName)
                [scriptblock]::create($checkService).invoke($asyncServiceName)
                } -args ${function:checkService},$asyncServiceName
            if($asyncServiceStatusCode -eq 1){
                    invoke-command -session $session -scriptblock $fixAsyncService
                }

            $emailRouterServiceStatusCode=invoke-command -Session $session -ScriptBlock{
                param ($checkService,$emailRouterServiceName)
                [scriptblock]::create($checkService).invoke($emailRouterServiceName)
                } -args ${function:checkService},$emailRouterServiceName
            if($emailRouterServiceStatusCode -eq 1){
                    invoke-command -session $session -scriptblock $fixEmailRouterService
                }

            $otherServicesResult=invoke-command -Session $session -ScriptBlock{
                param ($startAllAutoServices,$skipServices)
                [scriptblock]::create($startAllAutoServices).invoke($skipServices)
                } -args ${function:startAllAutoServices},$skipServices
            
            if($asyncServiceStatusCode -eq 1){
                if($otherServicesResult.stoppedServices -ne 'none'){
                    $otherServicesResult.stoppedServices+=','+$asyncServiceName
                }else{
                    $otherServicesResult.stoppedServices=$asyncServiceName
                }
            }
            if($emailRouterServiceStatusCode -eq 1){
                if($otherServicesResult.stoppedServices -ne 'none'){
                    $otherServicesResult.stoppedServices+=','+$emailRouterServiceName
                }else{
                    $otherServicesResult.stoppedServices=$emailRouterServiceName
                }
            }
            $dnsOK=invoke-command -Session $session -ScriptBlock{test-connection google.com -Count 1 -Quiet}
            Remove-PSSession $session
            $timeZoneName=[System.TimeZoneInfo]::Local.StandardName
            $abbreviatedZoneName=if($timeZoneName -match ' '){[regex]::replace($timeZoneName,'([A-Z])\w+\s*', '$1')}else{$timeZoneName}
            $timeStampFormat="yyyy-MM-dd HH:mm:ss $abbreviatedZoneName"
            $timeStamp=[System.TimeZoneInfo]::ConvertTimeBySystemTimeZoneId([datetime]::UtcNow,$timeZoneName).ToString($timeStampFormat)
            $otherStoppedServices=$otherServicesResult.stoppedServices
            $otherStoppedServices=.{if(!$dnsOK){
                    $x=$otherStoppedServices|?{$_ -ne 'none'}
                    $x+=,'dnsClient'
                    return $x
                }else{
                    return $otherStoppedServices}}
            $stoppedServices=($otherStoppedServices|out-string).replace("`n",'').trim()
            $fixedAutomatically=if($dnsOK){$otherServicesResult.fixedAutomatically}else{$false}
            $results+=[pscustomobject]@{
                timeStamp=$timeStamp
                computerName=$computerName
                stoppedServices=$stoppedServices
                fixedAutomatically=$fixedAutomatically
                }
        }else{
            write-warning "$env:computername cannnot connect to $computername via WinRM"
            $timeZoneName=[System.TimeZoneInfo]::Local.StandardName
            $abbreviatedZoneName=if($timeZoneName -match ' '){[regex]::replace($timeZoneName,'([A-Z])\w+\s*', '$1')}else{$timeZoneName}
            $timeStampFormat="yyyy-MM-dd HH:mm:ss $abbreviatedZoneName"
            $timeStamp=[System.TimeZoneInfo]::ConvertTimeBySystemTimeZoneId([datetime]::UtcNow,$timeZoneName).ToString($timeStampFormat)
            $results+=[pscustomobject]@{
                timeStamp=$timeStamp
                computerName=$computerName
                stoppedServices='Unknown'
                fixedAutomatically='Unknown'
                }
        }
    }
    return $results    
}

$results=invokeCrmServersMaintenance $computerNames $credentials $skipServices|select-object -Property timeStamp,computername,stoppedServices,fixedAutomatically # This last part is necessary to remove invoke-command junks

write-host $($results|ft|out-string).trim()
$problemsDetected=$results|?{$_.stoppedServices -ne 'none'}
if($problemsDetected){
    if(!(test-path $logFile)){
        if(!(test-path $(split-path $logFile -parent))){mkdir $(split-path $logFile -parent) -force}
        $header='"timeStamp","computerName","stoppedServices","fixedAutomatically"'
        Add-Content -Path $logFile  -Value $header
    }
    $problemsDetected|Export-Csv -Path $logfile -Append
    $css=@"
    <style>
    .h1 {
        font-size: 18px;
        height: 40px;
        padding-top: 80px;
        margin: auto;
        text-align: center;
    }
    .h5 {
        font-size: 22px;
        text-align: center;
    }
    .th {text-align: center;}
    .table {
        padding:7px;
        border:#4e95f4 1px solid;
        background-color: white;
        margin-left: auto;
        margin-right: auto;
        width: 100%
        }
    .colgroup {}
    .th { background: #0046c3; color: #fff; padding: 5px 10px; }
    .td { font-size: 11px; padding: 5px 20px; color: #000;
          width: 1px;
          white-space: pre;
        }
    .tr { background: #b8d1f3;}
    .tr:nth-child(even) {
        background: #dae5f4;
        width: 1%;
        white-space: nowrap
    }
    .tr:nth-child(odd) {
        background: #b8d1f3;
        width: 1%;
        white-space: nowrap
    }
    </style>
"@
    $currentReport=$problemsDetected | ConvertTo-Html -Fragment | Out-String
    $historicalReport=Import-CSV $logFile|sort timeStamp -Descending|select-object -first $limitEmailRecords|ConvertTo-Html -fragment|Out-String
    $currentReportHtml=$currentReport -replace '\<(?<item>\w+)\>', '<${item} class=''${item}''>'
    $historicalReportHtml=$historicalReport -replace '\<(?<item>\w+)\>', '<${item} class=''${item}''>'
    $emailContent='<html><head>'+$css+"</head><body><h5 class='h5'>Current Server Errors</h5>"+$currentReportHtml+"<h5 class='h5'>Historical Server Errors</h5>"+$historicalReportHtml+'</body></html>'
    write-host "Updating the report html file: $logFile.html"
    $null='<html><head>'+$css+"</head><body><h5 class='h5'>Server Errors</h5>"+$historicalReportHtml+'</body></html>' | Out-File "$logFile.html"

    Send-MailMessage -From $emailFrom `
    -To $emailTo `
    -Subject $subject `
    -Body $emailContent `
    -BodyAsHtml `
    -SmtpServer $smtpRelayServer
}

Version 2:

# invokeCrmServersMaintenance.ps1
# Version 0.0.2
# This script is a processes watcher on CRM Servers
# It ensures that all automatically starting services are running
# Moreover, there's a special handling of the service name 'MSCRMAsyncService$maintenance'
 
# Obtain credentials being passed by automation engines such as Jenkins
$username=$env:username
$password=$env:password
$encryptedPassword=ConvertTo-SecureString $password -AsPlainText -Force
$credentials = New-Object -TypeName System.Management.Automation.PSCredential -ArgumentList $userName,$encryptedPassword;
 
# List of servers to check
$computerNames=@(
    'CRM-WEB01',
    'CRM-SQL01',
    'CRM-DEV01'
)
 
# Email relay parameters
$emailFrom='admin@kimconnect.com'
$emailTo='webAdmins@kimconnect.com'
$subject='CRM Server Issues'
$smtpRelayServer='smtp.office365.com'

# Services to skip
$skipServices=@('TrustedInstaller','BITS','gupdate','MapsBroker','RemoteRegistry','sppsvc','WbioSrvc','twdservice','MSCRMAsyncService$maintenance','CDPSvc')
$logFile='C:\scripts\logs\ServersMaintenanceIssues.csv'

function invokeCrmServersMaintenance{
    param(
        [string[]]$computerNames=$env:computername,
        [pscredential]$credentials,
        [string[]]$skipServices=@('TrustedInstaller','gupdate','MapsBroker','RemoteRegistry','sppsvc','WbioSrvc','twdservice','MSCRMAsyncService$maintenance'),
        [string]$desiredStatus='running'
    )
    $results=@()
    $emailRouterServiceName='MSCRMEmail'
    $asyncServiceName='MSCRMAsyncService$maintenance'
    function checkService($serviceName,$status='Running'){
        # Sanitation
        #$systemInvalidChars =[Regex]::Escape(-join [System.Io.Path]::GetInvalidFileNameChars())
        #$regexInvalidChars = "[$systemInvalidChars]"
        #$serviceName=$serviceName -replace [regex]::Matches($serviceName, $regexInvalidChars, 'IgnoreCase').Value
        try{
            $service=get-service -name $serviceName -erroraction 'silentlycontinue'|select -first 1
            if($service.Status -eq $status){
                write-host "$serviceName status of $status is matching the desired state." -foregroundcolor Green
                return 0
            }elseif($null -ne $service.Status){
                write-host "$env:computername`: $serviceName status $($service.Status) doesn`'t match the desired state" -foregroundcolor Red
                return 1
            }else{
                #write-verbose "$serviceName was not found" -foregroundcolor Red
                return -1          
            }
        }catch{
            #Write-verbose $_
            #write-verbose "$serviceName was not found" -foregroundcolor Red
            return -1
        }
    }

    function startAllAutoServices{
        param(
            [string[]]$skipServices=@('TrustedInstaller','gupdate','MapsBroker','RemoteRegistry','sppsvc','WbioSrvc')
            )        
        $filterString=($skipServices|%{"AND name!='$_' "}) -join ''
        $stoppedServices=Get-WmiObject win32_service -Filter "startmode='auto' AND state!='running' $filterString"
        if($stoppedServices){
            write-warning "$env:computername`: these services were not running $($stoppedServices.Name).`r`nProgram now attempts to start them."
            $null=$stoppedServices|Invoke-WmiMethod -Name StartService
            (get-service $stoppedServices.Name).waitforstatus('Running')
            # Detect failed services
            $failedServices=Get-WmiObject win32_service -Filter "startmode = 'auto' AND state != 'running' $filterString"
            if($failedServices){
                write-warning "$env:computername`: failed to start these services $($failedServices.Name)"
            }else{
                write-host "$env:computername`: all auto-start services are running." -ForegroundColor Green
            }
            return [pscustomobject]@{
                stoppedServices=($stoppedServices.Name -join ','|out-string).trim()
                fixedAutomatically=if($failedServices){$false}else{$true}
            }
        }else{
            write-host "$env:computername`: all auto-start services are running." -ForegroundColor Green
            return [pscustomobject]@{
                stoppedServices='none'
                fixedAutomatically=$true
            }
        }        
    } 

    $fixAsyncService={
        $erroractionpreference='stop'
        $asyncServiceName='MSCRMAsyncService$maintenance'
        try{
            $crmTools=if(test-path "$env:programfiles\dynamics 365"){
                    "$env:programfiles\dynamics 365\Tools"
                }else{"$env:programfiles\Microsoft Dynamics CRM\Tools"}            
            $asyncServices=get-service|?{$_.name -like 'MSCRMAsyncService*'}
            write-host "Restarting services: $($asyncServices.Name)"
            restart-service $asyncServices
            write-host 'Renewing keys to CRM App server...'
            Start-Process -Wait -FilePath cmd -Verb RunAs -ArgumentList '/c',"`"$crmTools\Microsoft.Crm.Tools.WRPCKeyRenewal.exe`" /R"
            #cmd /c "`"$crmTools\Microsoft.Crm.Tools.WRPCKeyRenewal.exe`" /R" # alternate method of calling cmd from Powershell with switches
            # cmd /c "`"C:\Windows\system32\PING.EXE`" -n 10 google.com"
            # comment-out because $lastexitcode does not register properly within WinRM sessions
            # if($LASTEXITCODE -ne 0){
            #     write-host 'Microsoft.Crm.Tools.WRPCKeyRenewal.exe command failed.'
            #     return $false
            # }else{
                sleep 1
                write-host 'Restarting Microsoft Dynamics CRM Asynchronous Service (maintenance)...'
                restart-service $asyncServiceName
                return $true
            #    }
        }catch{
            write-host $_
            return $false
        }
    }

    $fixEmailRouterService={        
        $erroractionpreference='stop'
        $emailRouterServiceName='MSCRMEmail'
        $expectedXmlLocation='C:\Program Files\Microsoft CRM Email\Service\Microsoft.Crm.Tools.EmailAgent.SystemState.xml'
        try{
            $emailRouterXmlFile=if(test-path $expectedXmlLocation){
                    $expectedXmlLocation
                }else{
                    "${env:ProgramFiles(x86)}\Microsoft CRM Email\Service\Microsoft.Crm.Tools.EmailAgent.SystemState.xml"
                }
            write-host "Attempting to start email router service normally..."
            start-service $emailRouterServiceName
            $isRunning=(get-service $emailRouterServiceName).Status -eq 'Running'
            if(!$isRunning){
                write-host 'Fixing Microsoft Dynamics CRM Email Router Services...'            
                stop-service $emailRouterServiceName -force
                $null=rename-item $emailRouterXmlFile "$originalFileName.bak" -force
                start-service $emailRouterServiceName
            }
            return $true
        }catch{
            write-host $_
            return $false
        }
    }

    foreach ($computerName in $computerNames){
        $session=if($credentials){
                try{
                    New-PSSession -ComputerName $computername -Credential $credentials -ea Stop
                }catch{
                    New-PSSession -ComputerName $computername -Credential $credentials -SessionOption $(new-pssessionoption -IncludePortInSPN)
                }
            }else{
                try{
                    New-PSSession -ComputerName $computername -ea Stop
                }catch{
                    New-PSSession -ComputerName $computername -SessionOption $(new-pssessionoption -IncludePortInSPN)
                }
            }        
        if($session.state -eq 'Opened'){
            $asyncServiceStatusCode=invoke-command -Session $session -ScriptBlock{
                param ($checkService,$asyncServiceName)
                [scriptblock]::create($checkService).invoke($asyncServiceName)
                } -args ${function:checkService},$asyncServiceName
            if($asyncServiceStatusCode -eq 1){
                    invoke-command -session $session -scriptblock $fixAsyncService
                }

            $emailRouterServiceStatusCode=invoke-command -Session $session -ScriptBlock{
                param ($checkService,$emailRouterServiceName)
                [scriptblock]::create($checkService).invoke($emailRouterServiceName)
                } -args ${function:checkService},$emailRouterServiceName
            if($emailRouterServiceStatusCode -eq 1){
                    invoke-command -session $session -scriptblock $fixEmailRouterService
                }

            $otherServicesResult=invoke-command -Session $session -ScriptBlock{
                param ($startAllAutoServices,$skipServices)
                [scriptblock]::create($startAllAutoServices).invoke($skipServices)
                } -args ${function:startAllAutoServices},$skipServices
            
            if($asyncServiceStatusCode -eq 1){
                if($otherServicesResult.stoppedServices -ne 'none'){
                    $otherServicesResult.stoppedServices+=','+$asyncServiceName
                }else{
                    $otherServicesResult.stoppedServices=$asyncServiceName
                }
            }
            if($emailRouterServiceStatusCode -eq 1){
                if($otherServicesResult.stoppedServices -ne 'none'){
                    $otherServicesResult.stoppedServices+=','+$emailRouterServiceName
                }else{
                    $otherServicesResult.stoppedServices=$emailRouterServiceName
                }
            }
            Remove-PSSession $session
            $timeZoneName=[System.TimeZoneInfo]::Local.StandardName
            $abbreviatedZoneName=if($timeZoneName -match ' '){[regex]::replace($timeZoneName,'([A-Z])\w+\s*', '$1')}else{$timeZoneName}
            $timeStampFormat="yyyy-MM-dd HH:mm:ss $abbreviatedZoneName"
            $timeStamp=[System.TimeZoneInfo]::ConvertTimeBySystemTimeZoneId([datetime]::UtcNow,$timeZoneName).ToString($timeStampFormat)
            $results+=[pscustomobject]@{
                timeStamp=$timeStamp
                computerName=$computerName
                stoppedServices=($otherServicesResult.stoppedServices|out-string).replace("`n",'').trim()
                fixedAutomatically=$otherServicesResult.fixedAutomatically
                }
        }else{
            write-warning "$env:computername cannnot connect to $computername via WinRM"
            $timeZoneName=[System.TimeZoneInfo]::Local.StandardName
            $abbreviatedZoneName=if($timeZoneName -match ' '){[regex]::replace($timeZoneName,'([A-Z])\w+\s*', '$1')}else{$timeZoneName}
            $timeStampFormat="yyyy-MM-dd HH:mm:ss $abbreviatedZoneName"
            $timeStamp=[System.TimeZoneInfo]::ConvertTimeBySystemTimeZoneId([datetime]::UtcNow,$timeZoneName).ToString($timeStampFormat)
            $results+=[pscustomobject]@{
                timeStamp=$timeStamp
                computerName=$computerName
                stoppedServices='Unknown'
                fixedAutomatically='Unknown'
                }
        }
    }
    return $results    
}

$results=invokeCrmServersMaintenance $computerNames $credentials $skipServices|select-object -Property timeStamp,computername,stoppedServices,fixedAutomatically # This last part is necessary to remove invoke-command junks

write-host $($results|ft|out-string).trim()
$problemsDetected=$results|?{$_.stoppedServices -ne 'none'}
if($problemsDetected){
    if(!(test-path $logFile)){
        if(!(test-path $(split-path $logFile -parent))){mkdir $(split-path $logFile -parent) -force}
        $header='"timeStamp","computerName","stoppedServices","fixedAutomatically"'
        Add-Content -Path $logFile  -Value $header
    }
    $problemsDetected|Export-Csv -Path $logfile -Append
    $css=@"
    <style>
    .h1 {
        font-size: 18px;
        height: 40px;
        padding-top: 80px;
        margin: auto;
        text-align: center;
    }
    .h5 {
        font-size: 22px;
        text-align: center;
    }
    .th {text-align: center;}
    .table {
        padding:7px;
        border:#4e95f4 1px solid;
        background-color: white;
        margin-left: auto;
        margin-right: auto;
        width: 100%
        }
    .colgroup {}
    .th { background: #0046c3; color: #fff; padding: 5px 10px; }
    .td { font-size: 11px; padding: 5px 20px; color: #000;
          width: 1px;
          white-space: pre;
        }
    .tr { background: #b8d1f3;}
    .tr:nth-child(even) {
        background: #dae5f4;
        width: 1%;
        white-space: nowrap
    }
    .tr:nth-child(odd) {
        background: #b8d1f3;
        width: 1%;
        white-space: nowrap
    }
    </style>
"@
    $currentReport=$problemsDetected | ConvertTo-Html -Fragment | Out-String
    $historicalReport=Import-CSV $logFile | sort timeStamp -Descending | ConvertTo-Html -fragment | Out-String
    $currentReportHtml=$currentReport -replace '\<(?<item>\w+)\>', '<${item} class=''${item}''>'
    $historicalReportHtml=$historicalReport -replace '\<(?<item>\w+)\>', '<${item} class=''${item}''>'
    $emailContent='<html><head>'+$css+"</head><body><h5 class='h5'>Current Server Errors</h5>"+$currentReportHtml+"<h5 class='h5'>Historical Server Errors</h5>"+$historicalReportHtml+'</body></html>'
    write-host "Updating the report html file: $logFile.html"
    $null='<html><head>'+$css+"</head><body><h5 class='h5'>Server Errors</h5>"+$historicalReportHtml+'</body></html>' | Out-File "$logFile.html"

    Send-MailMessage -From $emailFrom `
    -To $emailTo `
    -Subject $subject `
    -Body $emailContent `
    -BodyAsHtml `
    -SmtpServer $smtpRelayServer
}

Version Multi-threading Test (Slower than Version 2):

# invokeCrmServersMaintenance.ps1
# Version 0.0.3
# This script is a processes watcher on CRM Servers:
# a. Ensures that all automatically starting services are running
# b. Special handling of delicate services: 'MSCRMAsyncService$maintenance' and 'MSCRMEmail'
# c. Simultaneus executions by dynamically limiting concurrent jobs depending on available CPU cores

# Obtain credentials being passed by Jenkins
$username=$env:username
$password=$env:password
$encryptedPassword=ConvertTo-SecureString $password -AsPlainText -Force
$credentials = New-Object -TypeName System.Management.Automation.PSCredential -ArgumentList $userName,$encryptedPassword;

# List of servers to check
$computerNames=@(
    'CRM-WEB01',
    'CRM-SQL01',
    'CRM-DEV01'
)
 
# Email relay parameters
$emailFrom='admin@kimconnect.com'
$emailTo='webAdmins@kimconnect.com'
$subject='CRM Server Issues'
$smtpRelayServer='smtp.office365.com'

# Services to skip
$skipServices=@('TrustedInstaller','BITS','gupdate','MapsBroker','RemoteRegistry','sppsvc','WbioSrvc','twdservice','MSCRMAsyncService$maintenance','CDPSvc')

function invokeCrmServersMaintenance{
    param(
        [string[]]$computerNames=$env:computername,
        [pscredential]$credentials,
        [string[]]$skipServices=@('TrustedInstaller','gupdate','MapsBroker','RemoteRegistry','sppsvc','WbioSrvc','twdservice','MSCRMAsyncService$maintenance'),
        [string]$desiredStatus='running'
    )
    $results=@()
    $emailRouterServiceName='MSCRMEmail'
    $asyncServiceName='MSCRMAsyncService$maintenance'
    function checkService($serviceName,$status='Running'){
        # Sanitation
        #$systemInvalidChars =[Regex]::Escape(-join [System.Io.Path]::GetInvalidFileNameChars())
        #$regexInvalidChars = "[$systemInvalidChars]"
        #$serviceName=$serviceName -replace [regex]::Matches($serviceName, $regexInvalidChars, 'IgnoreCase').Value
        try{
            $service=get-service -name $serviceName -erroraction 'silentlycontinue'|select -first 1
            if($service.Status -eq $status){
                write-host "$serviceName status of $status is matching the desired state." -foregroundcolor Green
                return 0
            }elseif($null -ne $service.Status){
                write-host "$env:computername`: $serviceName status $($service.Status) doesn`'t match the desired state" -foregroundcolor Red
                return 1
            }else{
                #write-verbose "$serviceName was not found" -foregroundcolor Red
                return -1          
            }
        }catch{
            #Write-verbose $_
            #write-verbose "$serviceName was not found" -foregroundcolor Red
            return -1
        }
    }

    function startAllAutoServices{
        param(
            [string[]]$skipServices=@('TrustedInstaller','gupdate','MapsBroker','RemoteRegistry','sppsvc','WbioSrvc')
            )        
        $filterString=($skipServices|%{"AND name!='$_' "}) -join ''
        $stoppedServices=Get-WmiObject win32_service -Filter "startmode='auto' AND state!='running' $filterString"
        if($stoppedServices){
            write-warning "$env:computername`: these services were not running $($stoppedServices.Name).`r`nProgram now attempts to start them."
            $null=$stoppedServices|Invoke-WmiMethod -Name StartService
            (get-service $stoppedServices.Name).waitforstatus('Running')
            # Detect failed services
            $failedServices=Get-WmiObject win32_service -Filter "startmode = 'auto' AND state != 'running' $filterString"
            if($failedServices){
                write-warning "$env:computername`: failed to start these services $($failedServices.Name)"
            }else{
                write-host "$env:computername`: all auto-start services are running." -ForegroundColor Green
            }
            return [pscustomobject]@{
                stoppedServices=($stoppedServices.Name -join ','|out-string).trim()
                allServicesNowRunning=if($failedServices){$false}else{$true}
            }
        }else{
            write-host "$env:computername`: all auto-start services are running." -ForegroundColor Green
            return [pscustomobject]@{
                stoppedServices='none'
                allServicesNowRunning=$true
            }
        }        
    } 

    $fixAsyncService={
        $erroractionpreference='stop'
        $asyncServiceName='MSCRMAsyncService$maintenance'
        try{
            $crmTools=if(test-path "$env:programfiles\dynamics 365"){
                    "$env:programfiles\dynamics 365\Tools"
                }else{"$env:programfiles\Microsoft Dynamics CRM\Tools"}            
            $asyncServices=get-service|?{$_.name -like 'MSCRMAsyncService*'}
            write-host "Restarting services: $($asyncServices.Name)"
            restart-service $asyncServices
            write-host 'Renewing keys to CRM App server...'
            Start-Process -Wait -FilePath cmd -Verb RunAs -ArgumentList '/c',"`"$crmTools\Microsoft.Crm.Tools.WRPCKeyRenewal.exe`" /R"
            #cmd /c "`"$crmTools\Microsoft.Crm.Tools.WRPCKeyRenewal.exe`" /R" # alternate method of calling cmd from Powershell with switches
            # cmd /c "`"C:\Windows\system32\PING.EXE`" -n 10 google.com"
            # comment-out because $lastexitcode does not register properly within WinRM sessions
            # if($LASTEXITCODE -ne 0){
            #     write-host 'Microsoft.Crm.Tools.WRPCKeyRenewal.exe command failed.'
            #     return $false
            # }else{
                sleep 1
                write-host 'Restarting Microsoft Dynamics CRM Asynchronous Service (maintenance)...'
                restart-service $asyncServiceName
                return $true
            #    }
        }catch{
            write-host $_
            return $false
        }
    }

    $fixEmailRouterService={        
        $erroractionpreference='stop'
        $emailRouterServiceName='MSCRMEmail'
        $expectedXmlLocation='C:\Program Files\Microsoft CRM Email\Service\Microsoft.Crm.Tools.EmailAgent.SystemState.xml'
        try{
            $emailRouterXmlFile=if(test-path $expectedXmlLocation){
                    $expectedXmlLocation
                }else{
                    "${env:ProgramFiles(x86)}\Microsoft CRM Email\Service\Microsoft.Crm.Tools.EmailAgent.SystemState.xml"
                }
            write-host "Attempting to start email router service normally..."
            start-service $emailRouterServiceName
            $isRunning=(get-service $emailRouterServiceName).Status -eq 'Running'
            if(!$isRunning){
                write-host 'Fixing Microsoft Dynamics CRM Email Router Services...'            
                stop-service $emailRouterServiceName -force
                $null=rename-item $emailRouterXmlFile "$originalFileName.bak" -force
                start-service $emailRouterServiceName
            }
            return $true
        }catch{
            write-host $_
            return $false
        }
    }

    foreach ($computerName in $computerNames){
        $session=if($credentials){
                try{
                    New-PSSession -ComputerName $computername -Credential $credentials -ea Stop
                }catch{
                    New-PSSession -ComputerName $computername -Credential $credentials -SessionOption $(new-pssessionoption -IncludePortInSPN)
                }
            }else{
                try{
                    New-PSSession -ComputerName $computername -ea Stop
                }catch{
                    New-PSSession -ComputerName $computername -SessionOption $(new-pssessionoption -IncludePortInSPN)
                }
            }        
        if($session.state -eq 'Opened'){
            $asyncServiceStatusCode=invoke-command -Session $session -ScriptBlock{
                param ($checkService,$asyncServiceName)
                [scriptblock]::create($checkService).invoke($asyncServiceName)
                } -args ${function:checkService},$asyncServiceName
            if($asyncServiceStatusCode -eq 1){
                    invoke-command -session $session -scriptblock $fixAsyncService
                }

            $emailRouterServiceStatusCode=invoke-command -Session $session -ScriptBlock{
                param ($checkService,$emailRouterServiceName)
                [scriptblock]::create($checkService).invoke($emailRouterServiceName)
                } -args ${function:checkService},$emailRouterServiceName
            if($emailRouterServiceStatusCode -eq 1){
                    invoke-command -session $session -scriptblock $fixEmailRouterService
                }

            $otherServicesResult=invoke-command -Session $session -ScriptBlock{
                param ($startAllAutoServices,$skipServices)
                [scriptblock]::create($startAllAutoServices).invoke($skipServices)
                } -args ${function:startAllAutoServices},$skipServices
            
            if($asyncServiceStatusCode -eq 1){
                if($otherServicesResult.stoppedServices -ne 'none'){
                    $otherServicesResult.stoppedServices+=','+$asyncServiceName
                }else{
                    $otherServicesResult.stoppedServices=$asyncServiceName
                }
            }
            if($emailRouterServiceStatusCode -eq 1){
                if($otherServicesResult.stoppedServices -ne 'none'){
                    $otherServicesResult.stoppedServices+=','+$emailRouterServiceName
                }else{
                    $otherServicesResult.stoppedServices=$emailRouterServiceName
                }
            }
            Remove-PSSession $session        
            $results+=[pscustomobject]@{
                computerName=$computerName
                stoppedServices=($otherServicesResult.stoppedServices|out-string).replace("`n",'').trim()
                allServicesNowRunning=$otherServicesResult.allServicesNowRunning
                timeStamp=(get-date|out-string).trim()
                }
        }else{
            write-warning "$env:computername cannnot connect to $computername via WinRM"
            $results+=[pscustomobject]@{
                computerName=$computerName
                stoppedServices='Unknown'
                allServicesNowRunning='Unknown'
                timeStamp=(get-date|out-string).trim()
                }
        }
    }
    return $results    
}

#$results=invokeCrmServersMaintenance $computerNames $credentials $skipServices|select-object -Property computername,stoppedServices,allServicesNowRunning # This last part is necessary to remove invoke-command junks

function checkCrmServices($computerNames,$credentials,$skipServices){
    $timer=[System.Diagnostics.Stopwatch]::StartNew()
    $jobResults=@()
    $lineBreak=60
    $dotCount=0
    $minute=0
    $processorsCount=(Get-CimInstance Win32_ComputerSystem).NumberOfLogicalProcessors
    $cpuLoad=(Get-WmiObject win32_processor|Measure-Object -property LoadPercentage -Average).Average
    $maxSimultaneousJobs=if($cpuLoad -gt 90){$processorsCount}else{($processorsCount*2)-1} # dynamically limiting concurrent jobs basing on available CPU cores
    write-host "CPU load detected as: $cpuLoad`%`r`nSetting concurrent jobs max count to be $maxSimultaneousJobs"
    foreach($computerName in $computerNames){
        $thisIterationCompleted=$false
        do {
            $jobsCount=(Get-Job -State 'Running').Count
            if ($jobsCount -lt $maxSimultaneousJobs){            
                write-host "Initiating job for $computerName"
                $null=Start-Job -name $computerName -ScriptBlock {
                    param($invokeCrmServersMaintenance,$computerName,$credentials,$skipServices)
                    [scriptblock]::create($invokeCrmServersMaintenance).invoke($computerName,$credentials,$skipServices)
                } -Args ${function:invokeCrmServersMaintenance},$computerName,$credentials,$skipServices
                $thisIterationCompleted=$true
            }else{
                if($dotCount++ -lt $lineBreak){
                    write-host '.' -NoNewline
                }else{
                    $minute++
                    write-host "`r`n$minute`t:" -ForegroundColor Yellow -NoNewline
                    $dotCount=0
                    }
                sleep -seconds 1
            }
        }until ($thisIterationCompleted)
    }
    $totalJobsCount=(get-job).count
    $processedCount=0
    while($processedCount -lt $totalJobsCount){
        $completedJobs=get-job|?{$_.State -eq 'Completed'}
        if($completedJobs){
            foreach ($job in $completedJobs){
                $computer=$job.Name
                write-host "`r`n===================================================`r`n$computer job completed with these messages:`r`n===================================================`r`n"
                $jobResult=receive-job -id $job.id
                $jobResults+=,$jobResult
                remove-job -id $job.id -force
                $processedCount++
            }
        }
    }
    $minutesElapsed=[math]::round($timer.Elapsed.TotalMinutes,2)
    $timer.stop()
    write-host "This process has completed in $minutesElapsed minutes."
    return $jobResults
}

$results=checkCrmServices $computerNames $credentials $skipServices
write-host $($results|ft|out-string).trim()
$problemsDetected=$results|?{$_.stoppedServices -ne 'none'}
if($problemsDetected){
    Send-MailMessage -From $emailFrom `
    -To $emailTo `
    -Subject $subject `
    -Body $($problemsDetected|convertto-html -Fragment|out-string) `
    -BodyAsHtml `
    -SmtpServer $smtpRelayServer
}

Version 1 (buggy):

# invokeCrmServersMaintenance.ps1
# Version 0.0.1
# This script is a processes watcher on CRM Servers
# It ensures that all automatically starting services are running
 
# Obtain credentials being passed by Jenkins
$username=$env:username
$password=$env:password
$encryptedPassword=ConvertTo-SecureString $password -AsPlainText -Force
$credentials = New-Object -TypeName System.Management.Automation.PSCredential -ArgumentList $userName,$encryptedPassword;
 
# List of servers to check
$computerNames=@(
    'CRM-WEB01',
    'CRM-SQL01',
    'CRM-DEV01'
)
 
# Email relay parameters
$emailFrom='admin@kimconnect.com'
$emailTo='webAdmins@kimconnect.com'
$subject='CRM Server Issues'
$smtpRelayServer='smtp.office365.com'
 
# Services to skip
$skipServices=@('BITS','gupdate','MapsBroker','RemoteRegistry','sppsvc','WbioSrvc','twdservice','MSCRMAsyncService$maintenance','CDPSvc')
 
function invokeCrmServersMaintenance{
    param(
        [string[]]$computerNames=$env:computername,
        [pscredential]$credentials,
        [string[]]$skipServices=@('gupdate','MapsBroker','RemoteRegistry','sppsvc','WbioSrvc','twdservice','MSCRMAsyncService$maintenance'),
        [string]$desiredStatus='running'
    )
    $results=@()
    $emailRouterServiceName='MSCRMEmail'
    $asyncServiceName='MSCRMAsyncService$maintenance'
    function checkService($serviceName,$status='Running'){
        # Sanitation
        #$systemInvalidChars =[Regex]::Escape(-join [System.Io.Path]::GetInvalidFileNameChars())
        #$regexInvalidChars = "[$systemInvalidChars]"
        #$serviceName=$serviceName -replace [regex]::Matches($serviceName, $regexInvalidChars, 'IgnoreCase').Value
        try{
            $service=get-service -name $serviceName -erroraction 'silentlycontinue'|select -first 1
            if($service.Status -eq $status){
                write-host "$serviceName status of $status is matching the desired state." -foregroundcolor Green
                return 0
            }elseif($null -ne $service.Status){
                write-host "$env:computername`: $serviceName status $($service.Status) doesn`'t match the desired state" -foregroundcolor Red
                return 1
            }else{
                #write-verbose "$serviceName was not found" -foregroundcolor Red
                return -1          
            }
        }catch{
            #Write-verbose $_
            #write-verbose "$serviceName was not found" -foregroundcolor Red
            return -1
        }
    }
 
    function startAllAutoServices{
        param(
            [string[]]$skipServices=@('gupdate','MapsBroker','RemoteRegistry','sppsvc','WbioSrvc')
            )        
        $filterString=($skipServices|%{"AND name!='$_' "}) -join ''
        $stoppedServices=Get-WmiObject win32_service -Filter "startmode='auto' AND state!='running' $filterString"
        if($stoppedServices){
            write-warning "$env:computername`: these services were not running $($stoppedServices.Name).`r`nProgram now attempts to start them."
            $null=$stoppedServices|Invoke-WmiMethod -Name StartService
            (get-service $stoppedServices.Name).waitforstatus('Running')
            # Detect failed services
            $failedServices=Get-WmiObject win32_service -Filter "startmode = 'auto' AND state != 'running' $filterString"
            if($failedServices){
                write-warning "$env:computername`: failed to start these services $($failedServices.Name)"
            }else{
                write-host "$env:computername`: all auto-start services are running." -ForegroundColor Green
            }
            return [pscustomobject]@{
                stoppedServices=($stoppedServices.Name|out-string).trim()
                failedToStartServices=if($failedServices){($failedServices.Name|out-string).trim()}else{'none'}
            }
        }else{
            write-host "$env:computername`: all auto-start services are running." -ForegroundColor Green
            return [pscustomobject]@{
                stoppedServices='none'
                failedToStartServices='none'
            }
        }        
    } 
 
    $fixAsyncService={
        $erroractionpreference='stop'
        try{
            $crmTools=if(test-path "$env:programfiles\dynamics 365"){"$env:programfiles\dynamics 365\Tools"
                }else{"$env:programfiles\Microsoft Dynamics CRM\Tools"}
            write-host 'Restarting Microsoft Dynamics CRM Asynchronous Services...'
            get-service|?{$_.name -like 'MSCRMAsyncService*'}|restart-service
            write-host 'Renewing keys to CRM App server...'
            & "$crmTools\Microsoft.Crm.Tools.WRPCKeyRenewal.exe" /R
            if($LASTEXITCODE -ne 0){
                write-host 'Microsoft.Crm.Tools.WRPCKeyRenewal.exe command failed.'
                $result=$false
            }else{
                sleep 3
                write-host 'Restarting Microsoft Dynamics CRM Asynchronous Service (maintenance)...'
                get-service|?{$_.DisplayName -like 'Microsoft Dynamics*(maintenance)'}|restart-service
                $result=$true
                }
            return $result
        }catch{
            write-host $_
            return $false
        }
    }
 
    $fixEmailRouterService={        
        $erroractionpreference='stop'
        $emailRouterServiceName='MSCRMEmail'
        $expectedXmlLocation='C:\Program Files\Microsoft CRM Email\Service\Microsoft.Crm.Tools.EmailAgent.SystemState.xml'
        try{
            $emailRouterXmlFile=if(test-path $expectedXmlLocation){
                    $expectedXmlLocation
                }else{
                    "${env:ProgramFiles(x86)}\Microsoft CRM Email\Service\Microsoft.Crm.Tools.EmailAgent.SystemState.xml"
                }
            write-host 'Restarting Microsoft Dynamics CRM Email Router Services...'
            stop-service $emailRouterServiceName -force
            $null=rename-item $emailRouterXmlFile "$originalFileName.bak" -force
            start-service $emailRouterServiceName
            return $true
        }catch{
            write-host $_
            return $false
        }
    }
 
    foreach ($computerName in $computerNames){
        $session=if($credentials){
                try{
                    New-PSSession -ComputerName $computername -Credential $credentials -ea Stop
                }catch{
                    New-PSSession -ComputerName $computername -Credential $credentials -SessionOption $(new-pssessionoption -IncludePortInSPN)
                }
            }else{
                try{
                    New-PSSession -ComputerName $computername -ea Stop
                }catch{
                    New-PSSession -ComputerName $computername -SessionOption $(new-pssessionoption -IncludePortInSPN)
                }
            }        
        if($session.state -eq 'Opened'){
            $asyncServiceStatusCode=invoke-command -Session $session -ScriptBlock{
                param ($checkService,$asyncServiceName)
                [scriptblock]::create($checkService).invoke($asyncServiceName)
                } -args ${function:checkService},$asyncServiceName
            if($asyncServiceStatusCode -eq 1){
                    invoke-command -session $session -scriptblock $fixAsyncService
                }
 
            $emailRouterServiceStatusCode=invoke-command -Session $session -ScriptBlock{
                param ($checkService,$emailRouterServiceName)
                [scriptblock]::create($checkService).invoke($emailRouterServiceName)
                } -args ${function:checkService},$emailRouterServiceName
            if($emailRouterServiceStatusCode -eq 1){
                    invoke-command -session $session -scriptblock $fixAsyncService
                }
 
            $otherServicesResult=invoke-command -Session $session -ScriptBlock{
                param ($startAllAutoServices,$skipServices)
                [scriptblock]::create($startAllAutoServices).invoke($skipServices)
                } -args ${function:startAllAutoServices},$skipServices
            if($asyncServiceStatusCode -eq 1){
                $otherServicesResult.stoppedServices+=$crmAsyncServiceName
            }
            if($emailRouterServiceStatusCode -eq 1){
                $otherServicesResult.stoppedServices+=$emailRouterServiceName
            }
            Remove-PSSession $session        
            $results+=[pscustomobject]@{
                computername=$computername
                #asyncServiceIsRunning=$asyncServiceResult
                stoppedServices=$otherServicesResult.stoppedServices
                failedToStartServices=$otherServicesResult.failedToStartServices
                }
        }else{
            write-warning "$env:computername cannnot connect to $computername via WinRM"
            $results+=[pscustomobject]@{
                computername=$computername
                #asyncServiceIsRunning='Unknown'
                stoppedServices='Unknown'
                failedToStartServices='Unknown'
                }
        }
    }
    return $results    
}
 
$results=invokeCrmServersMaintenance $computerNames $credentials $skipServices
$problemsDetected=$results|?{$_.stoppedServices -notmatch 'none'}
if($problemsDetected){
    Send-MailMessage -From $emailFrom `
    -To $emailTo `
    -Subject $subject `
    -Body $($problemsDetected|convertto-html -Fragment|out-string) `
    -BodyAsHtml `
    -SmtpServer $smtpRelayServer
}

Leave a Reply

Your email address will not be published. Required fields are marked *