Category: Systems Administration

Import-Module ServerManager 
Add-WindowsFeature PowerShell-ISE

# Windows-Systems-State-Backup-V0.1.ps1
#
# What this script does:
# 1. Create a Windows Scheduled task on an Active Directory controller to perform daily backups at a predetermined time
# 2. Output a log to include backup duration and result of last backup
# 3. Remove backup files that are outside of the retention period
#
# Requirements:
# Powershell 3.0 is necessary to avoid this error, "the term 'Register-ScheduledJob' is not recognized as the name of a cmdlet"

# Set variables:
$taskName="System State Backup";
$storageUncPath="\\Backups\ActiveDirectory\";
$frequency="Daily";
$executionTime="2:00AM";
$retentionDays=7;
$pdcServer=(Get-ADForest |Select-Object -ExpandProperty RootDomain |Get-ADDomain |Select-Object -Property PDCEmulator).PDCEmulator;

# Optional: obtain a domain admin service account credentials
$prompt = "Username and Password of Domain Admin"
$cred = $Host.UI.PromptForCredential("Credential",$prompt,"$env:userdomain\$env:username",$env:userdomain)

# Ensure that uncPath has a trailing backslash
if ($storageUncPath[$storageUncPath.length-1] -ne "\"){$storageUncPath+="\";}

Function createBackupScheduledTask{
    param(
        $taskName="System State Backup",
        $storageUncPath,        
        $frequency="Daily",
        $time="2:00AM",
        $retentionDays=7,
        $cred=$credential
        )
    
    Function removeBackupScheduledTask{
        param(
            $taskName="System State Backup"
            )
        Unregister-ScheduledTask $taskname -confirm:$false -ErrorAction SilentlyContinue
        Write-Host "Taskname $taskName is deleted."
        $checkJob=Get-ScheduledJob $taskname -ErrorAction SilentlyContinue #This command updates the metadata related to deleted task to fully void its remnants
        }

    $backupScheduledTask=Get-ScheduledTask $taskName -ErrorAction SilentlyContinue
    if ($backupScheduledTask){write-host "$taskName already exists. Program will now remove the previous task instance.";removeBackupScheduledTask;}

    Do {
        Try{
            $taskCreated=Register-ScheduledJob -ErrorAction Stop -Name $taskName -ScriptBlock{
                param($storageUncPath,$retentionDays)

                # Start a timer
	            $timer=[System.Diagnostics.Stopwatch]::StartNew();

                # Setup backup storage location
	            $backupDirectory=$storageUncPath;	
	            $dateStamp=Get-Date -Format yyyy-MM-dd;
	            $logFile=$backupDirectory+"backup-log.txt";
	            $dateStampedFolder=$backupDirectory+$dateStamp;
	            New-Item -ItemType Directory $dateStampedFolder -Force;
	            $backupStorageLocation = New-WBBackupTarget -Network $dateStampedFolder;

	            # Instantiate a new policy
	            $thisPolicy = New-WBPolicy;

	            #Add System State to the policy
	            Add-WBSystemState -Policy $thisPolicy;

	            #Add backup location to policy
	            Add-WBBackupTarget -Policy $thisPolicy -Target $backupStorageLocation;

	            #Start backup using policy
	            Start-WBBackup -Policy $thisPolicy;
                
                # Cleanup folders that are outside of retention period 
                $thisComputer=$env:computername;              
                if($backupDirectory -ne $null){
                    $expiredFolders=(get-childitem $backupDirectory|?{ $_.PSIsContainer }|?{$_.LastWriteTime -lt (get-date).AddDays(-$retentionDays)}|Select FullName).FullName;
                    }else{
                        $expiredFolders=$null;
                        }
                if($expiredFolders){
                    $removedBackupSets = $expiredFolders|%{Remove-WBBackupSet -NetworkPath $_ -ComputerName $thisComputer -Force}
                    if($removedBackupSets){$expiredFolders|%{for ($i=0;$i -lt 2;$i++){try{remove-item -LiteralPath $_ -Force -Recurse}catch{}}};} #First pass, delete children; second pass, delete parent
                    }
	
	            #Append result into a log
	            #$lastBackupTime=(Get-WinEvent -LogName Microsoft-Windows-Backup -FilterXPath "*[System[EventID=4]]"|select -First 1).TimeCreated;
	            $lastBackupTime=(Get-WBBackupSet|select -last 1).BackupTime;	            
                $hours=[math]::Round(($timer.Elapsed.TotalSeconds)/3600,2);
	            Add-Content $logFile "`r`n$lastBackupTime` --- $thisComputer backup completed in $hours hours.`r`n$removedBackupSets";
                } -ScheduledJobOption $(New-ScheduledJobOption -RunElevated) -Trigger @{Frequency = $frequency; At=$time} -ArgumentList $storageUncPath,$retentionDays -Credential $cred;
                if ($taskCreated){write-host "$taskName to run $frequency at $time has been successfully created.";};      
            }
        catch{
            $taskCreated=$false;
            write-host "Waiting 10 seconds for next retry...";
            $checkJob=Get-ScheduledJob $taskname -ErrorAction SilentlyContinue;
            sleep 10;
            continue;
            }
    }while($taskCreated -eq $false)
}

#createBackupScheduledTask -taskName $taskName -storageUncPath $storageUncPath -frequency $frequency -time $executionTime -retention $retentionDays

if($pdcServer){
    invoke-command -Credential $cred -computername $pdcServer -scriptBlock{
                                param($importedFunc,$variable1,$variable2,$variable3,$variable4,$variable5)
                                $credential=$using:cred
                                [ScriptBlock]::Create($importedFunc).invoke($variable1,$variable2,$variable3,$variable4,$variable5,$credential);
                                } -Args ${function:createBackupScheduledTask},$taskName,$storageUncPath,$frequency,$executionTime,$retentionDays
}else{write-host "set pdcServer variable before proceeding"}

Sample execution result:

System State Backup already exists. Program will now remove the previous task instance.
Taskname System State Backup is deleted.
System State Backup to run Daily at 2:00AM has been successfully created.

PS C:\Windows\system32> netsh.exe wlan show profiles name=* key=clear | select-string -Pattern "(SSID Name|Key Content)"

    SSID name              : "CiscoGear"
    Key Content            : The-Password-Is-Password
    SSID name              : "Wireless"
    Key Content            : What-Password?

1. Delete cluster role

2. Validate that the DNS entry has been purged
a. Check DNS

Example: entry still exists
-------------------------------------
C:\Users\176233>nslookup roguesherver
Server: dns01.kimconnect.local
Address: 192.168.500.152

Name: roguesherver.kimconnect.local
Address: 192.168.500.133

Example: entry has been removed
-------------------------------------
C:\Users\176233>nslookup roguesherver
Server: roguesherver.kimconnect.local
Address: 192.168.500.152
*** roguesherver.kimconnect.local can't find roguesherver: Non-existent domain

b. Purge instances of servername in DNS

3. Manually purge the virtual computer name object in AD
a. Search for roguesherver
b. Delete roguesherver
c. Manually trigger AD Sync or wait for automatic AD propagation schedule

4. Recreate the clustered role
a. Recreate the role using PowerShell or GUI
b. Possible Error:
– Error message: “The network name is already used on the network”
– Resolution: just wait 5 to 10 minutes or repeat previous steps
– Error message: “The IP Address ‘x.x.x.x’ is already used.”
– Resolution: purge the IP address reservation in the DNS Reverse lookup zone

1. Take volume off synchronous mode

– Pre-emptively avoid this error: “Failed to update the volume. Failed to update the volume: Operation is not permitted on volumes configured for synchronous replication.”
– Login to Nimble > click on Manage > Data Storage > click the target volume > Edit > Next to advance to the Space settings > Next to advance to the Protection Settings > select “No Volume Protection” > Save > select Okay to disassociate… > Make Edits to commit change

2. Expand volume size

– Click on Manage > Data Storage > click the target volume > Edit > Next to advance to the Space settings > type in a numerical value to represent the desired volume size (e.g. change from 2.4 TiB to 3.0 TiB) > Save > put a check mark next to Okay to resize volume > Make Edits

3. Resize volume in Windows

These commands are to be executed on the owner node of the target volume.

The quick method

# Resize volume to its available maximum
$driveLetter="V"
Update-HostStorageCache
$max=(Get-PartitionSupportedSize -DriveLetter $driveLetter).SizeMax
Resize-Partition -DriveLetter $driveLetter -Size $max

The struggles during troubleshooting

# View all partitions of disk 13
PS C:\Windows\system32> Get-Partition -DiskNumber 13
   DiskPath: \\?\Disk{0a0816ef-54f0-2eae-ff8c-8edb05b53e72}
PartitionNumber  DriveLetter Offset                                        Size Type
---------------  ----------- ------                                        ---- ----
1                            17408                                       128 MB Reserved
2                V           135266304                                  2.38 TB Basic

# View specific partition of disk
PS C:\Windows\system32> Get-Partition -DiskNumber 13 -PartitionNumber 2
   DiskPath: \\?\Disk{0a0816ef-54f0-2eae-ff8c-8edb05b53e72}
PartitionNumber  DriveLetter Offset                                        Size Type
---------------  ----------- ------                                        ---- ----
2                V           135266304                                  2.38 TB Basic

# collect partition sizes as a variable
$size = (Get-PartitionSupportedSize -DiskNumber 13 -PartitionNumber 2)

Get-PartitionSupportedSize -DiskNumber 13 -PartitionNumber 2)
   0/2+ completed
   [                                                                                                            ]

PS C:\Windows\system32> $size
      SizeMin       SizeMax
      -------       -------
2612375207936 2616702516736

# Demonstration of a failed attempt due to a bug in 
Resize-Partition -DiskNumber 13 -PartitionNumber 2 -Size $size.SizeMax

Error:
PS C:\Windows\system32> Resize-Partition -DiskNumber 13 -PartitionNumber 2 -Size $size.SizeMax
Resize-Partition : Size Not Supported

Extended information:
The size of the extent is less than the minimum of 1MB.

Activity ID: {90e2f332-c11d-4df3-a056-a52a6970dcf0}
At line:1 char:1
+ Resize-Partition -DiskNumber 13 -PartitionNumber 2 -Size $size.SizeMa ...
+ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    + CategoryInfo          : NotSpecified: (StorageWMI:ROOT/Microsoft/.../MSFT_Partition) [Resize-Partition], CimExce
   ption
    + FullyQualifiedErrorId : StorageWMI 4097,Resize-Partition

$wrongSize = (Get-PartitionSupportedSize -DiskNumber 13 -PartitionNumber 2)
$correctSize = (Get-PartitionSupportedSize -DriveLetter $driveLetter)

PS C:\Windows\system32> $correctSize
      SizeMin       SizeMax
      -------       -------
2612374355968 2748643786240


PS C:\Windows\system32> $wrongSize
      SizeMin       SizeMax
      -------       -------
2612375207936 2616702516736

4. Turn synchronous mode back on

a. Delete orphanated copy

– Login to Nimble > click on Manage > Data Storage > put a check-mark next to the target volume’s previous replicated copy (should be shown as offline) > click on “X” or “Remove” button > verify that the volume to delete is the orphanated copy > Delete

b. Add volume back to protection group

– Click on Manage > Data Protection > click on the correct Volume Collection (also known as protection group) > Actions > Edit > highlight the target volume in the Available pool > Add > Save

Error Message:

The group <group name> only accepts messages from people in its organization or on its allowed senders list, and your email address isn’t on the list.

More Info for Email Admins
Status code: 550 5.7.133 

This error occurs when the distribution group, security group, or Office 365 group is configured to accept messages only from authenticated senders (senders in the same organization or those added to the group's allowed senders list).

To fix the issue, the recipient's email admin or the group owner must add the sender's email address to the group's allowed senders list or change the group's delivery management setting to accept messages from senders inside and outside of the organization.

Usually this issue can only be fixed by the recipient's email admin or the group owner.

For more information and steps to fix this error, see Fix email delivery issues for error code 5.7.133 in Office 365.

Resolution:

Assuming that the Office365 to On-Premise Exchange connect have been validated and functional, this command would fix the problem.

$groupName="<group name>@kimconnect.com"
Set-DistributionGroup $groupName -RequireSenderAuthenticationEnabled $false

1 Byte = 8 bits
1 KB = 1024 bytes
1 MB = 1024 KB (or 1048576 bytes)
1 GB = 1024 MB (or  1073741824 bytes)
1 TB = 1024 GB (or 1099511627776 bytes)

Decommissioning Exchange hosts should have all Exchange services disabled. Thereafter, we may safely turn these Exchange server off without interrupting email services.

Deleting Exchange servers aren’t usually recommended as there are risks involved. We could choose to archive virtual machine files onto an external hard drive to save space on production VM hosts. This typically is the extent of a decom task, and I do recommend that we pause decom activities of these machines at this phase to allow us the ability to “spin up” the machines should they consist of unknown-at-this-time email data.

Optionally, we may choose to fully purge Exchange servers. Here are the steps:

If the server is still online, start its Exchange services and move its public folder replicas to another Exchange host using these commands
$decomServers=”MAIL02″,”MAIL03″
$productionServer=”MAIL007″
$decomServers | %{MoveAllReplicas -sourceserver $_ -targetsever $productionServer}

Clear all remnants of the downed server from Active Directory – be advised that this is irreversible without AD restores:
-Connect to the domain controler
-Launch the run dialog (Windows Key + R)
-Type in the command “adsiedit.msc” et then press “OK”
-Right click on the console then “Connect to:”
-In the connexion dialog select “Well known Naming Context”
-In the drop down menu select “Configuration”
-Explode “CN=Configuration [domain]\CN=Services\CN=Microsoft Exchange\CN=[organization]\CN=Administrative Groups\CN=Servers”
-Right click on the dead server and pick “Delete”
-We also need to delete Database information as well, navigate to “CN=Configuration [domain]\CN=Services\CN=Microsoft -Exchange\CN=[organization]\CN=Administrative Groups\CN=Databases”
-Explode each items to find wich one is related to the old server, then delete it as well.
-Launch Server Manager > Navigate to Roles > Active Directory Domain Services > Active Directory users and Computers $domain > $domain > Microsoft Exchange Security Groups > Exchange Servers > right-click the decom server > remove this server to remove it from the list of Exchange Trusted Subsystem

When Exchange Server certificates expire, it’s the responsibility of the System Administrator to update those certs. Here’s a sequence of execution to ensure server up time.

Quick PowerShell Commands:

Part 1: Self-Signed Certs

# Option A:
# Renew expired certificates in-place
$expiredCertThumbprints=(Get-ExchangeCertificate | where {$_.Status -eq "Valid" -and $_.IsSelfSigned -eq $true -and $_.NotAfter -lt $(get-date)} | Select Thumbprint).Thumbprint
$expiredCertThumbprints|Get-ExchangeCertificate -Thumbprint $_ | New-ExchangeCertificate -PrivateKeyExportable $true

# Option B:
# Check for self-signed certs - empty output means no expiring certs
$daysThreshold=30
Get-ExchangeCertificate | where {$_.Status -eq "Valid" -and $_.IsSelfSigned -eq $true -and $_.NotAfter -lt $(get-date).addDays($daysThreshold)} | `
	Format-List FriendlyName,Subject,CertificateDomains,Thumbprint,NotBefore,NotAfter

# Create a new cert for each expired cert and assign newly generated certs thumbprints into a variable
$exchangeServerName=$env:computername
$domain="$env:userdnsdomain"
$fqdn="$exchangeServerName.$domain"
$expiredCertNames=(Get-ExchangeCertificate | where {$_.Status -eq "Valid" -and $_.IsSelfSigned -eq $true -and $_.NotAfter -lt $(get-date).addDays($daysThreshold)} | Select FriendlyName).FriendlyName
$thumbprints=($expiredCertNames|%{New-ExchangeCertificate -FriendlyName $_ -SubjectName CN=$domain -DomainName $domain,$fqdn -PrivateKeyExportable $true}|Select Thumbprint).Thumpprint

# Add new certs into Exchange
$date=get-date
$thumbprints|%{Set-AuthConfig -NewCertificateThumbprint $_ -NewCertificateEffectiveDate $date}

# Publish new cert
Set-AuthConfig -PublishCertificate

# Remove old cert
Set-AuthConfig -ClearPreviousCertificate

Part 2: Public Certs

$certFile="\\FILESHERVER01\Software\Certificates\kimconnect_cert.pfx"
$plaintextPassword='$!JFQ$j0(jeVE$%@%$JADJAO%a94 jab j0w3jijrarski5034j-i-b-54-353j5-j@!'
$friendlyName=$env:USERDNSDOMAIN -replace ".$($env:USERDOMAIN)"
if($friendlyName){	
	Import-ExchangeCertificate -FileName $certFile `
	-Password (ConvertTo-SecureString -String $plaintextPassword -AsPlainText -Force) `
	-FriendlyName $friendlyName -PrivateKeyExportable $true -Services SMTP,IIS,UM,UMCallRouter,POP,IMAP
	}	

# Check Exchange Service Health
Test-ServiceHealth	

Current Instructions (Applicable to Updating Mailbox roles)

Obtain Public Cert

Get a public cert from a certificate authority such as Commodo, Godaddy, etc.

Optional: Create Updated Self Signed Cert

# Create New Cert
$domain="EXCH01"
$fqdn="$domain`.intra.domain.net"
$friendlyName="Exchange Certificate"
New-ExchangeCertificate -FriendlyName $friendlyName -SubjectName CN=$domain -DomainName $domain,$fqdn -PrivateKeyExportable $true

Add Updated Cert into Certificates Store of Local Machine and Remove Old Cert

Using Exchange PowerShell

$friendlyName="kimconnect.com"
Import-ExchangeCertificate -FileName "\\FILESHERVER01\Software\Certificates\kimconnect_cert.pfx" -Password (ConvertTo-SecureString -String '$!JFQ$j0(jeVE@!' -AsPlainText -Force) -FriendlyName $friendlyName -PrivateKeyExportable $true -Services SMTP,IIS,UM,UMCallRouter,POP,IMAP

Warning

This certificate with thumbprint XXXXXXXXXX and subject '*.kimconnect.com' cannot used for POP SSL/TLS connections because the subject is not a Fully Qualified Domain Name (FQDN). Use command Set-POPSettings to set X509CertificateName to the FQDN of the service.

This certificate with thumbprint XXXXXXXXXX and subject '*.kimconnect.com' cannot used for IMAP SSL/TLS connections because the subject is not a Fully Qualified Domain Name (FQDN). Use command Set-IMAPSettings to set X509CertificateName to the FQDN of the service.

# Set POP & IMAP with fqdn using Exchange PS
Set-POPSettings -X509CertificateName mail.kimconnect.com
Set-IMAPSettings -X509CertificateName mail.kimconnect.com

# Restart affected services using System PowerShell (not Exchange PS)
Stop-Service MSExchangePop3
Start-Service MSExchangePop3
Stop-Service MSExchangeImap4
Start-Service MSExchangeImap4

Using GUI

Run: certlm.msc

1.  Add New Cert
   – Personal > right-click Certificates > All Tasks > Import > navigate to the location of the New Cert > OK > OK
   – Right-click New Cert
2. Back up Old Cert
   Right-click old cert > All tasks > Export > Next > select “Yes, export the private key” > Next > put a check mark next to “Export all extended properties” > Next > put a check mark next to “Password” > input password > Next > input file name (e.g. C:\certs\$certname_expiring_date.pfx) > Next > OK
3. Document private key permissions of old cert
   Right-click old cert > All tasks > Manage Private Keys
   
   Add Network Service with READ permissions
   
4. Remove old cert – WARNING: this will render URL access offline until new cert is rebound within IIS!
   Right-click old cert > delete > OK

Bind New Cert to IIS on Exchange Server

Run: inetmgr.msc > $serverName > Sites > Default Web Site > Bindings > Set these
– https 443 * = new Public Cert
– https 443 127.0.0.1 = Local Server Self Signed Cert (e.g. EXCH01.intranet.company.com)

Restart services

# Restart IIS
iisreset

# Restart Exchange Transport
Stop-Service MSExchangeTransport
Start-Service MSExchangeTransport

# Restart MSExchangeMailboxAssistants
Stop-Service MSExchangeMailboxAssistants
Start-Service MSExchangeMailboxAssistants

# Optional: restart Exchange Information Store
Stop-Service MSExchangeIS
Start-Service MSExchangeIS

Validate Exchange Functionality

# Check Exchange Service Health
Test-ServiceHealth

# Validate access to Exchange Control Panel
$fqdn="EXCH01.intranet.company.com"
Function getDefaultBrowser {
	#Get the default Browser path
	New-PSDrive -Name HKCR -PSProvider registry -Root Hkey_Classes_Root | Out-Null
	$browserPath = ((Get-ItemProperty ‘HKCR:\http\shell\open\command’).'(default)’).Split(‘”‘)[1]
	return $browserPath
}
getDefaultBrowser;
start "$fqdn"

Troubleshooting commands:

# Check for server roles (pay attention to Edge Transport roles as that role will require special Cert updating sequence)
Get-ExchangeServer | select name, serverrole, edition, admindisplayversion, isClientAccessServer | fl

# Check for self-signed certs
Get-ExchangeCertificate | where {$_.Status -eq "Valid" -and $_.IsSelfSigned -eq $true} | Format-List FriendlyName,Subject,CertificateDomains,Thumbprint,NotBefore,NotAfter

Raw Notes from Prior

1. Run inetmgr.exe (Internet Information Services Manager)

• Default Web Site:
  – https * 443 => fqdn public certificate (e.g. Issued To = *.kimconnect.com; Issued By = “DigiCert SHA2 High Assurance Server CA”)
  – https 127.0.0.1 443 => fqdn public certificate
• Exchange Back End:
  – https * 444 => private server certificate (e.g. Issued To = exch01.intranet.kimconnect.com, Issued By = KimConnect SHA2 CA)

2. Run services.msc to restart these services

• IIS Admin Service (IISReset /noforce)
  Microsoft Account Signin Assistance
  Microsoft Exchange Information Store
  Microsoft Exchange Mailbox Assistants
  Microsoft Exchange Forms-based Authentication Service (may not be available on some instances)
• Start an EMS > C:\Program Files\Microsoft\Exchange Server\V15\Bin\UpdateCas.ps1
• Run inetmgr.exe > Reset the OWA virtual directory

3. Check Exchange Service Health to validate that there are no ServicesNotRunning items

[PS] C:\Windows\system32>Test-ServiceHealth

Role                    : Mailbox Server Role
RequiredServicesRunning : True
ServicesRunning         : {IISAdmin, MSExchangeADTopology, MSExchangeDelivery, MSExchangeIS,
                          MSExchangeMailboxAssistants, MSExchangeRepl, MSExchangeRPC, MSExchangeServiceHost,
                          MSExchangeSubmission, MSExchangeThrottling, MSExchangeTransportLogSearch, W3Svc, WinRM}
ServicesNotRunning      : {}

Role                    : Client Access Server Role
RequiredServicesRunning : True
ServicesRunning         : {IISAdmin, MSExchangeADTopology, MSExchangeIMAP4, MSExchangeMailboxReplication,
                          MSExchangeRPC, MSExchangeServiceHost, W3Svc, WinRM}
ServicesNotRunning      : {}

Role                    : Unified Messaging Server Role
RequiredServicesRunning : True
ServicesRunning         : {IISAdmin, MSExchangeADTopology, MSExchangeServiceHost, MSExchangeUM, W3Svc, WinRM}
ServicesNotRunning      : {}

Role                    : Hub Transport Server Role
RequiredServicesRunning : True
ServicesRunning         : {IISAdmin, MSExchangeADTopology, MSExchangeEdgeSync, MSExchangeServiceHost,
                          MSExchangeTransport, MSExchangeTransportLogSearch, W3Svc, WinRM}
ServicesNotRunning      : {}

Check the OWA VD settings:

Get-OwaVirtualDirectory | FL Identity,*auth*,*url*
Get-EcpVirtualDirectory | FL Identity,*auth*,*url*

4. Validate access to Exchange Admin Center

– Authenticate to https://{fqdn}/ecp
– If HTTP error 503 or 500 occurs, try https://{fqdn}/ecp/?ExchClientVer=15

5. Additional troubleshooting items

Change the authentication method of the “owa” virtual directory to Windows authentication

set-Owavirtualdirectory -identity "E15MBX\owa (Exchange Back End)" -WindowsAuthentication $True -Basicauthentication $false -Formsauthentication $false
IISReset /noforce

Raw Notes:

Error:
Failed to connect to the Edge Transport server ADAM instance with exception The supplied credential is invalid.. This could be caused by a failure to resolve the Edge Transport server name EXCH-EDGE.intra.net in DNS, a failure trying to connect to port 50636 on EXCH-EDGE.intra.net, network connectivity issues, an invalid certificate, or an expired subscription. Verify your network and server configuration.

Process to Resolve:

Preliminary steps to rule out easy to fix problems:

# Verify connectivity from Hub to Edge
ran check-netconnection function to verify connectivity between exch-hub to EXCH-EDGE port 50636 with success = $true

# Check to see whether there are any error messages in the queue:
Get-queue

Restart some services on Hub and Edge servers
1. Restart the following services on MBX Server
Microsoft Exchange EdgeSync
Microsoft Exchange Transport 
2. Restart the following services on Edge Server
Microsoft Exchange ADAM
Microsoft Exchange Credential service 
Microsoft exchange Transport

# Confirm if the certificate meets the FQDN of Edge Server if it has been enabled for SMTP service
get-exchangecertificate | FL

Intermediate Level steps to address connector issues:

Mail flow:
Outlook client <==> Hub Exchange <==> Edge Exchange <==> Barracuda (smart host) <==> Internet <==> Destination email systems

Generalization:
- Hub uses EdgeSync to connect to the edge server via ADAM credentials and those are periodically changed by the "Edge Credential Service"
- Only the Client Access Role server requires public certs. The rest of the other roles does not require such.
- Connectors between Edge and Hub servers require SSL, and those can be private certs.
- If the Edge server cert is updated, New-EdgeSubscription command needs to be ran to generate a newEdgeSubcription.xml file
- The newEdgeSubscription.xml needs to also be ran on the Hub server to import new Edge connector information
- Make sure the credential service is up and running on the edge. 
- Call start-edgesynchronization is required to synchronize between Edge and Hub is a new subscription has been created
- Send connectors and Receive connectors can be automatically generated

[PS] C:\ProgramData\Microsoft\Windows\Start Menu\Programs\Microsoft Exchange Server 2010>Start-EdgeSynchronization

RunspaceId : f9a541a8-51db-4b87-a392-b727eeae6c42
Result : CouldNotConnect
Type : Recipients
Name : EXCH-EDGE
FailureDetails : The supplied credential is invalid.
StartUTC : 8/9/2019 6:45:46 PM
EndUTC : 8/9/2019 6:45:46 PM
Added : 0
Deleted : 0
Updated : 0
Scanned : 0
TargetScanned : 0

RunspaceId : f9a541a8-51db-4b87-a392-b727eeae6c42
Result : CouldNotConnect
Type : Configuration
Name : EXCH-EDGE
FailureDetails : The supplied credential is invalid.
StartUTC : 8/9/2019 6:45:46 PM
EndUTC : 8/9/2019 6:45:46 PM
Added : 0
Deleted : 0
Updated : 0
Scanned : 0
TargetScanned : 0

[PS] C:\ProgramData\Microsoft\Windows\Start Menu\Programs\Microsoft Exchange Server 2010>Get-EdgeSubscription

Name Site Domain
---- ---- ------
EXCH-EDGE intra.net/Co... intra.net

==============================================================================

Recreate Edge Subscription:

On Hub server

# Generate new private Exchange certificate
$domain="exch-hub"
$fqdn="exch-hub.intra.net"
New-ExchangeCertficate -DomainName $domain, $fqdn -PrivateKeyExportable $true -KeySize 2048

# Check certs
get-ExchangeCertificate

# Get more details about cert
# $newcert = get-ExchangeCertificate | ? { $_.certdate -like "blah blah"} | select name
$newcert="#######"
get-exchangecertificate $number | fl

# set iis to bind to new cert
# perform iisreset 
# backup old cert and remove it

# New-SendConnector -Custom -Name Baracudda -AddressSpaces * -smarthost 10.10.11.1 -ForceHELO $true -SmartHostAuthMechanism None -Source $edgeServer

# Remove Edge Subscription
Get-EdgeSubscription | Remove-EdgeSubscription

On Edge

# Clean up old certs
lmcert.msc > remove Microsoft Exchange ADAM from Personal Certs folder

# Remove Edge Subscription
Get-EdgeSubscription | Remove-EdgeSubscription

# Generate new subscription file
New-EdgeSubscription -Filename c:\newEdgeSubscription.xml
Re-start the Microsoft Exchange ADAM

On Hub server
# New-EdgeSubscription -FileData ([byte[]]$(Get-Content -Path "\\EXCH-EDGE\c$\newEdgeSubscription.xml" -Encoding Byte -ReadCount 0)) #Experimental command
New-EdgeSubscription -Filename c:\newEdgeSubscription.xml 
Start-EdgeSynchronization
Test-EdgeSynchronization

[PS] C:\Windows\system32>New-EdgeSubscription -Filename c:\newEdgeSubscription.xml

Confirm
If you create an Edge Subscription, this Edge Transport server will be managed via EdgeSync replication. As a result,
any of the following objects that were created manually will be deleted: accepted domains, message classifications,
remote domains, and Send connectors. After creating the Edge Subscription, you must manage these objects from inside
the organization and allow EdgeSync to update the Edge Transport server. Also, the InternalSMTPServers list of the
TransportConfig object will be overwritten during the synchronization process.
EdgeSync requires that this Edge Transport server is able to resolve the FQDN of the Hub Transport servers in the
Active Directory site to which the Edge Transport server is being subscribed, and those Hub Transport servers be able
to resolve the FQDN of this Edge Transport server. You should complete the Edge Subscription inside the organization in
the next "1440" minutes before the bootstrap account expires.
[Y] Yes [A] Yes to All [N] No [L] No to All [S] Suspend [?] Help (default is "Y"): y


New-EdgeSubscription : Microsoft Exchange couldn't create or update the Edge Subscription account on the Edge Transport
server for the following reason: The LDAP server is unavailable.. Stack is at System.DirectoryServices.Protocols.LdapConnection.Connect()
at system.DirectoryServices.Protocols.LdapConnection.BindHelper(NetworkCredential newCredential, Boolean needSetCredential)
at Microsoft.Exchange.MessageSecurity.EdgeSync.AdamUserManagement.CreateOrUpdateADAMPrincipal(String user, String password, Boolean bootStrapAccount, TimeSpan expiry)
at Microsoft.Exchange.Management.SystemConfigurationTasks.NewEdgeSubscription.InitiateSubscriptionOnEdge()
At line:1 char:21
+ New-EdgeSubscription <<<< -Filename c:\newEdgeSubscription.xml
+ CategoryInfo : InvalidOperation: (:) [New-EdgeSubscription], InvalidOperationException
+ FullyQualifiedErrorId : 780DB3C3,Microsoft.Exchange.Management.SystemConfigurationTasks.NewEdgeSubscription

# Check status of Exchange ADAM Services
Get-Service *ADAM* | ft Di*,St*

# Check Exchange certificates
[PS] C:\Windows\system32>Get-ExchangeCertificate | fl

AccessRules : {System.Security.AccessControl.CryptoKeyAccessRule, System.Security.AccessControl.CryptoKeyAccessR
ule}
CertificateDomains : {ab0ee702-f37f-4dff-bfb2-66698a441d9a}
HasPrivateKey : True
IsSelfSigned : False
Issuer : CN=280b6975-b30a-4f5b-b2c3-7864e37f1c05
NotAfter : 8/9/2119 1:36:53 PM
NotBefore : 8/9/2019 12:36:53 PM
PublicKeySize : 2048
RootCAType : Unknown
SerialNumber : 73AC7DDB217BA7AF44847CC68A8B9CC9
Services : None
Status : Invalid
Subject : CN=ab0ee702-f37f-4dff-bfb2-66698a441d9a
Thumbprint : CFD78D7F9DFAA0BD537B3755C24089CE3ED0EC55

AccessRules :
CertificateDomains : {EXCH-EDGE, EXCH-EDGE.intra.net}
HasPrivateKey : True
IsSelfSigned : True
Issuer : CN=EXCH-EDGE
NotAfter : 10/11/2017 11:09:54 PM
NotBefore : 10/11/2012 11:09:54 PM
PublicKeySize : 2048
RootCAType : Registry
SerialNumber : 5DC03A0D09D1C594468C11CE9EC919D4
Services : SMTP
Status : DateInvalid
Subject : CN=EXCH-EDGE
Thumbprint : 4157434692710986BAC026FD2DFE32D4352DE9B3

AccessRules :
CertificateDomains : {intra.net, www.intra.net, exch-cas.intra.net, apollo.inglewood.kimconnect.com, autodisc
over.intra.net, autodiscover.inglewood.kimconnect.com, pop.inglewood.kimconnect.com, imap.inglewood.kimconnect.com, inglewood.kimconnect.com, legacy.intra.net, legacy.inglewood.kimconnect.com}
HasPrivateKey : True
IsSelfSigned : False
Issuer : SERIALNUMBER=07969287, CN=Go Daddy Secure Certification Authority, OU=http://certificates.godaddy.
com/repository, O="GoDaddy.com, Inc.", L=Scottsdale, S=Arizona, C=US
NotAfter : 5/16/2016 11:18:35 AM
NotBefore : 5/16/2011 11:18:35 AM
PublicKeySize : 2048
RootCAType : ThirdParty
SerialNumber : 2B94032E16C980
Services : SMTP
Status : DateInvalid
Subject : CN=intra.net, OU=Domain Control Validated, O=intra.net
Thumbprint : A05FBA0E72AD3D3E666973C9AFDE378535E24393

=============================================================================================

# Create New Cert
$domain="EXCH-EDGE"
$fqdn="exch-hub.intra.net"
$friendlyName="Exchange Certificate"
New-ExchangeCertificate -FriendlyName $friendlyName -SubjectName CN=$domain -DomainName $domain,$fqdn -PrivateKeyExportable $true #Optional:-Services SMTP -KeySize 2048

# Check for self-signed certs
Get-ExchangeCertificate | where {$_.Status -eq "Valid" -and $_.IsSelfSigned -eq $true} | Format-List FriendlyName,Subject,CertificateDomains,Thumbprint,NotBefore,NotAfter

# Restart Exchange Transport
Stop-Service MSExchangeTransport
Start-Service MSExchangeTransport

# Create new Subscription on Edge servers:
New-EdgeSubscription -Filename c:\newEdgeSubscription.xml

# Import subscription on Hub server
New-EdgeSubscription -Filename c:\newEdgeSubscription.xml

# On Hub, trigger New Edge Susbcription via Exchange Management Console GUI
$site='intra.net/Configuration/Sites/DistrictOffice'
New-EdgeSubscription -FileData '<Binary Data>' -Site $site -CreateInternetSendConnector $true -CreateInboundSendConnector $true

# Trigger sync
start-edgesynchronization -forcefullsync

# Restart Exchange Transport
Stop-Service MSExchangeTransport
Start-Service MSExchangeTransport

# Check mail queue
Get-Queue

# Check logs, navigate to:
%exchangeinstallpath%\TransportRoles\Logs\ProtocolLog\SmtpReceive

# Create new connector to point to the smart host (Barracuda spam filter). Make sure that the Source of Send Connector is Edge Server (not Hub Server)
# Disable the automatically generated connector that does not use the smart host

# Example of mail flow issue when the smart host does not accept connections from the Hub server. Resolution was to change the connector Source to the Edge transport

[PS] C:\Windows\system32>Get-Queue

Identity DeliveryType Status MessageCount NextHopDomain
-------- ------------ ------ ------------ -------------
exch-hub\1639048 MapiDelivery Active 17 school-mailboxdb3
exch-hub\1639053 SmartHost... Retry 5675 [10.10.1.11]
exch-hub\1639058 MapiDelivery Active 10 do-mailboxdb
exch-hub\1639059 MapiDelivery Active 12 school-mailboxdb4
exch-hub\1639060 MapiDelivery Active 14 school-mailboxdb2
exch-hub\Submission Undefined Ready 103 Submission
exch-hub\Shadow\1591071 ShadowRed... Ready 62 EXCH-EDGE.intra.net
exch-hub\Shadow\1639036 ShadowRed... Ready 166 EXCH-EDGE.intra.net

[PS] C:\Windows\system32>Get-Queue -Identity exch-hub\1639053 | fl #where 1639053 is Identity of the smart host

RunspaceId : b2e3dae0-ecb1-4508-b307-31da04271141
DeliveryType : SmartHostConnectorDelivery
NextHopDomain : [10.10.1.11]
TlsDomain :
NextHopConnector : 77215356-bf27-49bc-bd41-4603375ac561
Status : Retry
MessageCount : 5656
LastError : 451 4.4.0 Primary target IP address responded with: "421 4.4.2 Connection dropped due to SocketE
rror." Attempted failover to alternate host, but that did not succeed. Either there are no alter
nate hosts, or delivery failed to all alternate hosts.
LastRetryTime : 8/9/2019 5:45:39 PM
NextRetryTime : 8/9/2019 5:50:39 PM
DeferredMessageCount : 0
QueueIdentity : exch-hub\1639053
Identity : exch-hub\1639053
IsValid : True

In this scenario, the business decision is to limit exposure of certain internal accounts to only allow those to receive emails from the same “Exchange Organization”. This is an extra measure to improve enterprise security posture by further reducing spams and potential messaging vulnerabilities.

# Set "Require that all senders are authenticated" for one account
$targetUsername="PORequests"
$targetObject = Get-ADUser -Filter 'SamAccountName -eq $targetUsername'
Set-ADUser $targetObject -Replace @{msExchRequireAuthToSendTo = $True}

# Set "Require that all senders are authenticated" for all Distribution Groups
$distributionGroups = Get-ADGroup -Filter 'groupcategory -eq "distribution"'
ForEach ($group In $distributionGroups){
	#$group.Name
	Set-ADGroup $group -Replace @{msExchRequireAuthToSendTo = $True}
}

Proliant Gen8 Blade Servers

HP Servers are often chosen as VMWare & Hyper-V hosts. To further optimize these “pizza boxes” for their intended purposes, here are the steps to prep these things:

At cold boot, select the “System Options” menu > choose option “Disabled” for each NICs from “Network Boot” capability

“Power Management Options” menu > “HP Power Profile” > select “Maximum Performance”

“Power Management Options” menu > “Advanced Power Management Options” > select “Maximum Performance” for the “Memory Power Savings Mode”

Set the correct date and time in the “Date and Time” menu
 

“Advanced Options | Advanced System ROM Options” menu > select “Disabled” for the “Power-on Logo

“Advanced Options | Advanced Performance Tuning Options” menu > select “Enabled” for the “ACPI SLIT Preferences”

Also set “Intel Performance Counter Monitor (PCM)”

Proliant Gen9 / Gen10 Servers

⦁ For Synergy Blade Servers the BIOS settings are pre-configured in the Server Profile Template.
⦁ For HPE Synergy Blade servers, use the Composer to create a Server Profile using the ESXi template to apply it to the appropriate server hardware.
⦁ This screenshot provides the expected BIOS settings.

Remove Existing Profile

Run: control.exe > click on Mail

Click on Show Profiles

Click on Remove > Yes > OK

Purge Local Outlook Folder of all previous files

# run these commands in DOS COMMAND or PowerShell
del /q %LocalAppData%\Microsoft\Outlook\*
for /d %x in (%LocalAppData%\Microsoft\Outlook\*) do @rd /s /q "%x"

Trigger Outlook and re-create profile

Start > Outlook.exe > Follow wizard to create new profile

What Problems Does this Solve?

1. This one (after the “phantom” DNS record of mail.domain.com has been removed):
   
2. That one…
   I don’t have a screenshot of a frozen Outlook anymore, but you get the idea.
3. Crashing Outlook cases that are isolated to certain users.

----------------------------
CNAME: autodiscover autodiscover.outlook.com 5 minutes (300 seconds)
CNAME: sip sipdir.online.lync.com 5 minutes
CNAME: lyncdiscover webdir.online.lync.com 5 minutes
CNAME: enterpriseregistration enterpriseregistration.windows.net 5 minutes
CNAME: enterpriseenrollment enterpriseenrollment-s.manage.microsoft.com 5 minutes
TXT: @ v=spf1 include:spf.protection.outlook.com -all 5 minutes
SRV: _sip._tls.@ 100 1 443 sipdir.online.lync.com 5 minutes
SRV: _sipfederationtls._tcp.@ 100 1 5061 sipfed.online.lync.com 5 minutes
MX: <MX-prefix>.mail.protection.outlook.com 5 minutes (Obtain MX-prefix from: https://admin.microsoft.com/AdminPortal/Home#/Domains)
CNAME: imap outlook.office365.com
CNAME: mail outlook.office365.com
CNAME: pop outlook.office365.com
CNAME: smtp smtp.office365.com
----------------------------

# Quick DNS checkup:
nslookup -type=a autodiscover.domain.com
nslookup -type=cname autodiscover.domain.com
nslookup -type=a mail.domain.com 8.8.8.8
nslookup -type=cname mail.domain.com
nslookup -type=mx domain.com 8.8.8.8
nslookup -type=txt domain.com 8.8.8.8
nslookup -type=srv _sip._tls.domain.com 8.8.8.8
nslookup -type=srv _sipfederationtls._tcp.domain.com 8.8.8.8

Symptom:

The Fix:

Run: control.exe > click on Mail

Click on Show Profiles

Click on Add

Input a profile name and click OK

Toggle the radio button next to Manual Setup > Next

Select POP or IMAP > Next

Fill out this screen as shown > click More Settings

At the Internet Email Settings window, click on Outgoing Server and copy this configuration > click on Advanced when ready

Fill out the port numbers and other settings > OK

Click Next  > OK > Set this New-Profile as Default > OK

Run Outlook as Normal and allow it some time to download the entire mailbox from the Office 365 Exchange server.

The Point and Click Method

Prepare a CSV file in this format

EmailAddress
user1@kimconnect.com
user2@kimconnect.com
user3@kimconnect.com

Access Office 365 Exchange Admin Center > Recipients > Migration > click on the + sign > Migrate to Exchange Online

Select “Remote move migration” > Next

Toggle the radio button next to “Specify the users with a CSV file” > click on Choose File

Navigate to the prepared CSV > select it > Open > Next > Verify the FQDN of migration end point (of the on premise Exchange Edge role or Client Access role with MRS proxy feature enabled) > Next

Name the batch > select the Target delivery domain > set bad item limit as 99999 > Next

Select the time schedule to start the batch (e.g. Automatically start) > select Automatically complete the migration > New

Click OK on this warning

Ignore this notification

Go sip some coffee and check back to this Migration status view periodically

PowerShell snippet to initiate repair

# set user alias or email addresses here:
$userAliases="user1","user2"

foreach ($user in $userAliases){
	New-MailboxRepairRequest -Mailbox $user -CorruptionType ProvisionedFolder,SearchFolder,AggregateCounts,Folderview
}

PowerShell command to monitor progress:

# This command doesn't work
# Get-MailboxRepairRequest -Mailbox $userAlias | Format-List

Use Event Viewer to monitor progress:

Eventvwr.exe > Action > Create Custom View > select “By Source > select “MSExchangeIS Mailbox Store” > Event IDs = 10044,10045,01146,10047,10048,10049,10050,10051,10059,10062 > OK

Give this Custom View a name such as “Get-MailboxRepairRequest” > OK

Navigate to Event Viewer > Custom Views > click on Get-MailboxRepairRequest > check for Event ID 10048 to signify a successful completion of a mailbox repair job

Events IDs:

• 10047 A mailbox-level repair request started
• 10064 A Public Folder repair request started
• 10048 The repair request successfully completed
• 10050 The mailbox repair request task skipped a mailbox
• 10059 A database-level repair request started
• 10062 Corruption was detected