Merge pull request #900 from azsk/develop

Sprint 2001 changes
azsk · Jan 17, 2020 · ce5e1c5 · ce5e1c5
2 parents e5b211e + f3227de
commit ce5e1c5
Show file tree

Hide file tree

Showing 26 changed files with 897 additions and 345 deletions.
diff --git a/src/AzSK.AzureDevOps/AzSK.AzureDevOps.psd1 b/src/AzSK.AzureDevOps/AzSK.AzureDevOps.psd1
@@ -27,7 +27,7 @@
     Copyright              = '(c) 2017 Microsoft Corporation. All rights reserved.'
 
     # Description of the functionality provided by this module
-    Description            = 'DevSecOps Kit for AzureDevOps(AzSK) - Preview'
+    Description            = 'DevSecOps Kit for AzureDevOps (AzSK) - Preview'
 
     # Minimum version of the Windows PowerShell engine required by this module
     PowerShellVersion      = '5.0'

diff --git a/src/AzSK.Framework/Abstracts/CommandBase.ps1 b/src/AzSK.Framework/Abstracts/CommandBase.ps1
@@ -102,6 +102,12 @@ class CommandBase: AzSKRoot {
 		{
 			$folderPath = $this.GetOutputFolderPath();
 			$methodResult = $methodToCall.Invoke(@());
+
+			if(-not $this.DoNotOpenOutputFolder) {
+				if (Test-Path $folderPath) {
+					Invoke-Item -Path $folderPath;
+				}
+			}
 		}
 else {
 
@@ -150,8 +156,9 @@ else {
         $GeneratePDFReport = $this.InvocationContext.BoundParameters["GeneratePDF"];
         try {
             if (-not [string]::IsNullOrEmpty($folderpath)) {
+
                 switch ($GeneratePDFReport) {
-                    None {
+					None {
                         # Do nothing
                     }
                     Landscape {
@@ -172,6 +179,20 @@ else {
         $AttestControlParamFound = $this.InvocationContext.BoundParameters["AttestControls"];
 		if($null -eq $AttestControlParamFound)
 		{
+
+			#If controls are attested then open folder when rescan of attested controls is complete
+			$controlAttested = $false
+			if( ([FeatureFlightingManager]::GetFeatureStatus("EnableScanAfterAttestation","*"))) { 
+				#Global variable "AttestationValue" is set to true when one or more controls are attested in current scan
+				#Ignore if variable AttestationValue is not found
+				if (Get-Variable AttestationValue -Scope Global -ErrorAction Ignore){
+					if ( $Global:AttestationValue){
+						$controlAttested = $true
+					}
+				}
+			}
+
+			if ( !$controlAttested){
 			if((-not $this.DoNotOpenOutputFolder) -and (-not [string]::IsNullOrEmpty($folderPath)))
 			{
 				try
@@ -183,6 +204,7 @@ else {
 					#ignore if any exception occurs
 				}
 			}
+			}
 		}
 		}
         return $folderPath;

diff --git a/src/AzSK.Framework/Abstracts/ServicesSecurityStatus.ps1 b/src/AzSK.Framework/Abstracts/ServicesSecurityStatus.ps1
@@ -485,7 +485,37 @@ class ServicesSecurityStatus: AzSVTCommandBase
                         }
 
                     }   
-                    [AIOrgTelemetryHelper]::PublishEvent( "Partial Commit Details",@{"TotalSVTResources"= $($ScanResourcesList |Measure-Object).Count;"ScanCompletedResourcesCount"=$CompletedResources; "NonScannedResourcesCount" = $IncompleteScans;"ErrorStateResourcesCount"= $InErrorResources;"SubscriptionId"=$this.SubscriptionContext.SubscriptionId;"PartialScanIdentifier"=$this.PartialScanIdentifier;}, $null)
+					[AIOrgTelemetryHelper]::PublishEvent( "Partial Commit Details",@{"TotalSVTResources"= $($ScanResourcesList |Measure-Object).Count;"ScanCompletedResourcesCount"=$CompletedResources; "NonScannedResourcesCount" = $IncompleteScans;"ErrorStateResourcesCount"= $InErrorResources;"SubscriptionId"=$this.SubscriptionContext.SubscriptionId;"PartialScanIdentifier"=$this.PartialScanIdentifier;}, $null)
+
+					#By default below detail partial scan tracker telemetry will be in disabled state 
+					# and only be enabled using feature flag for perticular subscriptions to analaze the CA scan issues
+					# Register/Deregister all listeners to cleanup the memory
+					if([FeatureFlightingManager]::GetFeatureStatus("EnableDetailedResourceTrackerTelemetry",$this.SubscriptionContext.SubscriptionId) -eq $true)
+					{
+						$resourceTrackerEvents = [System.Collections.ArrayList]::new()
+						#Loop through all resource list present in tracker and prepare array of events with common properties like RunIdentifier, SubId,etc
+						foreach($resource in $ScanResourcesList){
+							$resourceEvent = "" | Select-Object Name, Properties, Metrics
+							#RunIdentifier value is not set at this stage. Its value is default. 
+							#Investigation needs to be done base don partialScanIdentifier 
+							#"RunIdentifier" = $this.RunIdentifier; 
+							$Properties = @{
+								"SubscriptionId"= $this.SubscriptionContext.SubscriptionId;
+								"PartialScanIdentifier"=$this.PartialScanIdentifier;
+								"ResourceId" = $resource.Id;
+								"ScanRetryCount" = $resource.ScanRetryCount;
+								"State" = $resource.State;
+								"StateModifiedDate" = $resource.ModifiedDate
+								"TrackerId" = $partialScanMngr.ResourceScanTrackerObj.Id
+							}
+								$resourceEvent.Name = "Partial Tracker Resource Details"
+								$resourceEvent.Properties = $properties
+								$resourceTrackerEvents.Add($resourceEvent) | Out-Null
+						}
+						#Push array of resourcelist to AI telemetry
+						[AIOrgTelemetryHelper]::TrackEvents($resourceTrackerEvents);
+					}
+
                 }
                 catch{
                     #Continue exexution if telemetry is not sent 
@@ -569,7 +599,7 @@ class ServicesSecurityStatus: AzSVTCommandBase
 		#If Scan source is in supported sources or UsePartialCommits switch is available
 		if ($this.UsePartialCommits -or ($baselineControlsDetails.SupportedSources -contains $scanSource))
 		{
-			$partialScanMngr.UpdateResourceScanRetryCount($_.ResourceId);
+			$partialScanMngr.UpdateResourceScanRetryCount($_.ResourceId,$this.SubscriptionContext.SubscriptionId);
 		}
 	}
 

diff --git a/src/AzSK.Framework/Helpers/AIOrgTelemetryHelper.ps1 b/src/AzSK.Framework/Helpers/AIOrgTelemetryHelper.ps1
@@ -498,25 +498,43 @@ static [void] PublishEvent([System.Collections.ArrayList] $servicescantelemetryE
 
         if($type -eq "Usage")
         {
-            Invoke-WebRequest -Uri "https://dc.services.visualstudio.com/v2/track" `
+            $uri = "https://dc.services.visualstudio.com/v2/track"
+        }
+        else {
+            $uri = [WebRequestHelper]::GetApplicationInsightsEndPoint()	
+        }
+
+        try {
+            Invoke-WebRequest -Uri $uri `
             -Method Post `
             -ContentType "application/x-json-stream" `
             -Body $eventJson `
             -UseBasicParsing | Out-Null
         }
-        else {
-                $uri = [WebRequestHelper]::GetApplicationInsightsEndPoint()	
-                Invoke-WebRequest -Uri $uri `
-                -Method Post `
-                -ContentType "application/x-json-stream" `
-                -Body $eventJson `
-                -UseBasicParsing | Out-Null
+        catch{
+            # Error while sending events to telemetry. Encode content to UTF8 and make API call again to handle BOM/special characters
+            if (($null -ne $eventJson)-and ($eventJson.length -gt 0)) {
+                [AIOrgTelemetryHelper]::PostUTF8Content($uri, $eventJson);
+            }
         }
     }
     catch {
 		# Left blank intentionally
 		# Error while sending CA events to telemetry. No need to break the execution.
     }
 }
+hidden static PostUTF8Content($uri, $eventJson) 
+{
+    try {
+        Invoke-WebRequest -Uri $uri `
+            -Method Post `
+            -Body ([System.Text.Encoding]::UTF8.GetBytes($eventJson)) `
+            -ContentType "application/x-json-stream" `
+            -UseBasicParsing
+    }
+    catch {
+        # Error while sending events to telemetry after UTF8 encoding.
+    }
+}
 
 }
diff --git a/src/AzSK.Framework/Helpers/RemoteApiHelper.ps1 b/src/AzSK.Framework/Helpers/RemoteApiHelper.ps1
@@ -22,9 +22,29 @@ class RemoteApiHelper {
                 return $result
             }
             catch {
+                #Error while sending events to Database. Encode content to UTF8 and make API call again to handle BOM/special characters
+                if (($null -ne $content)-and ($content.length -gt 0)) {
+                    return [RemoteApiHelper]::PostUTF8Content($uri, $content, "application/json")
+                }
                 return "ERROR"
             }
     }  
+    hidden static [psobject] PostUTF8Content($uri, $content, $type) 
+    {
+            try {
+                $accessToken = [RemoteApiHelper]::GetAccessToken()
+                $result = Invoke-WebRequest -Uri $([RemoteApiHelper]::ApiBaseEndpoint + $uri) `
+                    -Method Post `
+                    -Body ([System.Text.Encoding]::UTF8.GetBytes($content)) `
+                    -ContentType $type `
+                    -Headers @{"Authorization" = "Bearer $accessToken"} `
+                    -UseBasicParsing
+                return $result
+            }
+            catch {
+                return "ERROR"
+            }
+    }
 
     hidden static [psobject] GetContent($uri, $content, $type) 
     {

diff --git a/src/AzSK.Framework/Managers/AzSKPDFExtension.ps1 b/src/AzSK.Framework/Managers/AzSKPDFExtension.ps1
@@ -40,7 +40,7 @@ class AzSKPDFExtension
 				$selection.Style = "No Spacing"
 
 				# Region Front Page
-				[AzSKPDFExtension]::WriteText($selection, 'DevSecOps Kit for Azure (AzSK)','Title', $true, $true, $false)
+				[AzSKPDFExtension]::WriteText($selection, 'Secure DevOps Kit for Azure (AzSK)','Title', $true, $true, $false)
 				[AzSKPDFExtension]::WriteText($selection, 'Security Report','TOC Heading', $true, $true, $false)
 				$selection.InsertBreak(6)
 				$selection.InsertBreak(6)

diff --git a/src/AzSK.Framework/Managers/PartialScanManager.ps1 b/src/AzSK.Framework/Managers/PartialScanManager.ps1
@@ -156,7 +156,7 @@ class PartialScanManager
 		}
 	}
 
-	[void] UpdateResourceScanRetryCount([string] $resourceId)
+	[void] UpdateResourceScanRetryCount([string] $resourceId,[string] $subscriptionId)
 	{
 		$resourceValues = @();
 		$this.GetResourceScanTrackerObject();
@@ -171,7 +171,11 @@ class PartialScanManager
 				{
 					$resourceValue.State = [ScanState]::ERR
 				}
-				#$this.PersistStorageBlob();
+				if([FeatureFlightingManager]::GetFeatureStatus("EnableStorageBlobPersistPerResource",$($subscriptionId)) -eq $true)
+				{
+					$this.PersistStorageBlob();
+				}
+
 			}
 			else
 			{

diff --git a/src/AzSK/AzSKStaging.psd1 b/src/AzSK/AzSKStaging.psd1
@@ -1,4 +1,4 @@
-#
+#
 # Module manifest for module 'AzSK'
 #
 # Generated by: Microsoft AzSK Team
@@ -27,7 +27,7 @@
     Copyright              = '(c) 2017 Microsoft Corporation. All rights reserved.'
 
     # Description of the functionality provided by this module
-    Description            = 'DevSecOps Kit for Azure (AzSK)'
+    Description            = 'Secure DevOps Kit for Azure (AzSK)'
 
     # Minimum version of the Windows PowerShell engine required by this module
     PowerShellVersion      = '5.0'

diff --git a/src/AzSK/Framework/Configurations/ARMChecker/ARMControls.json b/src/AzSK/Framework/Configurations/ARMChecker/ARMControls.json
@@ -529,10 +529,10 @@
           "recommendation": "Run command 'Set-AzWebApp -Name '<WebAppName>' -ResourceGroupName '<RGName>' -NetFrameworkVersion 'v4.7''. Run 'Get-Help Set-AzWebApp -full' for more help.",
           "severity": "Low",
           "jsonPath": [ "$.properties.siteConfig.netFrameworkVersion", "$.properties.netFrameworkVersion" ],
-          "matchType": "StringSingleToken",
+          "matchType": "RegExpressionSingleToken",
           "data": {
             "type": "Allow",
-            "value": "v4.7",
+            "pattern": "^(v4.0|v4.7)$",
             "isCaseSensitive": false
           }
         },

diff --git a/src/AzSK/Framework/Configurations/FeatureFlighting.json b/src/AzSK/Framework/Configurations/FeatureFlighting.json
@@ -172,6 +172,33 @@
       "DisabledForSubs": [],
       "UnderPreview": false,
       "IsEnabled": true
+    },
+    {
+      "Name": "EnableVnetFixForSub",
+      "Description": "",
+      "Sources": [ "*" ],
+      "EnabledForSubs": [],
+      "DisabledForSubs": [],
+      "UnderPreview": false,
+      "IsEnabled": true
+    },
+    {
+      "Name": "EnableDetailedResourceTrackerTelemetry",
+      "Description": "",
+      "Sources": ["*"],
+      "EnabledForSubs": [],
+      "DisabledForSubs": [],
+      "UnderPreview": false,
+      "IsEnabled": false
+    },
+    {
+      "Name": "EnableStorageBlobPersistPerResource",
+      "Description": "",
+      "Sources": ["*"],
+      "EnabledForSubs": [],
+      "DisabledForSubs": [],
+      "UnderPreview": false,
+      "IsEnabled": true
     }
   ]
-}
+}
diff --git a/src/AzSK/Framework/Configurations/SVT/ControlSettings.json b/src/AzSK/Framework/Configurations/SVT/ControlSettings.json
@@ -41,7 +41,8 @@
       "Microsoft.WindowsAzure.ApiManagement.Gateway.Security.Backend.Protocols.Tls11",
       "Microsoft.WindowsAzure.ApiManagement.Gateway.Security.Backend.Protocols.Ssl30",
       "Microsoft.WindowsAzure.ApiManagement.Gateway.Security.Ciphers.TripleDes168"
-    ]
+    ],
+    "MaxAllowedAPICount": 30
   },
   "VirtualMachine": {
     "Windows": {
@@ -224,9 +225,13 @@
   },
   "CheckPIMCAPolicyTags": false,
   "PIMCAPolicyTags":[],
-  "PIMAppId":"",
+  "PIMAppId":"01fc33a7-78ba-4d2f-a4b7-768e336e890e",
   "ASCAlertsSeverityLevels": [ "High" ],
-  "ASCAlertsThresholdInDays": 30,
+  "ASCAlertsThresholdInDays": {
+    "High": 0,
+    "Medium": 30  
+
+  },
   "WhitelistedMgmtCerts": {
     "Thumbprints": [],
     "ApprovedValidityRangeInDays": 732
@@ -532,5 +537,7 @@
       "RequiredImagePath": "azsktest/akstest",
       "RequiredImageTag": "latest"
     }
-  }
+  },
+  "MaxResourceInventoryObjectsCount": 20000,
+  "AlernateAccountRegularExpressionForOrg": ""
 }