Feature/hibp apiv3 authentication (#83)

* see #80 Switch to API Version 3 for HaveIBeenPwned Username check. User needs to provide an API key, this is solved by searching for a password entry with the title "hibp-apikey" in the current database. This API key can be obtained at https://haveibeenpwned.com/API/Key. * see #80 Alpha Version so others with an api key can test the plugin without needing to build it them selves
andrew-schofield · Sep 7, 2019 · 101e287 · 101e287
1 parent 54b7b6c
commit 101e287
Show file tree

Hide file tree

Showing 8 changed files with 192 additions and 34 deletions.
diff --git a/Changelog.md b/Changelog.md
@@ -1,5 +1,9 @@
 # keepass2-haveibeenpwned Changelog
 
+### v1.3.2-alpha.1+hibpv3 - 2019-08-24
+* Upgrade to HIBP v3 Api and add support for api-keys (password of entry with title hibp-apikey)
+  Not tested as registration of API-Keys https://haveibeenpwned.com/api/key is momentarily disabled.
+
 ### v1.3.1 - 2019-02-01
 * Allow cancelling a running breach check.
 * Performance improvements. Thanks to SlightlyMadGargoyle.

diff --git a/HaveIBeenPwned.plgx b/HaveIBeenPwned.plgx
diff --git a/HaveIBeenPwned/BreachCheckers/HaveIBeenPwnedUsername/ApiKeyException.cs b/HaveIBeenPwned/BreachCheckers/HaveIBeenPwnedUsername/ApiKeyException.cs
@@ -0,0 +1,12 @@
+using System;
+
+namespace HaveIBeenPwned.BreachCheckers.HaveIBeenPwnedUsername
+{
+    public class ApiKeyException :Exception
+    {
+        public ApiKeyException(string message)
+            :base(message)
+        {
+        }
+    }
+}
diff --git a/HaveIBeenPwned/BreachCheckers/HaveIBeenPwnedUsername/HaveIBeenPwnedAbortException.cs b/HaveIBeenPwned/BreachCheckers/HaveIBeenPwnedUsername/HaveIBeenPwnedAbortException.cs
@@ -0,0 +1,15 @@
+using System;
+
+namespace HaveIBeenPwned.BreachCheckers.HaveIBeenPwnedUsername
+{
+    /// <summary>
+    /// Exception to signal known Errors, that should abort the current checking mechanism
+    /// </summary>
+    public class HaveIBeenPwnedAbortException : Exception
+    {
+        public HaveIBeenPwnedAbortException(string message)
+        :base(message)
+        {
+        }
+    }
+}
diff --git a/HaveIBeenPwned/BreachCheckers/HaveIBeenPwnedUsername/HaveIBeenPwnedUsernameChecker.cs b/HaveIBeenPwned/BreachCheckers/HaveIBeenPwnedUsername/HaveIBeenPwnedUsernameChecker.cs
@@ -8,13 +8,21 @@
 using KeePass.Plugins;
 using System.Threading.Tasks;
 using System.Drawing;
+using System.Net;
+using System.Security.Policy;
+using System.Text.RegularExpressions;
 using KeePassExtensions;
 using System.Threading;
+using System.Web;
+using KeePassLib.Collections;
 
 namespace HaveIBeenPwned.BreachCheckers.HaveIBeenPwnedUsername
 {
     public class HaveIBeenPwnedUsernameChecker : BaseChecker
     {
+        // Number of attempts to retrieve the breaches/pastes API when the rate limit is hit repeatedly
+        private const int DefaultRetries = 5;
+
         public HaveIBeenPwnedUsernameChecker(HttpClient httpClient, IPluginHost pluginHost)
             : base(httpClient, pluginHost)
         {
@@ -30,12 +38,27 @@ public override string BreachTitle
             get { return "Have I Been Pwned"; }
         }
 
-        public async override Task<List<BreachedEntry>> CheckGroup(PwGroup group, bool expireEntries, bool oldEntriesOnly, bool ignoreDeleted, bool ignoreExpired, IProgress<ProgressItem> progressIndicator, Func<bool> canContinue)
+        public async override Task<List<BreachedEntry>> CheckGroup(PwGroup group, bool expireEntries,
+            bool oldEntriesOnly, bool ignoreDeleted, bool ignoreExpired, IProgress<ProgressItem> progressIndicator,
+            Func<bool> canContinue)
         {
+            progressIndicator.Report(new ProgressItem(0, "Searching for HaveIBeenPwned Api Key..."));
+            try
+            {
+                var apiKey = this.RetrieveApiKey();
+                this.client.DefaultRequestHeaders.Add("hipd-api-key", apiKey);
+            }
+            catch (ApiKeyException e)
+            {
+                MessageBox.Show(e.Message, Resources.MessageTitle, MessageBoxButtons.OK);
+                return Enumerable.Empty<BreachedEntry>().ToList();
+            }
+
             progressIndicator.Report(new ProgressItem(0, "Getting HaveIBeenPwned breach list..."));
-            var entries = group.GetEntries(true).Where(e => (!ignoreDeleted || !e.IsDeleted(pluginHost)) && (!ignoreExpired || !e.Expires)).ToArray();
+            var entries = group.GetEntries(true)
+                .Where(e => (!ignoreDeleted || !e.IsDeleted(pluginHost)) && (!ignoreExpired || !e.Expires)).ToArray();
             var usernames = entries.Select(e => e.Strings.ReadSafe(PwDefs.UserNameField).Trim().ToLower()).Distinct();
-            var breaches = await GetBreaches(progressIndicator, usernames, canContinue);
+            var breaches = await this.GetBreaches(progressIndicator, usernames, canContinue);
             var breachedEntries = new List<BreachedEntry>();
 
             await Task.Run(() =>
@@ -44,7 +67,7 @@ await Task.Run(() =>
                 {
                     var username = breachGrp.Key;
                     var oldestUpdate = entries.Min(e => e.GetPasswordLastModified());
-                    
+
                     foreach (var breach in breachGrp)
                     {
                         if (oldEntriesOnly && oldestUpdate >= breach.BreachDate)
@@ -53,8 +76,11 @@ await Task.Run(() =>
                         }
 
                         var pwEntry =
-                            string.IsNullOrWhiteSpace(breach.Domain) ? null :
-                            entries.FirstOrDefault(e => e.GetUrlDomain() == breach.Domain && breach.Username == e.Strings.ReadSafe(PwDefs.UserNameField).Trim().ToLower());
+                            string.IsNullOrWhiteSpace(breach.Domain)
+                                ? null
+                                : entries.FirstOrDefault(e =>
+                                    e.GetUrlDomain() == breach.Domain && breach.Username ==
+                                    e.Strings.ReadSafe(PwDefs.UserNameField).Trim().ToLower());
                         if (pwEntry != null)
                         {
                             var lastModified = pwEntry.GetPasswordLastModified();
@@ -77,7 +103,8 @@ await Task.Run(() =>
             return breachedEntries;
         }
 
-        private async Task<List<HaveIBeenPwnedUsernameEntry>> GetBreaches(IProgress<ProgressItem> progressIndicator, IEnumerable<string> usernames, Func<bool> canContinue)
+        private async Task<List<HaveIBeenPwnedUsernameEntry>> GetBreaches(IProgress<ProgressItem> progressIndicator,
+            IEnumerable<string> usernames, Func<bool> canContinue)
         {
             List<HaveIBeenPwnedUsernameEntry> allBreaches = new List<HaveIBeenPwnedUsernameEntry>();
             var filteredUsernames = usernames.Where(u => !string.IsNullOrWhiteSpace(u) && !u.StartsWith("{REF:"));
@@ -90,43 +117,141 @@ private async Task<List<HaveIBeenPwnedUsernameEntry>> GetBreaches(IProgress<Prog
                 }
 
                 counter++;
-                progressIndicator.Report(new ProgressItem((uint)((double)counter / filteredUsernames.Count() * 100), string.Format("Checking \"{0}\" for breaches", username)));
-                List<HaveIBeenPwnedUsernameEntry> breaches = null;
-                HttpResponseMessage response = null;
+                progressIndicator.Report(new ProgressItem((uint) ((double) counter / filteredUsernames.Count() * 100),
+                    string.Format("Checking \"{0}\" for breaches", username)));
                 try
+                {            
+                    var breaches = await GetBreachesForUserName(HttpUtility.UrlEncode(username), DefaultRetries);
+                    if (breaches != null)
+                    {
+                        allBreaches.AddRange(breaches);
+                    }
+                }
+                catch (HaveIBeenPwnedAbortException)
                 {
-                    response = await client.GetAsync(new Uri("https://haveibeenpwned.com/api/v2/breachedaccount/" + username));
+                    // error was already reported to user.
+                    break;
                 }
-                catch (Exception ex)
+
+                // hibp has a rate limit of 1500ms
+                await Task.Delay(1600);
+            }
+
+
+            return allBreaches;
+        }
+
+        private async Task<IEnumerable<HaveIBeenPwnedUsernameEntry>> GetBreachesForUserName(string username,
+            int remainingRetries)
+        {
+            IEnumerable<HaveIBeenPwnedUsernameEntry> breaches = Enumerable.Empty<HaveIBeenPwnedUsernameEntry>();
+            HttpResponseMessage response = null;
+            try
+            {
+                response = await client.GetAsync(
+                    new Uri("https://haveibeenpwned.com/api/v3/breachedaccount/" + username));
+            }
+            catch (Exception ex)
+            {
+                throw ex;
+            }
+
+            if (response.IsSuccessStatusCode)
+            {
+                var jsonString = await response.Content.ReadAsStringAsync();
+                breaches = JsonConvert.DeserializeObject<List<HaveIBeenPwnedUsernameEntry>>(jsonString);
+                foreach (var b in breaches)
                 {
-                    throw ex;
+                    b.Username = username;
                 }
+            }
+            else if ((int) response.StatusCode == 429) // The Rate limit of our API Key was exceeded
+            {
+                var whenToRetry = response.Headers.RetryAfter.Delta;
 
-                if (response.IsSuccessStatusCode)
+                if (whenToRetry.HasValue == false)
                 {
-                    var jsonString = await response.Content.ReadAsStringAsync();
-                    breaches = JsonConvert.DeserializeObject<List<HaveIBeenPwnedUsernameEntry>>(jsonString);
-                    breaches.ForEach(b => b.Username = username);
+                    MessageBox.Show(
+                        "The Rate limit for haveibeenpwned.com was exceeded.\n" +
+                        "Unfortunately there was no hint when to retry.\n" +
+                        "Please try the website manually.",
+                        Resources.MessageTitle,
+                        MessageBoxButtons.OK,
+                        MessageBoxIcon.Error);
+
+                    throw new HaveIBeenPwnedAbortException("Rate limit exceeded, missing retry-after header.");
                 }
-                else if (response.StatusCode != System.Net.HttpStatusCode.NotFound)
+                else if (remainingRetries > 0)
                 {
-                    DialogResult dialogButton = MessageBox.Show(string.Format("Unable to check haveibeenpwned.com (returned Status: {0})", response.StatusCode), 
-                                                                Resources.MessageTitle, MessageBoxButtons.OKCancel, MessageBoxIcon.Error);
-                    if (dialogButton == DialogResult.Cancel)
-                    {
-                        break;
-                    }
+                    await Task.Delay(whenToRetry.Value.Add(TimeSpan.FromMilliseconds(100)));
+                    return await this.GetBreachesForUserName(username, remainingRetries--);
                 }
-                if (breaches != null)
+                else
                 {
-                    allBreaches.AddRange(breaches);
+                    // give up to respect the rate limit after several failed attempts, as there is most likely running another process trying to check the API
+                    // the specification does not detail what is considered "consistently exceeded" so I hope 5 tries is a reasonable amount
+                    MessageBox.Show(
+                        "The rate limit for haveibeenpwned.com has been exceeded five times in a row.\n" +
+                        "Please make sure there is no other process querying the API with your api key.",
+                        Resources.MessageTitle,
+                        MessageBoxButtons.OK,
+                        MessageBoxIcon.Error);
+
+                    throw new HaveIBeenPwnedAbortException("Rate limit exceeded five times.");
                 }
-                // hibp has a rate limit of 1500ms
-                await Task.Delay(1600);
             }
-
-
-            return allBreaches;
+            else if (response.StatusCode == HttpStatusCode.Unauthorized)
+            {
+                // Further Requests are useless, as we will always get the Unauthorized Return code.
+                DialogResult dialogResult = MessageBox.Show(
+                    "Unable to check haveibeenpwned.com. Your api key is invalid.",
+                    Resources.MessageTitle, MessageBoxButtons.OK, MessageBoxIcon.Error);
+
+                throw new HaveIBeenPwnedAbortException("Invalid Api Key");
+            }
+            else if (response.StatusCode != System.Net.HttpStatusCode.NotFound)
+            {
+                DialogResult dialogButton = MessageBox.Show(
+                    string.Format("Unable to check haveibeenpwned.com (returned Status: {0})", response.StatusCode),
+                    Resources.MessageTitle, MessageBoxButtons.OKCancel, MessageBoxIcon.Error);
+                if (dialogButton == DialogResult.Cancel)
+                {
+                    throw new HaveIBeenPwnedAbortException(
+                        string.Format("Unable to check haveibeenpwned.com (returned Status: {0})",
+                            response.StatusCode));
+                }
+            }
+
+            return breaches;
+        }
+
+        private string RetrieveApiKey()
+        {
+            const string regex = "^(hibp-?|haveibeenpwned)apikey$";
+            var candidates = new PwObjectList<PwEntry>();
+            var searchParameters = new SearchParameters()
+            {
+                SearchInTitles = true,
+                SearchString = "apikey",
+            };
+            this.pluginHost.Database.RootGroup.SearchEntries(searchParameters, candidates);
+
+            var apiKeys = candidates.Where(x => Regex.IsMatch(x.Strings.ReadSafe(PwDefs.TitleField), regex, RegexOptions.IgnoreCase));
+
+            if (apiKeys.Count() > 1)
+            {
+                throw new ApiKeyException(string.Format("Found more than one api key matching the pattern: {0}", regex));
+            }
+            else if (apiKeys.Any() == false)
+            {
+                throw new ApiKeyException("Found no Api key. Please create an Entry \"hibp-apikey\" in your Database\n" +
+                                          "and set its password to the Api key optained from https://haveibeenpwned.com/API/Key");
+            }
+            else
+            {
+                var key = apiKeys.Single();
+                return key.Strings.ReadSafe(PwDefs.PasswordField);
+            }
         }
     }
-}
+}
diff --git a/HaveIBeenPwned/HaveIBeenPwned.csproj b/HaveIBeenPwned/HaveIBeenPwned.csproj
@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="utf-8"?>
+<?xml version="1.0" encoding="utf-8"?>
 <Project ToolsVersion="14.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
   <Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
   <PropertyGroup>
@@ -83,6 +83,8 @@
     <Compile Include="BreachCheckers\HaveIBeenPwnedPassword\HaveIBeenPwnedPasswordChecker.cs" />
     <Compile Include="BreachCheckers\HaveIBeenPwnedPassword\HaveIBeenPwnedPasswordEntry.cs" />
     <Compile Include="BreachCheckers\CombinedChecker.cs" />
+    <Compile Include="BreachCheckers\HaveIBeenPwnedUsername\ApiKeyException.cs" />
+    <Compile Include="BreachCheckers\HaveIBeenPwnedUsername\HaveIBeenPwnedAbortException.cs" />
     <Compile Include="BreachCheckers\HaveIBeenPwnedUsername\HaveIBeenPwnedUsernameChecker.cs" />
     <Compile Include="BreachCheckers\HaveIBeenPwnedUsername\HaveIBeenPwnedUsernameEntry.cs" />
     <Compile Include="DisplayAttribute.cs" />

diff --git a/VERSION b/VERSION
@@ -1,3 +1,3 @@
 :
-HaveIBeenPwned checker:1.3.1
+HaveIBeenPwned checker:1.3.2-alpha.1+hibpv3
 :
diff --git a/mono/HaveIBeenPwned.dll b/mono/HaveIBeenPwned.dll