From 7a3faaa95e955fd23a45d09a66293762ed4136d3 Mon Sep 17 00:00:00 2001 From: Anika Henke Date: Mon, 18 Dec 2017 23:12:25 +0000 Subject: [PATCH] Generate files --- analysis.json | 124 +++-- assets/stylesheets/application.css | 27 +- index.html | 33 +- results.html | 456 +++++++++++++++++- test-cases.html | 8 +- ...f-at-least-45-1-so-does-not-meet-aaa.html} | 0 ...of-at-least-45-1-so-does-not-meet-aa.html} | 0 7 files changed, 576 insertions(+), 72 deletions(-) rename tests/{colour-and-contrast-large-text-does-not-have-a-contrast-ratio-of-at-least-45:1-so-does-not-meet-aaa.html => colour-and-contrast-large-text-does-not-have-a-contrast-ratio-of-at-least-45-1-so-does-not-meet-aaa.html} (100%) rename tests/{colour-and-contrast-small-text-does-not-have-a-contrast-ratio-of-at-least-45:1-so-does-not-meet-aa.html => colour-and-contrast-small-text-does-not-have-a-contrast-ratio-of-at-least-45-1-so-does-not-meet-aa.html} (100%) diff --git a/analysis.json b/analysis.json index 160cb37..0cf3d1a 100644 --- a/analysis.json +++ b/analysis.json @@ -109,20 +109,30 @@ "identified": 0, "wrong": 0, "false-positive": 0 + }, + "fae": { + "notfound": 70, + "error": 37, + "warning": 3, + "manual": 32, + "different": 0, + "identified": 0, + "wrong": 0, + "false-positive": 0 } }, "totals": { "total": 142, - "detectable": 105, - "undetectable": 37 + "detectable": 118, + "undetectable": 24 }, "percentages": { - "detectable": 74, + "detectable": 83, "tools": { "google": { "detectable": { - "error_warning": 23, - "error_warning_manual": 23 + "error_warning": 20, + "error_warning_manual": 20 }, "total": { "error_warning": 17, @@ -131,8 +141,8 @@ }, "tenon": { "detectable": { - "error_warning": 50, - "error_warning_manual": 50 + "error_warning": 44, + "error_warning_manual": 44 }, "total": { "error_warning": 37, @@ -141,8 +151,8 @@ }, "wave": { "detectable": { - "error_warning": 37, - "error_warning_manual": 39 + "error_warning": 33, + "error_warning_manual": 35 }, "total": { "error_warning": 27, @@ -151,8 +161,8 @@ }, "codesniffer": { "detectable": { - "error_warning": 30, - "error_warning_manual": 46 + "error_warning": 27, + "error_warning_manual": 41 }, "total": { "error_warning": 23, @@ -161,8 +171,8 @@ }, "axe": { "detectable": { - "error_warning": 36, - "error_warning_manual": 38 + "error_warning": 32, + "error_warning_manual": 34 }, "total": { "error_warning": 27, @@ -171,8 +181,8 @@ }, "asqatasun": { "detectable": { - "error_warning": 38, - "error_warning_manual": 58 + "error_warning": 34, + "error_warning_manual": 52 }, "total": { "error_warning": 28, @@ -181,8 +191,8 @@ }, "sortsite": { "detectable": { - "error_warning": 36, - "error_warning_manual": 36 + "error_warning": 32, + "error_warning_manual": 32 }, "total": { "error_warning": 27, @@ -191,8 +201,8 @@ }, "eiii": { "detectable": { - "error_warning": 22, - "error_warning_manual": 22 + "error_warning": 19, + "error_warning_manual": 19 }, "total": { "error_warning": 16, @@ -201,8 +211,8 @@ }, "achecker": { "detectable": { - "error_warning": 41, - "error_warning_manual": 44 + "error_warning": 36, + "error_warning_manual": 39 }, "total": { "error_warning": 30, @@ -211,8 +221,8 @@ }, "nu": { "detectable": { - "error_warning": 18, - "error_warning_manual": 22 + "error_warning": 16, + "error_warning_manual": 19 }, "total": { "error_warning": 13, @@ -221,13 +231,23 @@ }, "siteimprove": { "detectable": { - "error_warning": 39, - "error_warning_manual": 49 + "error_warning": 35, + "error_warning_manual": 43 }, "total": { "error_warning": 29, "error_warning_manual": 36 } + }, + "fae": { + "detectable": { + "error_warning": 34, + "error_warning_manual": 61 + }, + "total": { + "error_warning": 28, + "error_warning_manual": 51 + } } } }, @@ -253,48 +273,54 @@ }, { "position": 4, + "name": "fae", + "error_warning": 28, + "error_warning_manual": 51 + }, + { + "position": 5, "name": "asqatasun", "error_warning": 28, "error_warning_manual": 43 }, { - "position": 5, - "name": "axe", + "position": 6, + "name": "wave", "error_warning": 27, - "error_warning_manual": 28 + "error_warning_manual": 29 }, { - "position": 6, + "position": 7, "name": "sortsite", "error_warning": 27, "error_warning_manual": 27 }, { - "position": 7, - "name": "wave", + "position": 8, + "name": "axe", "error_warning": 27, - "error_warning_manual": 29 + "error_warning_manual": 28 }, { - "position": 8, + "position": 9, "name": "codesniffer", "error_warning": 23, "error_warning_manual": 34 }, { - "position": 9, + "position": 10, "name": "google", "error_warning": 17, "error_warning_manual": 17 }, { - "position": 10, + "position": 11, "name": "eiii", "error_warning": 16, "error_warning_manual": 16 }, { - "position": 11, + "position": 12, "name": "nu", "error_warning": 13, "error_warning_manual": 16 @@ -303,66 +329,72 @@ "by_error_warning_manual": [ { "position": 1, + "name": "fae", + "error_warning": 28, + "error_warning_manual": 51 + }, + { + "position": 2, "name": "asqatasun", "error_warning": 28, "error_warning_manual": 43 }, { - "position": 2, + "position": 3, "name": "tenon", "error_warning": 37, "error_warning_manual": 37 }, { - "position": 3, + "position": 4, "name": "siteimprove", "error_warning": 29, "error_warning_manual": 36 }, { - "position": 4, + "position": 5, "name": "codesniffer", "error_warning": 23, "error_warning_manual": 34 }, { - "position": 5, + "position": 6, "name": "achecker", "error_warning": 30, "error_warning_manual": 32 }, { - "position": 6, + "position": 7, "name": "wave", "error_warning": 27, "error_warning_manual": 29 }, { - "position": 7, + "position": 8, "name": "axe", "error_warning": 27, "error_warning_manual": 28 }, { - "position": 8, + "position": 9, "name": "sortsite", "error_warning": 27, "error_warning_manual": 27 }, { - "position": 9, + "position": 10, "name": "google", "error_warning": 17, "error_warning_manual": 17 }, { - "position": 10, + "position": 11, "name": "eiii", "error_warning": 16, "error_warning_manual": 16 }, { - "position": 11, + "position": 12, "name": "nu", "error_warning": 13, "error_warning_manual": 16 diff --git a/assets/stylesheets/application.css b/assets/stylesheets/application.css index 53cc80c..c4c805b 100644 --- a/assets/stylesheets/application.css +++ b/assets/stylesheets/application.css @@ -2160,7 +2160,8 @@ body { font-family: "GDS-Logo", sans-serif; } -body.results, body .intro-summary { +body.results, +body .intro-summary { font-family: "GDS-Logo", sans-serif; font-size: 16px; line-height: 1.25; @@ -2169,7 +2170,8 @@ body.results, body .intro-summary { } @media (min-width: 641px) { - body.results, body .intro-summary { + body.results, + body .intro-summary { font-size: 19px; line-height: 1.31579; } @@ -2458,7 +2460,8 @@ table { } } -table td, table th { +table td, +table th { vertical-align: top; text-align: left; padding: 10px; @@ -2474,7 +2477,8 @@ table.entry tr { background-color: #f5f5f5; } -table.entry th, table.entry td { +table.entry th, +table.entry td { border-bottom: 1px solid #bfc1c3; } @@ -2494,7 +2498,8 @@ table.numeric-results th { font-weight: bold; } -table.numeric-results td, table.numeric-results thead th { +table.numeric-results td, +table.numeric-results thead th { text-align: right; } @@ -2554,6 +2559,15 @@ h3 { top: 0; } +#results-table th, +#results-table td { + padding-right: 5px; +} + +#results-table th { + hyphens: manual; +} + #results-table .error:before { content: ""; background: #43b93c; @@ -2631,7 +2645,8 @@ h3 { padding: 0; } -.summary-data .timestamp, .summary-data .result-type { +.summary-data .timestamp, +.summary-data .result-type { font-weight: 700; display: block; } diff --git a/index.html b/index.html index 24cbb82..dd8d287 100644 --- a/index.html +++ b/index.html @@ -34,7 +34,7 @@

How do automated accessibility checkers compare?

- We ran an audit to see what issues the different accessibility testing tools pick up. We deliberately introduced 142 accessibility barriers to a page of content, and ran them through 11 automated tools. We also had a look at features like usability and cost. + We ran an audit to see what issues the different accessibility testing tools pick up. We deliberately introduced 142 accessibility barriers to a page of content, and ran them through 12 automated tools. We also had a look at features like usability and cost.

You can use our audit to help choose the best accessibility testing tool for your service.

@@ -57,7 +57,7 @@

What we found

The second column of results takes into account potential barriers that tools noticed, but needed a human being to check, like whether alt text descriptions were accurate. A tool that flags things like this can help you pick up more issues overall.

-

The best performing tool in this category picked up 43% of our deliberate mistakes.

+

The best performing tool in this category picked up 51% of our deliberate mistakes.

How did each tool do?

@@ -83,22 +83,26 @@

How did each tool do?

Siteimprove 29% 36% + + FAE + 28% + 51% Asqatasun 28% 43% - aXe + WAVE 27% - 28% + 29% SortSite 27% 27% - WAVE + aXe 27% - 29% + 28% HTML_CodeSniffer 23% @@ -119,7 +123,7 @@

How did each tool do?

-

Last updated: 13 December 2017

+

Last updated: 19 December 2017

How do features compare?

@@ -384,6 +388,21 @@

Usability

Yes Good + + FAE + Free + N/A + Yes + Yes + No + No + Yes + Yes + Yes + No + Yes + OK + diff --git a/results.html b/results.html index e3f090a..c55c47c 100644 --- a/results.html +++ b/results.html @@ -40,7 +40,7 @@

Numbers

- 11 + 12 tools tested

@@ -52,7 +52,7 @@

Numbers

-

We sometimes update specific test results for individual tools. See the changelog for full details. We last made updates on 13 December 2017.

+

We sometimes update specific test results for individual tools. See the changelog for full details. We last made updates on 19 December 2017.

@@ -120,14 +120,14 @@

Negative results

Test case - Google Accessibility Developer Tools + Google Accessi­bility Developer Tools Tenon WAVE - HTML CodeSniffer + HTML_­Code­Sniffer aXe - Asqatasun - SortSite + Asqa­tasun + Sort­Site EIII @@ -139,7 +139,12 @@

Negative results

Nu Html Checker - Siteimprove + Site­improve + + + FAE + + @@ -183,6 +188,9 @@

Negative results

Not found + + User to check + @@ -224,6 +232,9 @@

Negative results

Not found + + Not found + @@ -265,6 +276,9 @@

Negative results

Not found + + Not found + @@ -306,6 +320,9 @@

Negative results

Not found + + Not found + @@ -347,6 +364,9 @@

Negative results

Not found + + Not found + @@ -390,6 +410,9 @@

Negative results

Not found + + User to check + @@ -433,11 +456,14 @@

Negative results

Not found + + User to check + - + Small text does not have a contrast ratio of at least 4.5:1 so does not meet AA @@ -474,6 +500,9 @@

Negative results

Issue found + + Issue found + @@ -515,6 +544,9 @@

Negative results

Issue found + + Not found + @@ -556,11 +588,14 @@

Negative results

Issue found + + Issue found + - + Large text does not have a contrast ratio of at least 4.5:1 so does not meet AAA @@ -597,6 +632,9 @@

Negative results

Issue found + + Not found + @@ -638,6 +676,9 @@

Negative results

Not found + + User to check + @@ -681,6 +722,9 @@

Negative results

Not found + + Not found + @@ -722,6 +766,9 @@

Negative results

Not found + + Not found + @@ -763,6 +810,9 @@

Negative results

Not found + + User to check + @@ -804,6 +854,9 @@

Negative results

Not found + + Not found + @@ -845,6 +898,9 @@

Negative results

Issue found + + Issue found + @@ -886,6 +942,9 @@

Negative results

Not found + + Not found + @@ -927,6 +986,9 @@

Negative results

Not found + + Not found + @@ -968,6 +1030,9 @@

Negative results

Not found + + Not found + @@ -1011,6 +1076,9 @@

Negative results

Not found + + Not found + @@ -1052,6 +1120,9 @@

Negative results

Issue found + + Issue found + @@ -1093,6 +1164,9 @@

Negative results

Not found + + User to check + @@ -1134,6 +1208,9 @@

Negative results

Not found + + Not found + @@ -1175,6 +1252,9 @@

Negative results

Issue found + + Issue found + @@ -1216,6 +1296,9 @@

Negative results

Issue found + + Issue found + @@ -1257,6 +1340,9 @@

Negative results

Issue found + + Issue found + @@ -1298,6 +1384,9 @@

Negative results

Not found + + Not found + @@ -1339,6 +1428,9 @@

Negative results

Not found + + Not found + @@ -1382,6 +1474,9 @@

Negative results

Not found + + User to check + @@ -1423,6 +1518,9 @@

Negative results

Issue found + + Issue found + @@ -1464,6 +1562,9 @@

Negative results

Issue found + + Issue found + @@ -1507,6 +1608,9 @@

Negative results

Issue found + + Warning only + @@ -1548,6 +1652,9 @@

Negative results

Issue found + + Warning only + @@ -1589,6 +1696,9 @@

Negative results

Not found + + Not found + @@ -1630,6 +1740,9 @@

Negative results

Issue found + + Issue found + @@ -1673,6 +1786,9 @@

Negative results

Not found + + User to check + @@ -1714,6 +1830,9 @@

Negative results

Not found + + Not found + @@ -1755,6 +1874,9 @@

Negative results

Not found + + Not found + @@ -1796,6 +1918,9 @@

Negative results

Not found + + User to check + @@ -1839,6 +1964,9 @@

Negative results

User to check + + Issue found + @@ -1880,6 +2008,9 @@

Negative results

Not found + + Not found + @@ -1921,6 +2052,9 @@

Negative results

Not found + + Not found + @@ -1962,6 +2096,9 @@

Negative results

Not found + + Not found + @@ -2003,6 +2140,9 @@

Negative results

Not found + + Issue found + @@ -2044,6 +2184,9 @@

Negative results

User to check + + Not found + @@ -2085,6 +2228,9 @@

Negative results

Not found + + Not found + @@ -2126,6 +2272,9 @@

Negative results

Issue found + + Issue found + @@ -2167,6 +2316,9 @@

Negative results

Not found + + User to check + @@ -2208,6 +2360,9 @@

Negative results

Issue found + + Not found + @@ -2249,6 +2404,9 @@

Negative results

Not found + + Not found + @@ -2292,6 +2450,9 @@

Negative results

Not found + + Not found + @@ -2333,6 +2494,9 @@

Negative results

Not found + + Not found + @@ -2374,6 +2538,9 @@

Negative results

Issue found + + Issue found + @@ -2415,6 +2582,9 @@

Negative results

Not found + + Not found + @@ -2456,6 +2626,9 @@

Negative results

Issue found + + Not found + @@ -2497,6 +2670,9 @@

Negative results

Not found + + User to check + @@ -2538,6 +2714,9 @@

Negative results

User to check + + User to check + @@ -2579,6 +2758,9 @@

Negative results

Not found + + User to check + @@ -2620,6 +2802,9 @@

Negative results

Issue found + + Issue found + @@ -2661,6 +2846,9 @@

Negative results

Not found + + User to check + @@ -2704,6 +2892,9 @@

Negative results

User to check + + User to check + @@ -2745,6 +2936,9 @@

Negative results

Not found + + Not found + @@ -2786,6 +2980,9 @@

Negative results

User to check + + Issue found + @@ -2829,6 +3026,9 @@

Negative results

Issue found + + Issue found + @@ -2870,6 +3070,9 @@

Negative results

Not found + + Not found + @@ -2911,6 +3114,9 @@

Negative results

Issue found + + User to check + @@ -2952,6 +3158,9 @@

Negative results

Not found + + Not found + @@ -2993,6 +3202,9 @@

Negative results

Not found + + Not found + @@ -3034,6 +3246,9 @@

Negative results

Not found + + Not found + @@ -3075,6 +3290,9 @@

Negative results

Not found + + Not found + @@ -3116,6 +3334,9 @@

Negative results

Not found + + User to check + @@ -3157,6 +3378,9 @@

Negative results

Not found + + Not found + @@ -3198,6 +3422,9 @@

Negative results

Not found + + Not found + @@ -3239,6 +3466,9 @@

Negative results

Not found + + Issue found + @@ -3280,6 +3510,9 @@

Negative results

Issue found + + Warning only + @@ -3321,6 +3554,9 @@

Negative results

Not found + + User to check + @@ -3362,6 +3598,9 @@

Negative results

Issue found + + Not found + @@ -3403,6 +3642,9 @@

Negative results

Not found + + User to check + @@ -3444,6 +3686,9 @@

Negative results

Issue found + + Not found + @@ -3485,6 +3730,9 @@

Negative results

Issue found + + Not found + @@ -3526,6 +3774,9 @@

Negative results

Not found + + Not found + @@ -3567,6 +3818,9 @@

Negative results

Issue found + + User to check + @@ -3608,6 +3862,9 @@

Negative results

Not found + + Not found + @@ -3651,6 +3908,9 @@

Negative results

Issue found + + Issue found + @@ -3692,6 +3952,9 @@

Negative results

Not found + + Issue found + @@ -3733,6 +3996,9 @@

Negative results

Not found + + Not found + @@ -3774,6 +4040,9 @@

Negative results

Issue found + + Issue found + @@ -3817,6 +4086,9 @@

Negative results

Not found + + User to check + @@ -3858,6 +4130,9 @@

Negative results

Issue found + + Issue found + @@ -3899,6 +4174,9 @@

Negative results

Not found + + User to check + @@ -3940,6 +4218,9 @@

Negative results

Not found + + Not found + @@ -3981,6 +4262,9 @@

Negative results

Not found + + Issue found + @@ -4022,6 +4306,9 @@

Negative results

Issue found + + Issue found + @@ -4063,6 +4350,9 @@

Negative results

Not found + + Issue found + @@ -4104,6 +4394,9 @@

Negative results

Not found + + Issue found + @@ -4145,6 +4438,9 @@

Negative results

Issue found + + Issue found + @@ -4186,6 +4482,9 @@

Negative results

Not found + + Not found + @@ -4227,6 +4526,9 @@

Negative results

Issue found + + Issue found + @@ -4268,6 +4570,9 @@

Negative results

Not found + + Not found + @@ -4309,6 +4614,9 @@

Negative results

Issue found + + Issue found + @@ -4350,6 +4658,9 @@

Negative results

Not found + + Issue found + @@ -4391,6 +4702,9 @@

Negative results

Issue found + + Issue found + @@ -4432,6 +4746,9 @@

Negative results

Not found + + Not found + @@ -4473,6 +4790,9 @@

Negative results

Issue found + + Issue found + @@ -4514,6 +4834,9 @@

Negative results

Not found + + User to check + @@ -4555,6 +4878,9 @@

Negative results

Not found + + User to check + @@ -4598,6 +4924,9 @@

Negative results

Not found + + Not found + @@ -4641,6 +4970,9 @@

Negative results

User to check + + User to check + @@ -4682,6 +5014,9 @@

Negative results

Not found + + Issue found + @@ -4723,6 +5058,9 @@

Negative results

User to check + + User to check + @@ -4764,6 +5102,9 @@

Negative results

User to check + + Not found + @@ -4805,6 +5146,9 @@

Negative results

Issue found + + User to check + @@ -4846,6 +5190,9 @@

Negative results

Not found + + Not found + @@ -4887,6 +5234,9 @@

Negative results

Not found + + Issue found + @@ -4928,6 +5278,9 @@

Negative results

Not found + + User to check + @@ -4969,6 +5322,9 @@

Negative results

Not found + + Not found + @@ -5010,6 +5366,9 @@

Negative results

Not found + + Not found + @@ -5051,6 +5410,9 @@

Negative results

User to check + + User to check + @@ -5092,6 +5454,9 @@

Negative results

Not found + + User to check + @@ -5133,6 +5498,9 @@

Negative results

Not found + + Not found + @@ -5174,6 +5542,9 @@

Negative results

Not found + + Not found + @@ -5215,6 +5586,9 @@

Negative results

Not found + + Not found + @@ -5256,6 +5630,9 @@

Negative results

Not found + + Issue found + @@ -5299,6 +5676,9 @@

Negative results

Issue found + + Issue found + @@ -5340,6 +5720,9 @@

Negative results

Not found + + Not found + @@ -5383,6 +5766,9 @@

Negative results

Not found + + User to check + @@ -5424,6 +5810,9 @@

Negative results

Not found + + Not found + @@ -5465,6 +5854,9 @@

Negative results

Not found + + Not found + @@ -5506,6 +5898,9 @@

Negative results

Not found + + Not found + @@ -5547,6 +5942,9 @@

Negative results

Not found + + User to check + @@ -5590,6 +5988,9 @@

Negative results

Issue found + + Not found + @@ -5631,6 +6032,9 @@

Negative results

User to check + + Not found + @@ -5672,6 +6076,9 @@

Negative results

Not found + + Not found + @@ -5713,6 +6120,9 @@

Negative results

Issue found + + Not found + @@ -5754,6 +6164,9 @@

Negative results

Issue found + + Issue found + @@ -5795,6 +6208,9 @@

Negative results

Not found + + Not found + @@ -5836,6 +6252,9 @@

Negative results

Not found + + Not found + @@ -5877,6 +6296,9 @@

Negative results

Not found + + Not found + @@ -5918,6 +6340,9 @@

Negative results

Not found + + Not found + @@ -5959,6 +6384,9 @@

Negative results

Not found + + Not found + @@ -6000,6 +6428,9 @@

Negative results

Issue found + + Not found + @@ -6011,6 +6442,13 @@

Changelog

    +
  1. + + + Added new tool: FAE + +
    2. +
  2. diff --git a/test-cases.html b/test-cases.html index 88d6735..028431d 100644 --- a/test-cases.html +++ b/test-cases.html @@ -159,8 +159,8 @@

    - +

    + Small text does not have a contrast ratio of at least 4.5:1 so does not meet AA

    @@ -189,8 +189,8 @@

    This small text does not have enough contrast with it's background

-

- +

+ Large text does not have a contrast ratio of at least 4.5:1 so does not meet AAA

diff --git a/tests/colour-and-contrast-large-text-does-not-have-a-contrast-ratio-of-at-least-45:1-so-does-not-meet-aaa.html b/tests/colour-and-contrast-large-text-does-not-have-a-contrast-ratio-of-at-least-45-1-so-does-not-meet-aaa.html similarity index 100% rename from tests/colour-and-contrast-large-text-does-not-have-a-contrast-ratio-of-at-least-45:1-so-does-not-meet-aaa.html rename to tests/colour-and-contrast-large-text-does-not-have-a-contrast-ratio-of-at-least-45-1-so-does-not-meet-aaa.html diff --git a/tests/colour-and-contrast-small-text-does-not-have-a-contrast-ratio-of-at-least-45:1-so-does-not-meet-aa.html b/tests/colour-and-contrast-small-text-does-not-have-a-contrast-ratio-of-at-least-45-1-so-does-not-meet-aa.html similarity index 100% rename from tests/colour-and-contrast-small-text-does-not-have-a-contrast-ratio-of-at-least-45:1-so-does-not-meet-aa.html rename to tests/colour-and-contrast-small-text-does-not-have-a-contrast-ratio-of-at-least-45-1-so-does-not-meet-aa.html