chore: Refactor zap and a11y tasks to upload just JSON results

ci/pipeline.yml

@@ -108,6 +108,8 @@ jobs:
             params:
               image: image/image.tar
               additional_tags: src/tag-list.txt
+            get_params:
+              skip_download: true
           - task: deploy
             file: src/ci/partials/deploy.yml
             image: general-task

common/lib/task.py

@@ -88,17 +88,20 @@ def status_error(self, err):
             "message": err
         })
 
-    def upload_file(self):
+    def upload_results(self):
         """upload file to S3"""
-        filename = self.results["artifact"]
-        base = os.path.basename(filename)
-        self.key = f'_tasks/artifacts/{self.task_id}/{base}'
-
-        self.s3_client.upload_file(
-            Filename=filename,
-            Bucket=self.bucket,
-            Key=self.key,
-        )
+        results_dir = self.results["artifact"]
+        self.key = f'_tasks/artifacts/{self.task_id}/'
+
+        for filename in os.listdir(results_dir):
+            base = os.path.basename(filename)
+            s3key = f'{self.key}{base}'
+
+            self.s3_client.upload_file(
+                Filename=f'{results_dir}/{filename}',
+                Bucket=self.bucket,
+                Key=s3key,
+            )
 
     def handler():
         raise NotImplementedError

common/main.py

@@ -11,7 +11,7 @@
         try:
             task.status_start()
             task.results = task.handler()
-            task.upload_file()
+            task.upload_results()
             task.status_end()
         except NotImplementedError:
             """operator didn't write a handler"""

tasks/a11y/definition.py

@@ -33,8 +33,7 @@ def handler(self):
             cf.write(config)
 
         results_dir = '/build-task/results'
-        reports_dir = '/build-task/reports'
-        templates_dir = '/build-task/reporter/templates'
+        output_dir = '/build-task/output'
         os.makedirs(results_dir, exist_ok=True)
 
         # crawl
@@ -66,16 +65,13 @@ def handler(self):
                 with open(os.path.join(results_dir, str(idx)), 'w') as f:
                     json.dump([dict(url=url, error=True)], f)
 
-        # report
         output = run([
             'node',
             'build-task/reporter/generate-report.js',
             '--inputDir',
             results_dir,
             '--outputDir',
-            reports_dir,
-            '--templateDir',
-            templates_dir,
+            output_dir,
             '--target',
             target,
             '--buildId',
@@ -84,6 +80,7 @@ def handler(self):
             config_file
         ], capture_output=True)
 
+        # Keeping until we remove report generation
         # regex test on output for count
         summary_regex = r'Issue Count: (\d+)'
         match = re.search(summary_regex, output.stdout)
@@ -92,12 +89,8 @@ def handler(self):
         except Exception:
             count = 0
 
-        # bundle
-        filename = f'/accessibility-scan-for-{owner}-{repository}-{buildid}'  # noqa: E501
-        shutil.make_archive(filename, 'zip', reports_dir)
-
         return dict(
-            artifact=f'{filename}.zip',
+            artifact=output_dir,
             message=None,
             count=count,
         )

tasks/a11y/reporter/README.md

@@ -5,10 +5,96 @@
 Creates an HTML report from Axe results listing violations, passes, incomplete
 and incompatible results.
 
-Given an `inputDir` containing JSON files from [@axe-core/cli](https://www.npmjs.com/package/@axe-core/cli) for a given `target` and a `templateDir` containing `.ejs` files, the following command creates matching HTML reports in an `outputDir`.
+Given an `inputDir` containing JSON files from [@axe-core/cli](https://www.npmjs.com/package/@axe-core/cli) for a given `target`, the following command creates matching JSON reports in an `outputDir`.
 
 ```sh
 
-node generate-report.js --inputDir results --outputDir reports --templateDir templates --target https://example.gov
+node generate-report.js --inputDir results --outputDir reports --target https://example.gov
 
 ```
+
+## Output
+
+The scan results are stored as JSON objects in the site's S3 bucket under the task id's key.
+The file structure consist of an `index.json` that is used to summarize all of the identified violated rules, list the violations, and provide a path to the results page.
+
+```json
+// index
+{
+  // The site's base url
+  "baseurl": "https://example.gov",
+  // Total number of site pages with rule violations
+  "totalPageCount": 8,
+  // Total Violations
+  "totalViolationsCount": 11,
+  // List of the rules that are violated across the site
+  "violatedRules": [
+    {
+      // Violation ID
+      "id": "rules-x-violated",
+      // Violation impact level
+      "impact": "serious",
+      // Description of violation
+      "description": "This rule x expects this to be y.",
+      // Help text to fix the violation
+      "help": "Update x to be more like y",
+      // List of nodes violating the rule in the site
+      "nodes": [{}],
+      // Error color
+      "color": "error-dark",
+      // Error order
+      "order": 1,
+      // Total number of occurences the rule was violated
+      "total": 8,
+      // If the violation rule should be ignored
+      "ignore": false,
+      // The source used to ignore the rule
+      "ignoreSource": ""
+    }
+  ],
+  "currentPage": 8,
+  // List of all of the report pages with information summarrized the page's violations
+  "reportPages": [
+    {
+      // The ouput scan result page
+      // /sites/:site_id/builds/:build_id/scans/:task_id/axe-results-12345
+      "path": "axe-results-12345",
+      // The url of the site's page with the reported violations
+      "absoluteURL": "https://test.example.gov/page/",
+      // Timestamp of the test
+      "timestamp": "YYYY-MM-DDTHH:ss:SSS",
+      // Total violations found on site's page
+      "violationsCount": 2,
+      // Summary of the violation types an count
+      "groupedViolationsCounts": [{ "name": "serious", "count": 2 }],
+      // Summary of the violation types an count
+      "indexPills": [{ "name": "serious", "count": 2 }],
+      "moreCount": 0
+    }
+  ]
+}
+
+```
+
+The additional results generated are identifying each page of the site that has at least one violation identitfied.
+
+```json
+// axe-result-12345
+{
+    // The url of the site's page with the reported violations
+    "url": "https://test.example.gov/page/",
+    // Timestamp of the test
+    "timestamp": "YYYY-MM-DDTHH:ss:SSS",
+    // An array of rules tested that passed on the page
+    "passes": [],
+    // An object grouping violations based their violation type
+    // Each violation type is an array of the violations identified
+    "groupedViolations": {
+      "critical": [...],
+      "serious": [...],
+      ...
+    },
+    // Total number of violations found on pages
+    "violationsCount": 2
+}
+```

tasks/a11y/reporter/generate-report.js

@@ -1,5 +1,4 @@
 #!/usr/bin/node
-import * as ejs from 'ejs';
 import { Glob } from 'glob'
 import fs from 'fs';
 import groupBy from 'core-js/actual/object/group-by.js';
@@ -54,21 +53,18 @@ function violationEnhancer(violation, config, url) {
     total: violation.nodes.length,
     helpUrl: violation.helpUrl,
     ignore,
-    ignoreSource
+    ignoreSource,
+    urls: [url],
   }
 }
 
-
-async function renderFromTemplate(renderData, accumulator, filePath, outputDir, templateDir, templateName, buildId) {
-
-  const templatePath = path.join(templateDir, templateName);
-  fs.mkdir(outputDir, (err) => {
+async function writeToJSON(data, filePath, outputDir) {
+  fs.mkdir(outputDir, { recursive: true }, (err) => {
     console.error(err)
   })
-  const outputPath = path.join(outputDir, `${filePath}.html`);
-  const template = fs.readFileSync(templatePath, 'utf8');
-  const html = await ejs.render(template, { ...renderData, accumulator, utils, buildId }, { filename: `${templateDir}/${templateName}` })
-  fs.writeFileSync(outputPath, html, 'utf8');
+  const outputPath = path.join(outputDir, `${filePath}.json`);
+  const output = JSON.stringify(data)
+  fs.writeFileSync(outputPath, output, 'utf8');
 }
 
 function groupViolations(allViolations) {
@@ -115,6 +111,7 @@ function keepUniqueObjectsWithTotalSorted(array) {
       acc[uniqueKey] = el;
     } else {
       acc[uniqueKey].total += el.total;
+      acc[uniqueKey].urls.push(...el.urls);
     }
     return acc;
   }, {})).sort((a, b) => a.order - b.order || b.total - a.total || a.id.localeCompare(b.id));
@@ -124,7 +121,6 @@ const argv = minimist(process.argv.slice(2));
 
 let inputPath = argv.inputDir;
 let outputPath = argv.outputDir;
-let templatePath = argv.templateDir;
 let buildId = argv.buildId;
 let configFile = argv.config;
 
@@ -183,10 +179,10 @@ for await (const file of g) {
     let moreCount = groupedViolationsCounts.slice(2).reduce((total, pill) => total + pill.count,
       0,)
 
-    await renderFromTemplate(thisPage.renderData, accumulator, fileName, outputPath, templatePath, "reportPage.ejs", buildId).then(() => {
+    await writeToJSON(thisPage.renderData, fileName, outputPath).then(() => {
 
       accumulator.reportPages.push({
-        path: `${fileName}.html`,
+        path: `${fileName}`,
         absoluteURL: thisPage.renderData.url,
         relativeURL: thisPage.renderData.url.split(accumulator.baseurl)[1],
         timestamp: thisPage.renderData.timestamp,
@@ -200,7 +196,7 @@ for await (const file of g) {
       accumulator.totalViolationsCount += thisPage.renderData.violationsCount;
 
       thisPage.shallowRulesViolated.forEach((rule) => {
-        if (!rule.ignore) accumulator.violatedRules.push(rule)
+        accumulator.violatedRules.push(rule)
       });
 
       // note that url is a sort of user-provided value; it's using whatever the scanned URL was, not the json results filename.
@@ -214,11 +210,10 @@ for await (const file of g) {
     // sort summary by most results per page
     accumulator.reportPages = accumulator.reportPages.sort((a, b) => b.violationsCount - a.violationsCount)
     console.log(`Generating report index for ${buildId}: ${accumulator.violatedRules.length} accessibility violations found in ${accumulator.totalViolationsCount} locations across ${totalLength} URLs.`)
-    await renderFromTemplate(null, accumulator, '/index', outputPath, templatePath, "reportIndex.ejs", buildId).then(console.log(`Report generation for build id: ${buildId} complete; open ${outputPath}/index.html to review.`))
+    await writeToJSON(accumulator, '/index', outputPath).then(console.log(`Report generation for build id: ${buildId} complete; open ${outputPath}/index.html to review.`))
 
     // write summary count to stdout to be picked up by subprocess.run
     console.log(`Issue Count: ${accumulator.violatedRules.length}`)
 
   }
 }
-

tasks/a11y/reporter/package.json

@@ -9,7 +9,7 @@
   "scripts": {
     "lint": "ejslint templates/*",
     "build": "node generate-report.js",
-    "watch": "ejslint templates/* && node --watch-path=./templates --watch-path=./generate-report.js  generate-report.js --inputDir results --outputDir reports --templateDir templates --target https://example.gov --buildId 123456"
+    "watch": "ejslint templates/* && node --watch-path=./templates --watch-path=./generate-report.js  generate-report.js --inputDir results --outputDir reports --target https://example.gov --buildId 123456"
   },
   "devDependencies": {
     "@babel/core": "^7.20.12",

tasks/owasp-zap/definition.py

@@ -27,8 +27,7 @@ def handler(self):
             cf.write(config)
 
         tmp_report = 'report.json'
-        templates_dir = '/build-task/reporter/templates'
-
+        output_dir = '/build-task/output'
         filename = f'/zap-scan-for-{owner}-{repository}-{buildid}.html'
 
         output = run([
@@ -45,10 +44,8 @@ def handler(self):
             'build-task/reporter/generate-report.js',
             '--input',
             f'/zap/wrk/{tmp_report}',
-            '--output',
-            filename,
-            '--templateDir',
-            templates_dir,
+            '--outputDir',
+            output_dir,
             '--target',
             target,
             '--buildId',
@@ -57,6 +54,7 @@ def handler(self):
             config_file
         ], capture_output=True)
 
+        # Keeping until we remove report generation
         # regex test on output for count
         summary_regex = r'Issue Count: (\d+)'
         match = re.search(summary_regex, output.stdout)
@@ -66,7 +64,7 @@ def handler(self):
             count = 0
 
         return dict(
-            artifact=filename,
+            artifact=output_dir,
             message=None,
             count=count,
         )

tasks/owasp-zap/reporter/README.md

@@ -2,10 +2,10 @@
 
 Creates an HTML report from ZAP vulnerability scan
 
-Given an `input` JSON file for a given `target` and a `templateDir` containing `.ejs` files, the following command creates a matching HTML report`.
+Given an `input` JSON file for a given `target`, the following command creates a matching JSON report`.
 
 ```sh
 
-node generate-report.js --input results.json --templateDir templates --target https://example.gov
+node generate-report.js --input results.json --target https://example.gov
 
 ```

tasks/owasp-zap/reporter/generate-report.js

@@ -1,18 +1,18 @@
 #!/usr/bin/node
-import * as ejs from 'ejs';
-import { Glob } from 'glob'
 import fs from 'fs';
 import * as utils from './templates/utils.js';
 import minimist from 'minimist';
 import path from 'path';
 import groupBy from 'core-js/actual/object/group-by.js';
 import { marked } from 'marked';
 
-async function renderFromTemplate(renderData, output, templateDir, templateName, buildId) {
-  const templatePath = path.join(templateDir, templateName);
-  const template = fs.readFileSync(templatePath, 'utf8');
-  const html = await ejs.render(template, { ...renderData, utils, buildId: buildId }, { filename: `${templateDir}/${templateName}` })
-  fs.writeFileSync(output, html, 'utf8');
+async function writeToJSON(data, outputDir) {
+  fs.mkdir(outputDir, { recursive: true }, (err) => {
+    if (err) console.error(err)
+  })
+  const outputPath = path.join(outputDir, 'index.json');
+  const output = JSON.stringify(data)
+  fs.writeFileSync(outputPath, output, 'utf8');
 }
 
 function reparseHTML(str) {
@@ -106,20 +106,17 @@ const argv = minimist(process.argv.slice(2));
 
 const {
   input: inputFile,
-  output,
-  templateDir,
-  buildId,
+  outputDir,
   config: configFile
 } = argv;
 
-console.log(`Generating report page at ${output}`)
+console.log(`Generating report page at ${outputDir}`)
 
 const contents = JSON.parse(fs.readFileSync(inputFile, "utf8"));
 const config = JSON.parse(fs.readFileSync(configFile, "utf8"))
-
 const results = prepareResults(contents, config);
 
-await renderFromTemplate(results, output, templateDir, "report.ejs", buildId).then(console.log(`Report generation complete; open ${output} to review.`))
+await writeToJSON(results, outputDir).then(console.log(`Report generation complete; open ${outputDir} to review.`))
 
 // write summary count to stdout to be picked up by subprocess.run
 console.log(`Issue Count: ${results.site.issueCount}`)