Merge pull request #9 from github/csv-todo

Allow specifying query files with no known expected CSV

Author: Sam Lanning

README.md

@@ -79,7 +79,7 @@ The file should look something like this:
   "expectedResults": {
     "step-01.ql": "step-01.csv",
     "step-02.ql": "step-02.csv",
-    "step-03.ql": "step-03.csv",
+    "step-03.ql": false,
   }
 }
 ```
@@ -104,6 +104,12 @@ and csv files for the expected results.
 * `expectedResults` is an object that maps expected query filenames to a csv
   file detailing what the expected results for this query should be.
   Only the first expression for each row in the query results is checked.
+  If instead of a CSV filename, `false` is used,
+  then the checker will assume that the CSV file has simply
+  never been generated,
+  and will print out the resulting output from the query for you to copy into a
+  new file.
+
 
 To simplify course creation,
 we recommend structuring your course folder like so:
@@ -200,18 +206,19 @@ and add a similar workflow file to the one mentioned above.
 
 When testing the action ([as detailed above](#testing-the-action)),
 when a query that is run produces unexpected results,
+or it is specified as `false` in `config.yml` instead of listing a CSV filename,
 the actual results that it produces are printed out in the console.
 You can then store this output as the relevant CSV file.
 
 So the workflow for adding a new query and CSV file looks like:
 
-* add the query (`.ql` file) to `answers/`
-* add the query to `config.json`
-  (remember that the path should be relative to `answers/`)
-* Test the action (whichever method you prefer)
-* Copy the CSV output to the appropriate file in `image/config/`
+* add the query (`.ql` file) to `answers/`.
+* add the query to the `expectedResults` property in `config.json`,
+  with a starting value of `false`.
+* Test the action (whichever method you prefer).
+* Copy the CSV output to the appropriate file in `image/config/`.
 * Re-test the action to ensure it marks the query
-  as producing the correct results
+  as producing the correct results.
 
 #### Publishing your action
 

codeql-learninglab-check/package/src/check.ts

@@ -38,6 +38,9 @@ export interface Result {
 }
 
 export type ResultsCheck = {
+  // CSV file has not been defined for this query yet
+  status: 'undefined';
+} | {
   status: 'correct';
   count: number;
 } | {

codeql-learninglab-check/package/src/index.ts

@@ -55,7 +55,7 @@ type Config = {
   /**
    * Mapping from query filename to expected results csv file
    */
-  expectedResults: {[id: string]: string};
+  expectedResults: {[id: string]: string | false};
 };
 
 function isConfig(config: any): config is Config {
@@ -74,7 +74,7 @@ function isConfig(config: any): config is Config {
       throw new Error('Configuration property locationPaths must have the placeholder "{line-start}"');
   }
   for (const k of Object.keys(config.expectedResults)) {
-    if (typeof config.expectedResults[k] !== 'string') {
+    if (typeof config.expectedResults[k] !== 'string' && config.expectedResults[k] !== false) {
       throw new Error(`Confiuration property "expectedResults" -> "${k}" must be a string`);
     }
   }
@@ -319,8 +319,13 @@ function isConfig(config: any): config is Config {
     console.log(result.stdout);
     const csvOutput = csvPath(query);
     await execFile('codeql', ['bqrs', 'decode', '--entities=url,string', bqrsOutput, '--format=csv', `--output=${csvOutput}`]);
-    const expectedCSV = path.join(CONFIG_PATH, config.expectedResults[query]);
-    results.set(query, await checkResults(expectedCSV, csvOutput));
+    const relativeExpectedCSV = config.expectedResults[query];
+    if (relativeExpectedCSV) {
+      const expectedCSV = path.join(CONFIG_PATH, relativeExpectedCSV);
+      results.set(query, await checkResults(expectedCSV, csvOutput));
+    } else {
+      results.set(query, {status: 'undefined'});
+    }
   }
 
   for(const entry of results.entries()) {
@@ -330,18 +335,22 @@ function isConfig(config: any): config is Config {
     if (r.status === 'correct') {
       comment += ` (${pluralize(r.count, 'result')})`;
     } else {
-      if (r.results) {
-        comment += ` (${pluralize(r.results.actualCount, 'result')}):\n\n`;
-        comment += r.explanation;
-        comment += `\nExpected query to produce ${r.results.expectedCount} results`;
-        comment += formatResults(config.locationPaths, r.results.missingResults, 'Missing results');
-        comment += formatResults(config.locationPaths, r.results.unexpectedResults, 'Unexpected results');
-        comment += formatResults(config.locationPaths, r.results.extraResults, 'Results selected too many times');
+      if (r.status === 'incorrect') {
+        if (r.results) {
+          comment += ` (${pluralize(r.results.actualCount, 'result')}):\n\n`;
+          comment += r.explanation;
+          comment += `\nExpected query to produce ${r.results.expectedCount} results`;
+          comment += formatResults(config.locationPaths, r.results.missingResults, 'Missing results');
+          comment += formatResults(config.locationPaths, r.results.unexpectedResults, 'Unexpected results');
+          comment += formatResults(config.locationPaths, r.results.extraResults, 'Results selected too many times');
+        } else {
+          comment += r.explanation;
+        }
+        core.setFailed(`Incorrect results for ${query}`);
       } else {
-        comment += r.explanation;
+        console.log(`No CSV defined for ${query}:`);
       }
       // Print CSV in console
-      core.setFailed(`Incorrect results for ${query}`);
       core.startGroup('Actual Results CSV:');
       console.log((await (readFile(csvPath(query)))).toString());
       core.endGroup();

codeql-learninglab-check/publish.sh

@@ -5,8 +5,8 @@ set -x
 
 docker login docker.pkg.github.com -u github-actions -p ${GITHUB_TOKEN}
 
-PREV_IMAGE_VERSION=v0.0.6
-IMAGE_VERSION=v0.0.7
+PREV_IMAGE_VERSION=v0.0.7
+IMAGE_VERSION=v0.0.8
 IMAGE_PATH=docker.pkg.github.com/github/codeql-learninglab-actions/codeql-learninglab-check
 IMAGE_TAG=${IMAGE_PATH}:${IMAGE_VERSION}
 

courses/template/image/config/config.json

@@ -3,6 +3,6 @@
   "locationPaths": "https://github.com/<org>/<repo>/blob/<sha>{path}#L{line-start}-L{line-end}",
   "expectedResults": {
     "step-01.ql": "step-01.csv",
-    "step-02.ql": "step-02.csv"
+    "step-02.ql": "step-01.csv"
   }
 }