Set up test runner (#7)

* use Dyalog docker image

* set up runner #3

* actually run a test

* update dyalogscript file permissions

* modify docker setup

* write results to /tmp directory

* reset docker run scripts to match production

* cleanup

* remove redundant WORKDIR command

* set WORKDIR in Dockerfile

* updated to use Link, which works read-only even if SALT init fails

* No need for CDShy

* Assert injected by test framework

* ensure testns expunged for debugging

* CD from Dyalog to solution dir to run tests

* Using link, no longer need 2⎕FIX

* get test code from source

* fix expected results for success test

* loop over tests

* exit 0 is default

* add tests

* fix empty-file test directory for docker

* use UNIX LF line endings for test files

* Update bin/run-tests.sh

Co-authored-by: Isaac Good <[email protected]>

* Update bin/run-tests.sh

Co-authored-by: Isaac Good <[email protected]>

* Update bin/run.sh

Co-authored-by: Isaac Good <[email protected]>

* don't capture output, remove optional comments

* swap error one ←→ error single

* overall status error if all tests error

* test run slug is test_dir_name

---------

Co-authored-by: Isaac Good <[email protected]>

Author: rikedyp

APLSource/Assert.aplf

@@ -0,0 +1 @@
+Assert←{ 0∊⍵: ⎕SIGNAL ⊂('EN' 8)('EM' 'ASSERTION ERROR')('Message' ⍺) ⋄ shy←0}

APLSource/CD.aplf

@@ -0,0 +1,38 @@
+ old←force CD new;OS;is64;t;get;lib;chdir;⎕ML ⍝ Current/Change Directory
+    ⍝ empty "new": only report; non-empty "new": change dir
+    ⍝ warns about tied files with relative tie names if force=0, skips check if force=1
+ ⎕ML←1
+ OS←3↑⊃# ⎕WG'APLVersion'
+ :If 'Win'≡OS                 ⍝ Under Windows we do the association here because we need
+     t←'A*'[1+80=⎕DR'']       ⍝ the programs to GET the current directory
+     'get'⎕NA'U Kernel32|GetCurrentDirectory',t,' U >0T' ⍝ Associate Get function
+     old←2⊃get 260 260                                   ⍝ in all cases we return the current dir
+     'chdir'⎕NA'U Kernel32|SetCurrentDirectory',t,' <0T' ⍝ Associate Set function
+ :Else
+     old←⊃⎕SH'pwd'
+ :EndIf
+
+ :If ~0∊⍴new         ⍝ change directory
+
+     :If 'Win'≡OS
+             ⍝ chdir returns a single int: 1=ok
+         ('Unable to change directory: ',new,': Invalid directory')⎕SIGNAL(chdir⊂new)↓22
+     :Else
+         :Select OS
+         :Case 'AIX'
+             is64←32={z←⍵ ⋄ ⎕SIZE'z'}⍬
+             lib←'I libc.a(shr',(is64/'_64'),'.o)|chdir <0T1[]'
+         :Case 'Lin'
+             ⍝ Extract real name of libc that we are actually using
+             t←⊃⎕SH'echo $PPID'
+             t←⎕SH'ldd /proc/',t,'/exe'
+             lib←⊃('^\s*libc\.so\b.*=>\s*(\S*)'⎕S'\1')t
+             lib←'I ',lib,'|chdir <0T1[]'
+         :Case 'Mac'
+             lib←'I /usr/lib/libc.dylib|chdir <0T1[]'
+                  ⍝ chdir returns a single integer: 0=ok
+         :EndSelect
+         ⎕NA lib
+         ('Unable to change directory: ',new,': Invalid directory')⎕SIGNAL(0≠chdir⊂new)/22
+     :EndIf
+ :EndIf

APLSource/DoTest.aplf

@@ -0,0 +1,12 @@
+ DoTest←{
+    ⍝ Execute test function ⍵
+    ⍝ In namespace ⍺
+     ⍺←⎕THIS
+     _←(⍕⍺)⎕NS'Assert'   ⍝ Inject Assert function into test namespace
+     nl←⎕UCS 13
+     Format←{1↓∊nl∘,¨⊆⍵}
+     0::'error'(Format ⎕DMX.DM)
+     8::'fail'(Format ⎕DMX.Message)
+     1=⍺⍎⍵:'pass' ''
+     'fail' 'Test failed.'
+ }

APLSource/Run.aplf

@@ -0,0 +1,20 @@
+ Run←{
+⍝ ⍵: Directory of test suite to run
+⍝ ←: Namespace of test results (can be output with (1⎕JSON ns)⎕NPUT filename )
+
+     r←⍎'result'⎕NS''⊣⎕EX'result'
+     r.version←2
+     0::r⊣r.status←'error'⊣r.message←'Could not load test suite ',⍵
+     settings.overwrite←1⊣settings←⎕NS''
+     link←settings ⎕SE.Link.Import'testns'⍵⊣⎕EX'testns'
+     tests←'Test_'testns.⎕NL ¯3
+     0∊⍴tests:r⊣r.status←'error'⊣r.message←'No test files found in directory ',⍵
+     r.tests←⎕NS¨''⍨¨tests
+     r.tests.name←tests
+     r.tests.test_code←testns.⎕VR¨tests
+     r.tests.(status message)←testns∘DoTest¨tests
+     status←'fail' 'error' 'pass'
+     r.status←status⊃⍨1+0 1 2+.×∧/status∘.≡r.tests.status
+
+     r
+ }

APLSource/Version.aplf

@@ -0,0 +1,2 @@
+v←Version
+v←'0.1.0'

Dockerfile

@@ -1,8 +1,5 @@
-FROM alpine:3.18
-
-# install packages required to run the tests
-RUN apk add --no-cache jq coreutils
+FROM dyalog/dyalog:19.0
 
+COPY . /opt/test-runner
 WORKDIR /opt/test-runner
-COPY . .
 ENTRYPOINT ["/opt/test-runner/bin/run.sh"]

README.md

@@ -2,15 +2,21 @@
 
 The Docker image to automatically run tests on Dyalog APL solutions submitted to [Exercism].
 
-## Getting started
+## Developing the Runner
+The runner consists of functions in the **APLSource** directory. It is developed using Link.
 
-Build the test runner, conforming to the [Test Runner interface specification](https://github.com/exercism/docs/blob/main/building/tooling/test-runners/interface.md).
-Update the files to match your track's needs. At the very least, you'll need to update `bin/run.sh`, `Dockerfile` and the test solutions in the `tests` directory
+In Dyalog:
 
-- Tip: look for `TODO:` comments to point you towards code that need updating
-- Tip: look for `OPTIONAL:` comments to point you towards code that _could_ be useful
-- Tip: if it proves impossible for the Docker image to work on a read-only filesystem, remove the `--read-only` flag from the `bin/run-in-docker.sh` and `bin/run-tests-in-docker.sh` files.
-  We don't yet enforce a read-only file system in production, but we might in the future!
+```apl
+]LINK.Create # APLSource
+```
+
+The `Run` function is the interface that simply returns JSON text of the results object. The script **bin/dylaog-apl-runner.apls** gets parameters from the command line and outputs the result of the `Run` function to the file specified.
+
+## Design
+This implementation of the version 2 test runner interface allows multiple tests: one APL function per test. This is one **.aplf** file per function, which allows natural development and debugging using [Link](https://dyalog.github.io/link).
+
+The tests make use of an `Assert` function which takes a left argument that is the message to be displayed to the user in case the test case fails. In this way, a single test function can test multiple aspects of an exercise and usefully report failures back to the user.
 
 ## Run the test runner
 

bin/dyalog-apl-runner.apls

@@ -0,0 +1,19 @@
+#!/usr/bin/dyalogscript DYALOG_INITSESSION=1
+⍝ Outputs JSON text to file describing the results of the test run
+
+⍝ Import test runner
+⎕SE.Link.Import 'Runner' 'APLSource'
+
+⍝ Get from envrionment rather than arguments so that we do not have to check count or order of parameters
+(slug soln outdir outfile)←{2⎕NQ#'GetEnvironment' ⍵}¨'SLUG' 'SOLN' 'OUTDIR' 'OUTFILE'
+
+home ← 0 Runner.CD soln
+
+results←Runner.Run soln ⍝ Why slug needed if solution expected to be in solution folder?
+output←1(⎕JSON⎕OPT'Compact'0)results   ⍝ Readable (non-compact) JSON result
+output (⎕NPUT⎕OPT'NEOL'2) outfile 1    ⍝ Output using OS line endings (NEOL 2)
+
+0 Runner.CD home
+
+⎕←'Test results:'
+⎕←results.status

bin/run-tests.sh

@@ -12,24 +12,22 @@
 # ./bin/run-tests.sh
 
 exit_code=0
+echo "current directory ${PWD}"
 
 # Iterate over all test directories
 for test_dir in tests/*; do
     test_dir_name=$(basename "${test_dir}")
     test_dir_path=$(realpath "${test_dir}")
-
-    bin/run.sh "${test_dir_name}" "${test_dir_path}" "${test_dir_path}"
-
-    # OPTIONAL: Normalize the results file
-    # If the results.json file contains information that changes between 
-    # different test runs (e.g. timing information or paths), you should normalize
-    # the results file to allow the diff comparison below to work as expected
+    test_out_path="/tmp/${test_dir_name}"
+    # Let Dyalog create the files to avoid its stdout (⎕←) length limit
+    bin/run.sh "$test_dir_name" "${test_dir_path}" "${test_out_path}"
 
     file="results.json"
     expected_file="expected_${file}"
+
     echo "${test_dir_name}: comparing ${file} to ${expected_file}"
 
-    if ! diff "${test_dir_path}/${file}" "${test_dir_path}/${expected_file}"; then
+    if ! diff "${test_out_path}/${file}" "${test_dir_path}/${expected_file}"; then
         exit_code=1
     fi
 done

bin/run.sh

@@ -29,32 +29,6 @@ results_file="${output_dir}/results.json"
 # Create the output directory if it doesn't exist
 mkdir -p "${output_dir}"
 
-echo "${slug}: testing..."
-
-# Run the tests for the provided implementation file and redirect stdout and
-# stderr to capture it
-test_output=$(false)
-# TODO: substitute "false" with the actual command to run the test:
-# test_output=$(command_to_run_tests 2>&1)
-
-# Write the results.json file based on the exit code of the command that was 
-# just executed that tested the implementation file
-if [ $? -eq 0 ]; then
-    jq -n '{version: 1, status: "pass"}' > ${results_file}
-else
-    # OPTIONAL: Sanitize the output
-    # In some cases, the test output might be overly verbose, in which case stripping
-    # the unneeded information can be very helpful to the student
-    # sanitized_test_output=$(printf "${test_output}" | sed -n '/Test results:/,$p')
-
-    # OPTIONAL: Manually add colors to the output to help scanning the output for errors
-    # If the test output does not contain colors to help identify failing (or passing)
-    # tests, it can be helpful to manually add colors to the output
-    # colorized_test_output=$(echo "${test_output}" \
-    #      | GREP_COLOR='01;31' grep --color=always -E -e '^(ERROR:.*|.*failed)$|$' \
-    #      | GREP_COLOR='01;32' grep --color=always -E -e '^.*passed$|$')
-
-    jq -n --arg output "${test_output}" '{version: 1, status: "fail", message: $output}' > ${results_file}
-fi
+echo "Testing ${slug}..."
 
-echo "${slug}: done"
+SLUG="$slug" SOLN="$solution_dir" OUTDIR="$output_dir" OUTFILE="$results_file" ./bin/dyalog-apl-runner.apls