Add initial version (#7)

Author: ErikSchierboom

.gitignore

@@ -1 +1,5 @@
-/tests/**/*/results.json
+/tests/**/results.json
+/tests/**/test
+/tests/**/test.c
+/tests/**/*.actual
+/tests/**/*.expected

Dockerfile

@@ -1,7 +1,18 @@
-FROM alpine:3.18
+FROM alpine:3.20 AS builder
 
-# install packages required to run the tests
-RUN apk add --no-cache jq coreutils
+RUN apk add --no-cache curl
+
+ARG VERSION=0.25.31
+
+RUN curl -O "https://futhark-lang.org/releases/futhark-${VERSION}-linux-x86_64.tar.xz" && \
+    tar -xJf "futhark-${VERSION}-linux-x86_64.tar.xz" && \
+    cp "futhark-${VERSION}-linux-x86_64/bin/futhark" /usr/local/bin/
+
+FROM alpine:3.20 AS runtime
+
+RUN apk add --no-cache jq gcc musl-dev
+
+COPY --from=builder /usr/local/bin/futhark /usr/local/bin/futhark
 
 WORKDIR /opt/test-runner
 COPY . .

bin/run.sh

@@ -33,27 +33,13 @@ echo "${slug}: testing..."
 
 # Run the tests for the provided implementation file and redirect stdout and
 # stderr to capture it
-test_output=$(false)
-# TODO: substitute "false" with the actual command to run the test:
-# test_output=$(command_to_run_tests 2>&1)
+test_output=$(futhark test "${solution_dir}/test.fut" 2>&1)
 
 # Write the results.json file based on the exit code of the command that was 
 # just executed that tested the implementation file
 if [ $? -eq 0 ]; then
     jq -n '{version: 1, status: "pass"}' > ${results_file}
 else
-    # OPTIONAL: Sanitize the output
-    # In some cases, the test output might be overly verbose, in which case stripping
-    # the unneeded information can be very helpful to the student
-    # sanitized_test_output=$(printf "${test_output}" | sed -n '/Test results:/,$p')
-
-    # OPTIONAL: Manually add colors to the output to help scanning the output for errors
-    # If the test output does not contain colors to help identify failing (or passing)
-    # tests, it can be helpful to manually add colors to the output
-    # colorized_test_output=$(echo "${test_output}" \
-    #      | GREP_COLOR='01;31' grep --color=always -E -e '^(ERROR:.*|.*failed)$|$' \
-    #      | GREP_COLOR='01;32' grep --color=always -E -e '^.*passed$|$')
-
     jq -n --arg output "${test_output}" '{version: 1, status: "fail", message: $output}' > ${results_file}
 fi
 

tests/all-fail/all_fail.fut

@@ -0,0 +1,2 @@
+def is_leap (year: i32): bool =
+  !(year % 4 == 0 && (year % 100 != 0 || year % 400 == 0))

tests/all-fail/expected_results.json

@@ -1,5 +1,5 @@
 {
   "version": 1,
   "status": "fail",
-  "message": "TODO: replace with correct output"
+  "message": "0 failed, 0 passed, 1 to go.\n/opt/test-runner/tests/all-fail/test.fut:\nCompiling with --backend=c:\nRunning compiled program:\nRunning /opt/test-runner/tests/all-fail/test:\nEntry point: main; dataset: #0 (\"2015i32\"):\n/opt/test-runner/tests/all-fail/test.fut.main.0.actual and /opt/test-runner/tests/all-fail/test.fut.main.0.expected do not match:\nValue #0: expected False, got True\nEntry point: main; dataset: #0 (\"1960i32\"):\n/opt/test-runner/tests/all-fail/test.fut.main.0.actual and /opt/test-runner/tests/all-fail/test.fut.main.0.expected do not match:\nValue #0: expected True, got False\nEntry point: main; dataset: #0 (\"2100i32\"):\n/opt/test-runner/tests/all-fail/test.fut.main.0.actual and /opt/test-runner/tests/all-fail/test.fut.main.0.expected do not match:\nValue #0: expected False, got True\n0/1 passed."
 }

tests/all-fail/test.fut

@@ -0,0 +1,19 @@
+import "all_fail"
+
+-- Year not divisible by 4 in common year
+-- ==
+-- input { 2015 }
+-- output { false }
+
+-- Year divisible by 4 and 5 is still a leap year
+-- ==
+-- input { 1960 }
+-- output { true }
+
+-- Year divisible by 100, not divisible by 400 in common year
+-- ==
+-- input { 2100 }
+-- output { false }
+
+let main (year: i32): bool =
+  is_leap year

tests/empty-file/empty_file.fut

@@ -1,5 +1,5 @@
 {
   "version": 1,
   "status": "fail",
-  "message": "TODO: replace with correct output"
+  "message": "0 failed, 0 passed, 1 to go.\n/opt/test-runner/tests/empty-file/test.fut:\nCompiling with --backend=c:\nError at /opt/test-runner/tests/empty-file/test.fut:19:3-10:\nUnknown name \"is_leap\"\nIf you find this error message confusing, uninformative, or wrong, please open an issue:\n  https://github.com/diku-dk/futhark/issues\n\n0/1 passed."
 }

tests/empty-file/expected_results.json

@@ -0,0 +1,19 @@
+import "empty_file"
+
+-- Year not divisible by 4 in common year
+-- ==
+-- input { 2015 }
+-- output { false }
+
+-- Year divisible by 4 and 5 is still a leap year
+-- ==
+-- input { 1960 }
+-- output { true }
+
+-- Year divisible by 100, not divisible by 400 in common year
+-- ==
+-- input { 2100 }
+-- output { false }
+
+let main (year: i32): bool =
+  is_leap year

tests/empty-file/test.fut

@@ -1,5 +1,5 @@
 {
   "version": 1,
   "status": "fail",
-  "message": "TODO: replace with correct output"
+  "message": "0 failed, 0 passed, 1 to go.\n/opt/test-runner/tests/partial-fail/test.fut:\nCompiling with --backend=c:\nRunning compiled program:\nRunning /opt/test-runner/tests/partial-fail/test:\nEntry point: main; dataset: #0 (\"2100i32\"):\n/opt/test-runner/tests/partial-fail/test.fut.main.0.actual and /opt/test-runner/tests/partial-fail/test.fut.main.0.expected do not match:\nValue #0: expected False, got True\n0/1 passed."
 }