Skip to content

Commit

Permalink
Test fix: do not depend on map order (#399)
Browse files Browse the repository at this point in the history 
* update submodule

* compare decoded json

* woops

* order compiler warnings
  • Loading branch information
@jiegillet
jiegillet authored Oct 13, 2023
1 parent 7bb938a commit 78cb44a
Show file tree
Hide file tree
Showing 4 changed files with 210 additions and 47 deletions.
2 changes: 1 addition & 1 deletion elixir
Submodule elixir updated 2 files
+1 −1 exercises/concept/language-list/.meta/config.json
+4 −2 exercises/practice/phone-number/.docs/instructions.md
2 changes: 1 addition & 1 deletion lib/elixir_analyzer/exercise_test/common_checks/compiler_warnings.ex
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ defmodule ElixirAnalyzer.ExerciseTest.CommonChecks.CompilerWarnings do
type: :actionable,
name: Constants.solution_compiler_warnings(),
comment: Constants.solution_compiler_warnings(),
params: %{warnings: Enum.map_join(warnings, &format_warning/1)}
params: %{warnings: warnings |> Enum.sort() |> Enum.map_join(&format_warning/1)}
}}
]
end
Expand Down
251 changes: 207 additions & 44 deletions test/elixir_analyzer_test.exs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,13 @@ defmodule ElixirAnalyzerTest do
path = "./test_data/two_fer/perfect_solution/"
analyzed_exercise = ElixirAnalyzer.analyze_exercise(exercise, path, path, @options)

expected_output = """
{\"comments\":[],\"summary\":\"Submission analyzed. No automated suggestions found.\"}
"""
expected_output =
%{
"comments" => [],
"summary" => "Submission analyzed. No automated suggestions found."
}

assert Submission.to_json(analyzed_exercise) == String.trim(expected_output)
assert Submission.to_json(analyzed_exercise) |> Jason.decode!() == expected_output
end

test "referred solution with comments" do
Expand All @@ -28,9 +30,53 @@ defmodule ElixirAnalyzerTest do
analyzed_exercise = ElixirAnalyzer.analyze_exercise(exercise, path, path, @options)

expected_output =
"{\"comments\":[{\"type\":\"actionable\",\"comment\":\"elixir.two-fer.use_of_function_header\"},{\"type\":\"actionable\",\"comment\":\"elixir.solution.use_specification\"},{\"type\":\"actionable\",\"comment\":\"elixir.solution.raise_fn_clause_error\"},{\"type\":\"actionable\",\"params\":{\"actual\":\"_nameInPascalCase\",\"expected\":\"_name_in_pascal_case\"},\"comment\":\"elixir.solution.variable_name_snake_case\"},{\"type\":\"actionable\",\"params\":{\"actual\":\"someUnusedModuleAttribute\",\"expected\":\"some_unused_module_attribute\"},\"comment\":\"elixir.solution.module_attribute_name_snake_case\"},{\"type\":\"actionable\",\"params\":{\"actual\":\"My_empty_module\",\"expected\":\"MyEmptyModule\"},\"comment\":\"elixir.solution.module_pascal_case\"},{\"type\":\"actionable\",\"params\":{\"warnings\":\"warning: module attribute @someUnusedModuleAttribute was set but never used\\n lib/two_fer.ex:2\\n\\n\"},\"comment\":\"elixir.solution.compiler_warnings\"},{\"type\":\"informative\",\"comment\":\"elixir.solution.use_module_doc\"},{\"type\":\"informative\",\"comment\":\"elixir.solution.indentation\"},{\"type\":\"informative\",\"params\":{\"actual\":\"def public_helper(_)\",\"expected\":\"defp public_helper(_)\"},\"comment\":\"elixir.solution.private_helper_functions\"},{\"type\":\"informative\",\"comment\":\"elixir.general.feedback_request\"}],\"summary\":\"Check the comments for some suggestions. 📣\"}"

assert Submission.to_json(analyzed_exercise) == expected_output
%{
"comments" => [
%{"comment" => "elixir.two-fer.use_of_function_header", "type" => "actionable"},
%{"comment" => "elixir.solution.use_specification", "type" => "actionable"},
%{"comment" => "elixir.solution.raise_fn_clause_error", "type" => "actionable"},
%{
"comment" => "elixir.solution.variable_name_snake_case",
"params" => %{"actual" => "_nameInPascalCase", "expected" => "_name_in_pascal_case"},
"type" => "actionable"
},
%{
"comment" => "elixir.solution.module_attribute_name_snake_case",
"params" => %{
"actual" => "someUnusedModuleAttribute",
"expected" => "some_unused_module_attribute"
},
"type" => "actionable"
},
%{
"comment" => "elixir.solution.module_pascal_case",
"params" => %{"actual" => "My_empty_module", "expected" => "MyEmptyModule"},
"type" => "actionable"
},
%{
"comment" => "elixir.solution.compiler_warnings",
"params" => %{
"warnings" =>
"warning: module attribute @someUnusedModuleAttribute was set but never used\n lib/two_fer.ex:2\n\n"
},
"type" => "actionable"
},
%{"comment" => "elixir.solution.use_module_doc", "type" => "informative"},
%{"comment" => "elixir.solution.indentation", "type" => "informative"},
%{
"comment" => "elixir.solution.private_helper_functions",
"params" => %{
"actual" => "def public_helper(_)",
"expected" => "defp public_helper(_)"
},
"type" => "informative"
},
%{"comment" => "elixir.general.feedback_request", "type" => "informative"}
],
"summary" => "Check the comments for some suggestions. 📣"
}

assert Submission.to_json(analyzed_exercise) |> Jason.decode!() == expected_output
end

test "solution with informative comments only" do
Expand All @@ -40,9 +86,14 @@ defmodule ElixirAnalyzerTest do
analyzed_exercise = ElixirAnalyzer.analyze_exercise(exercise, path, path, @options)

expected_output =
"{\"comments\":[{\"type\":\"informative\",\"comment\":\"elixir.solution.use_module_doc\"}],\"summary\":\"Check the comments for some things to learn. 📖\"}"

assert Submission.to_json(analyzed_exercise) == expected_output
%{
"comments" => [
%{"comment" => "elixir.solution.use_module_doc", "type" => "informative"}
],
"summary" => "Check the comments for some things to learn. 📖"
}

assert Submission.to_json(analyzed_exercise) |> Jason.decode!() == expected_output
end

test "error solution" do
Expand All @@ -52,11 +103,22 @@ defmodule ElixirAnalyzerTest do
assert capture_log(fn ->
analyzed_exercise = ElixirAnalyzer.analyze_exercise(exercise, path, path, @options)

expected_output = """
{\"comments\":[{\"type\":\"essential\",\"params\":{\"error\":\"missing terminator: end (for \\\"do\\\" starting at line 1)\",\"line\":14},\"comment\":\"elixir.general.parsing_error\"}],\"summary\":\"Check the comments for things to fix. 🛠\"}
"""

assert Submission.to_json(analyzed_exercise) == String.trim(expected_output)
expected_output =
%{
"comments" => [
%{
"comment" => "elixir.general.parsing_error",
"params" => %{
"error" => "missing terminator: end (for \"do\" starting at line 1)",
"line" => 14
},
"type" => "essential"
}
],
"summary" => "Check the comments for things to fix. 🛠"
}

assert Submission.to_json(analyzed_exercise) |> Jason.decode!() == expected_output
end) =~ "Exemploid file could not be parsed."
end

Expand All @@ -67,11 +129,22 @@ defmodule ElixirAnalyzerTest do
assert capture_log(fn ->
analyzed_exercise = ElixirAnalyzer.analyze_exercise(exercise, path, path, @options)

expected_output = """
{\"comments\":[{\"type\":\"essential\",\"params\":{\"file_name\":\"two_fer.ex\",\"path\":\"test_data/two_fer/missing_file_solution/\"},\"comment\":\"elixir.general.file_not_found\"}],\"summary\":\"Check the comments for things to fix. 🛠\"}
"""

assert Submission.to_json(analyzed_exercise) == String.trim(expected_output)
expected_output =
%{
"comments" => [
%{
"comment" => "elixir.general.file_not_found",
"params" => %{
"file_name" => "two_fer.ex",
"path" => "test_data/two_fer/missing_file_solution/"
},
"type" => "essential"
}
],
"summary" => "Check the comments for things to fix. 🛠"
}

assert Submission.to_json(analyzed_exercise) |> Jason.decode!() == expected_output
end) =~ "Code file not found. Reason: enoent"
end

Expand All @@ -91,9 +164,50 @@ defmodule ElixirAnalyzerTest do
analyzed_exercise = ElixirAnalyzer.analyze_exercise(exercise, path, path, @options)

expected_output =
"{\"comments\":[{\"type\":\"actionable\",\"comment\":\"elixir.solution.raise_fn_clause_error\"},{\"type\":\"actionable\",\"params\":{\"actual\":\"_nameInPascalCase\",\"expected\":\"_name_in_pascal_case\"},\"comment\":\"elixir.solution.variable_name_snake_case\"},{\"type\":\"actionable\",\"params\":{\"actual\":\"someUnusedModuleAttribute\",\"expected\":\"some_unused_module_attribute\"},\"comment\":\"elixir.solution.module_attribute_name_snake_case\"},{\"type\":\"actionable\",\"params\":{\"actual\":\"My_empty_module\",\"expected\":\"MyEmptyModule\"},\"comment\":\"elixir.solution.module_pascal_case\"},{\"type\":\"actionable\",\"params\":{\"warnings\":\"warning: module attribute @someUnusedModuleAttribute was set but never used\\n lib/two_fer.ex:2\\n\\n\"},\"comment\":\"elixir.solution.compiler_warnings\"},{\"type\":\"informative\",\"comment\":\"elixir.solution.indentation\"},{\"type\":\"informative\",\"params\":{\"actual\":\"def public_helper(_)\",\"expected\":\"defp public_helper(_)\"},\"comment\":\"elixir.solution.private_helper_functions\"},{\"type\":\"informative\",\"comment\":\"elixir.general.feedback_request\"}],\"summary\":\"Check the comments for some suggestions. 📣\"}"

assert Submission.to_json(analyzed_exercise) == String.trim(expected_output)
%{
"comments" => [
%{"comment" => "elixir.solution.raise_fn_clause_error", "type" => "actionable"},
%{
"comment" => "elixir.solution.variable_name_snake_case",
"params" => %{"actual" => "_nameInPascalCase", "expected" => "_name_in_pascal_case"},
"type" => "actionable"
},
%{
"comment" => "elixir.solution.module_attribute_name_snake_case",
"params" => %{
"actual" => "someUnusedModuleAttribute",
"expected" => "some_unused_module_attribute"
},
"type" => "actionable"
},
%{
"comment" => "elixir.solution.module_pascal_case",
"params" => %{"actual" => "My_empty_module", "expected" => "MyEmptyModule"},
"type" => "actionable"
},
%{
"comment" => "elixir.solution.compiler_warnings",
"params" => %{
"warnings" =>
"warning: module attribute @someUnusedModuleAttribute was set but never used\n lib/two_fer.ex:2\n\n"
},
"type" => "actionable"
},
%{"comment" => "elixir.solution.indentation", "type" => "informative"},
%{
"comment" => "elixir.solution.private_helper_functions",
"params" => %{
"actual" => "def public_helper(_)",
"expected" => "defp public_helper(_)"
},
"type" => "informative"
},
%{"comment" => "elixir.general.feedback_request", "type" => "informative"}
],
"summary" => "Check the comments for some suggestions. 📣"
}

assert Submission.to_json(analyzed_exercise) |> Jason.decode!() == expected_output
end
end

Expand All @@ -114,9 +228,14 @@ defmodule ElixirAnalyzerTest do
analyzed_exercise = ElixirAnalyzer.analyze_exercise(exercise, path, path, @options)

expected_output =
"{\"comments\":[{\"type\":\"celebratory\",\"comment\":\"elixir.solution.same_as_exemplar\"}],\"summary\":\"You're doing something right. 🎉\"}"

assert Submission.to_json(analyzed_exercise) == String.trim(expected_output)
%{
"comments" => [
%{"comment" => "elixir.solution.same_as_exemplar", "type" => "celebratory"}
],
"summary" => "You're doing something right. 🎉"
}

assert Submission.to_json(analyzed_exercise) |> Jason.decode!() == expected_output
end

test "perfect solution for exercise with multiple solution files" do
Expand All @@ -125,9 +244,12 @@ defmodule ElixirAnalyzerTest do
analyzed_exercise = ElixirAnalyzer.analyze_exercise(exercise, path, path, @options)

expected_output =
"{\"comments\":[],\"summary\":\"Submission analyzed. No automated suggestions found.\"}"
%{
"comments" => [],
"summary" => "Submission analyzed. No automated suggestions found."
}

assert Submission.to_json(analyzed_exercise) == String.trim(expected_output)
assert Submission.to_json(analyzed_exercise) |> Jason.decode!() == expected_output
end

test "failing solution with comments" do
Expand All @@ -136,9 +258,24 @@ defmodule ElixirAnalyzerTest do
analyzed_exercise = ElixirAnalyzer.analyze_exercise(exercise, path, path, @options)

expected_output =
"{\"comments\":[{\"type\":\"actionable\",\"comment\":\"elixir.lasagna.function_reuse\"},{\"type\":\"informative\",\"params\":{\"actual\":\"def public_helper(_)\",\"expected\":\"defp public_helper(_)\"},\"comment\":\"elixir.solution.private_helper_functions\"},{\"type\":\"informative\",\"comment\":\"elixir.solution.todo_comment\"},{\"type\":\"informative\",\"comment\":\"elixir.general.feedback_request\"}],\"summary\":\"Check the comments for some suggestions. 📣\"}"

assert Submission.to_json(analyzed_exercise) == expected_output
%{
"comments" => [
%{"comment" => "elixir.lasagna.function_reuse", "type" => "actionable"},
%{
"comment" => "elixir.solution.private_helper_functions",
"params" => %{
"actual" => "def public_helper(_)",
"expected" => "defp public_helper(_)"
},
"type" => "informative"
},
%{"comment" => "elixir.solution.todo_comment", "type" => "informative"},
%{"comment" => "elixir.general.feedback_request", "type" => "informative"}
],
"summary" => "Check the comments for some suggestions. 📣"
}

assert Submission.to_json(analyzed_exercise) |> Jason.decode!() == expected_output
end

test "failing solution that uses deprecated modules" do
Expand All @@ -147,9 +284,22 @@ defmodule ElixirAnalyzerTest do
analyzed_exercise = ElixirAnalyzer.analyze_exercise(exercise, path, path, @options)

expected_output =
"{\"comments\":[{\"type\":\"actionable\",\"params\":{\"warnings\":\"warning: HashDict.new/0 is deprecated. Use maps and the Map module instead\\n lib/lasagna.ex:7\\n\\nwarning: HashSet.member?/2 is deprecated. Use the MapSet module instead\\n lib/lasagna.ex:12\\n\\nwarning: HashSet.new/0 is deprecated. Use the MapSet module instead\\n lib/lasagna.ex:12\\n\\nwarning: Behaviour.defcallback/1 is deprecated. Use the @callback module attribute instead\\n lib/lasagna.ex:4\\n\\n\"},\"comment\":\"elixir.solution.compiler_warnings\"},{\"type\":\"informative\",\"comment\":\"elixir.general.feedback_request\"}],\"summary\":\"Check the comments for some suggestions. 📣\"}"

assert Submission.to_json(analyzed_exercise) == expected_output
%{
"comments" => [
%{
"comment" => "elixir.solution.compiler_warnings",
"params" => %{
"warnings" =>
"warning: Behaviour.defcallback/1 is deprecated. Use the @callback module attribute instead\n lib/lasagna.ex:4\n\nwarning: HashDict.new/0 is deprecated. Use maps and the Map module instead\n lib/lasagna.ex:7\n\nwarning: HashSet.member?/2 is deprecated. Use the MapSet module instead\n lib/lasagna.ex:12\n\nwarning: HashSet.new/0 is deprecated. Use the MapSet module instead\n lib/lasagna.ex:12\n\n"
},
"type" => "actionable"
},
%{"comment" => "elixir.general.feedback_request", "type" => "informative"}
],
"summary" => "Check the comments for some suggestions. 📣"
}

assert Submission.to_json(analyzed_exercise) |> Jason.decode!() == expected_output
end

test "solution with missing exemplar" do
Expand All @@ -160,9 +310,12 @@ defmodule ElixirAnalyzerTest do
analyzed_exercise = ElixirAnalyzer.analyze_exercise(exercise, path, path, @options)

expected_output =
"{\"comments\":[],\"summary\":\"Submission analyzed. No automated suggestions found.\"}"
%{
"comments" => [],
"summary" => "Submission analyzed. No automated suggestions found."
}

assert Submission.to_json(analyzed_exercise) == String.trim(expected_output)
assert Submission.to_json(analyzed_exercise) |> Jason.decode!() == expected_output
end) =~ "Exemploid file not found. Reason: enoent"
end

Expand All @@ -174,9 +327,12 @@ defmodule ElixirAnalyzerTest do
analyzed_exercise = ElixirAnalyzer.analyze_exercise(exercise, path, path, @options)

expected_output =
"{\"comments\":[],\"summary\":\"Submission analyzed. No automated suggestions found.\"}"
%{
"comments" => [],
"summary" => "Submission analyzed. No automated suggestions found."
}

assert Submission.to_json(analyzed_exercise) == String.trim(expected_output)
assert Submission.to_json(analyzed_exercise) |> Jason.decode!() == expected_output
end) =~ "Exemploid file could not be parsed."
end
end
Expand All @@ -196,11 +352,14 @@ defmodule ElixirAnalyzerTest do
Output written to ... a/b
"""

assert Submission.to_json(submission) ==
"{\"comments\":[],\"summary\":\"Submission analyzed. No automated suggestions found.\"}"
assert Submission.to_json(submission) |> Jason.decode!() ==
%{
"comments" => [],
"summary" => "Submission analyzed. No automated suggestions found."
}

assert Submission.to_json(%{submission | halted: true}) ==
"{\"comments\":[],\"summary\":\"Analysis was halted.\"}"
assert Submission.to_json(%{submission | halted: true}) |> Jason.decode!() ==
%{"comments" => [], "summary" => "Analysis was halted."}
end

test "solution with wrong analysis module" do
Expand Down Expand Up @@ -231,8 +390,12 @@ defmodule ElixirAnalyzerTest do
Output written to ... a/b
"""

assert Submission.to_json(analyzed_exercise) ==
"{\"comments\":[],\"summary\":\"Analysis was halted. Analysis skipped, unexpected error Elixir.ArgumentError\"}"
assert Submission.to_json(analyzed_exercise) |> Jason.decode!() ==
%{
"comments" => [],
"summary" =>
"Analysis was halted. Analysis skipped, unexpected error Elixir.ArgumentError"
}
end) =~ "[error] Loading exercise test suite 'Elixir.NonSense' failed"
end

Expand Down
Loading

0 comments on commit 78cb44a

Please sign in to comment.