Skip to content

Commit

Permalink
compare decoded json
Browse files Browse the repository at this point in the history
  • Loading branch information
@jiegillet
jiegillet committed Oct 12, 2023
1 parent 0cb160c commit e186f52
Showing 1 changed file with 200 additions and 41 deletions.
241 changes: 200 additions & 41 deletions test/elixir_analyzer_test.exs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,13 @@ defmodule ElixirAnalyzerTest do
path = "./test_data/two_fer/perfect_solution/"
analyzed_exercise = ElixirAnalyzer.analyze_exercise(exercise, path, path, @options)

expected_output = """
{\"comments\":[],\"summary\":\"Submission analyzed. No automated suggestions found.\"}
"""
expected_output =
%{
"comments" => [],
"summary" => "Submission analyzed. No automated suggestions found."
}

assert Submission.to_json(analyzed_exercise) == String.trim(expected_output)
assert Submission.to_json(analyzed_exercise) |> Jason.decode!() == expected_output
end

test "referred solution with comments" do
Expand All @@ -28,9 +30,53 @@ defmodule ElixirAnalyzerTest do
analyzed_exercise = ElixirAnalyzer.analyze_exercise(exercise, path, path, @options)

expected_output =
"{\"comments\":[{\"type\":\"actionable\",\"comment\":\"elixir.two-fer.use_of_function_header\"},{\"type\":\"actionable\",\"comment\":\"elixir.solution.use_specification\"},{\"type\":\"actionable\",\"comment\":\"elixir.solution.raise_fn_clause_error\"},{\"type\":\"actionable\",\"params\":{\"actual\":\"_nameInPascalCase\",\"expected\":\"_name_in_pascal_case\"},\"comment\":\"elixir.solution.variable_name_snake_case\"},{\"type\":\"actionable\",\"params\":{\"actual\":\"someUnusedModuleAttribute\",\"expected\":\"some_unused_module_attribute\"},\"comment\":\"elixir.solution.module_attribute_name_snake_case\"},{\"type\":\"actionable\",\"params\":{\"actual\":\"My_empty_module\",\"expected\":\"MyEmptyModule\"},\"comment\":\"elixir.solution.module_pascal_case\"},{\"type\":\"actionable\",\"params\":{\"warnings\":\"warning: module attribute @someUnusedModuleAttribute was set but never used\\n lib/two_fer.ex:2\\n\\n\"},\"comment\":\"elixir.solution.compiler_warnings\"},{\"type\":\"informative\",\"comment\":\"elixir.solution.use_module_doc\"},{\"type\":\"informative\",\"comment\":\"elixir.solution.indentation\"},{\"type\":\"informative\",\"params\":{\"actual\":\"def public_helper(_)\",\"expected\":\"defp public_helper(_)\"},\"comment\":\"elixir.solution.private_helper_functions\"},{\"type\":\"informative\",\"comment\":\"elixir.general.feedback_request\"}],\"summary\":\"Check the comments for some suggestions. 📣\"}"

assert Submission.to_json(analyzed_exercise) == expected_output
%{
"comments" => [
%{"comment" => "elixir.two-fer.use_of_function_header", type: "actionable"},
%{"comment" => "elixir.solution.use_specification", type: "actionable"},
%{"comment" => "elixir.solution.raise_fn_clause_error", type: "actionable"},
%{
"comment" => "elixir.solution.variable_name_snake_case",
"params" => %{"actual" => "_nameInPascalCase", "expected" => "_name_in_pascal_case"},
type: "actionable"
},
%{
"comment" => "elixir.solution.module_attribute_name_snake_case",
"params" => %{
"actual" => "someUnusedModuleAttribute",
"expected" => "some_unused_module_attribute"
},
type: "actionable"
},
%{
"comment" => "elixir.solution.module_pascal_case",
"params" => %{"actual" => "My_empty_module", "expected" => "MyEmptyModule"},
type: "actionable"
},
%{
"comment" => "elixir.solution.compiler_warnings",
"params" => %{
"warnings" =>
"warning: module attribute @someUnusedModuleAttribute was set but never used\n lib/two_fer.ex:2\n\n"
},
type: "actionable"
},
%{"comment" => "elixir.solution.use_module_doc", type: "informative"},
%{"comment" => "elixir.solution.indentation", type: "informative"},
%{
"comment" => "elixir.solution.private_helper_functions",
"params" => %{
"actual" => "def public_helper(_)",
"expected" => "defp public_helper(_)"
},
type: "informative"
},
%{"comment" => "elixir.general.feedback_request", type: "informative"}
],
"summary" => "Check the comments for some suggestions. 📣"
}

assert Submission.to_json(analyzed_exercise) |> Jason.decode!() == expected_output
end

test "solution with informative comments only" do
Expand All @@ -40,7 +86,10 @@ defmodule ElixirAnalyzerTest do
analyzed_exercise = ElixirAnalyzer.analyze_exercise(exercise, path, path, @options)

expected_output =
"{\"comments\":[{\"type\":\"informative\",\"comment\":\"elixir.solution.use_module_doc\"}],\"summary\":\"Check the comments for some things to learn. 📖\"}"
%{
"comments" => [%{"comment" => "elixir.solution.use_module_doc", type: "informative"}],
"summary" => "Check the comments for some things to learn. 📖"
}

assert Submission.to_json(analyzed_exercise) == expected_output
end
Expand All @@ -52,11 +101,22 @@ defmodule ElixirAnalyzerTest do
assert capture_log(fn ->
analyzed_exercise = ElixirAnalyzer.analyze_exercise(exercise, path, path, @options)

expected_output = """
{\"comments\":[{\"type\":\"essential\",\"params\":{\"error\":\"missing terminator: end (for \\\"do\\\" starting at line 1)\",\"line\":14},\"comment\":\"elixir.general.parsing_error\"}],\"summary\":\"Check the comments for things to fix. 🛠\"}
"""

assert Submission.to_json(analyzed_exercise) == String.trim(expected_output)
expected_output =
%{
"comments" => [
%{
"comment" => "elixir.general.parsing_error",
"params" => %{
"error" => "missing terminator: end (for \"do\" starting at line 1)",
"line" => 14
},
type: "essential"
}
],
"summary" => "Check the comments for things to fix. 🛠"
}

assert Submission.to_json(analyzed_exercise) |> Jason.decode!() == expected_output
end) =~ "Exemploid file could not be parsed."
end

Expand All @@ -67,11 +127,22 @@ defmodule ElixirAnalyzerTest do
assert capture_log(fn ->
analyzed_exercise = ElixirAnalyzer.analyze_exercise(exercise, path, path, @options)

expected_output = """
{\"comments\":[{\"type\":\"essential\",\"params\":{\"file_name\":\"two_fer.ex\",\"path\":\"test_data/two_fer/missing_file_solution/\"},\"comment\":\"elixir.general.file_not_found\"}],\"summary\":\"Check the comments for things to fix. 🛠\"}
"""

assert Submission.to_json(analyzed_exercise) == String.trim(expected_output)
expected_output =
%{
"comments" => [
%{
"comment" => "elixir.general.file_not_found",
"params" => %{
"file_name" => "two_fer.ex",
"path" => "test_data/two_fer/missing_file_solution/"
},
type: "essential"
}
],
"summary" => "Check the comments for things to fix. 🛠"
}

assert Submission.to_json(analyzed_exercise) |> Jason.decode!() == expected_output
end) =~ "Code file not found. Reason: enoent"
end

Expand All @@ -91,9 +162,50 @@ defmodule ElixirAnalyzerTest do
analyzed_exercise = ElixirAnalyzer.analyze_exercise(exercise, path, path, @options)

expected_output =
"{\"comments\":[{\"type\":\"actionable\",\"comment\":\"elixir.solution.raise_fn_clause_error\"},{\"type\":\"actionable\",\"params\":{\"actual\":\"_nameInPascalCase\",\"expected\":\"_name_in_pascal_case\"},\"comment\":\"elixir.solution.variable_name_snake_case\"},{\"type\":\"actionable\",\"params\":{\"actual\":\"someUnusedModuleAttribute\",\"expected\":\"some_unused_module_attribute\"},\"comment\":\"elixir.solution.module_attribute_name_snake_case\"},{\"type\":\"actionable\",\"params\":{\"actual\":\"My_empty_module\",\"expected\":\"MyEmptyModule\"},\"comment\":\"elixir.solution.module_pascal_case\"},{\"type\":\"actionable\",\"params\":{\"warnings\":\"warning: module attribute @someUnusedModuleAttribute was set but never used\\n lib/two_fer.ex:2\\n\\n\"},\"comment\":\"elixir.solution.compiler_warnings\"},{\"type\":\"informative\",\"comment\":\"elixir.solution.indentation\"},{\"type\":\"informative\",\"params\":{\"actual\":\"def public_helper(_)\",\"expected\":\"defp public_helper(_)\"},\"comment\":\"elixir.solution.private_helper_functions\"},{\"type\":\"informative\",\"comment\":\"elixir.general.feedback_request\"}],\"summary\":\"Check the comments for some suggestions. 📣\"}"

assert Submission.to_json(analyzed_exercise) == String.trim(expected_output)
%{
"comments" => [
%{"comment" => "elixir.solution.raise_fn_clause_error", type: "actionable"},
%{
"comment" => "elixir.solution.variable_name_snake_case",
"params" => %{"actual" => "_nameInPascalCase", "expected" => "_name_in_pascal_case"},
type: "actionable"
},
%{
"comment" => "elixir.solution.module_attribute_name_snake_case",
"params" => %{
"actual" => "someUnusedModuleAttribute",
"expected" => "some_unused_module_attribute"
},
type: "actionable"
},
%{
"comment" => "elixir.solution.module_pascal_case",
"params" => %{"actual" => "My_empty_module", "expected" => "MyEmptyModule"},
type: "actionable"
},
%{
"comment" => "elixir.solution.compiler_warnings",
"params" => %{
"warnings" =>
"warning: module attribute @someUnusedModuleAttribute was set but never used\n lib/two_fer.ex:2\n\n"
},
type: "actionable"
},
%{"comment" => "elixir.solution.indentation", type: "informative"},
%{
"comment" => "elixir.solution.private_helper_functions",
"params" => %{
"actual" => "def public_helper(_)",
"expected" => "defp public_helper(_)"
},
type: "informative"
},
%{"comment" => "elixir.general.feedback_request", type: "informative"}
],
"summary" => "Check the comments for some suggestions. 📣"
}

assert Submission.to_json(analyzed_exercise) |> Jason.decode!() == expected_output
end
end

Expand All @@ -114,9 +226,12 @@ defmodule ElixirAnalyzerTest do
analyzed_exercise = ElixirAnalyzer.analyze_exercise(exercise, path, path, @options)

expected_output =
"{\"comments\":[{\"type\":\"celebratory\",\"comment\":\"elixir.solution.same_as_exemplar\"}],\"summary\":\"You're doing something right. 🎉\"}"
%{
"comments" => [%{"comment" => "elixir.solution.same_as_exemplar", type: "celebratory"}],
"summary" => "You're doing something right. 🎉"
}

assert Submission.to_json(analyzed_exercise) == String.trim(expected_output)
assert Submission.to_json(analyzed_exercise) |> Jason.decode!() == expected_output
end

test "perfect solution for exercise with multiple solution files" do
Expand All @@ -125,9 +240,12 @@ defmodule ElixirAnalyzerTest do
analyzed_exercise = ElixirAnalyzer.analyze_exercise(exercise, path, path, @options)

expected_output =
"{\"comments\":[],\"summary\":\"Submission analyzed. No automated suggestions found.\"}"
%{
"comments" => [],
"summary" => "Submission analyzed. No automated suggestions found."
}

assert Submission.to_json(analyzed_exercise) == String.trim(expected_output)
assert Submission.to_json(analyzed_exercise) |> Jason.decode!() == expected_output
end

test "failing solution with comments" do
Expand All @@ -136,9 +254,24 @@ defmodule ElixirAnalyzerTest do
analyzed_exercise = ElixirAnalyzer.analyze_exercise(exercise, path, path, @options)

expected_output =
"{\"comments\":[{\"type\":\"actionable\",\"comment\":\"elixir.lasagna.function_reuse\"},{\"type\":\"informative\",\"params\":{\"actual\":\"def public_helper(_)\",\"expected\":\"defp public_helper(_)\"},\"comment\":\"elixir.solution.private_helper_functions\"},{\"type\":\"informative\",\"comment\":\"elixir.solution.todo_comment\"},{\"type\":\"informative\",\"comment\":\"elixir.general.feedback_request\"}],\"summary\":\"Check the comments for some suggestions. 📣\"}"

assert Submission.to_json(analyzed_exercise) == expected_output
%{
"comments" => [
%{"comment" => "elixir.lasagna.function_reuse", type: "actionable"},
%{
"comment" => "elixir.solution.private_helper_functions",
"params" => %{
"actual" => "def public_helper(_)",
"expected" => "defp public_helper(_)"
},
type: "informative"
},
%{"comment" => "elixir.solution.todo_comment", type: "informative"},
%{"comment" => "elixir.general.feedback_request", type: "informative"}
],
"summary" => "Check the comments for some suggestions. 📣"
}

assert Submission.to_json(analyzed_exercise) |> Jason.decode!() == expected_output
end

test "failing solution that uses deprecated modules" do
Expand All @@ -147,9 +280,22 @@ defmodule ElixirAnalyzerTest do
analyzed_exercise = ElixirAnalyzer.analyze_exercise(exercise, path, path, @options)

expected_output =
"{\"comments\":[{\"type\":\"actionable\",\"params\":{\"warnings\":\"warning: HashDict.new/0 is deprecated. Use maps and the Map module instead\\n lib/lasagna.ex:7\\n\\nwarning: HashSet.member?/2 is deprecated. Use the MapSet module instead\\n lib/lasagna.ex:12\\n\\nwarning: HashSet.new/0 is deprecated. Use the MapSet module instead\\n lib/lasagna.ex:12\\n\\nwarning: Behaviour.defcallback/1 is deprecated. Use the @callback module attribute instead\\n lib/lasagna.ex:4\\n\\n\"},\"comment\":\"elixir.solution.compiler_warnings\"},{\"type\":\"informative\",\"comment\":\"elixir.general.feedback_request\"}],\"summary\":\"Check the comments for some suggestions. 📣\"}"

assert Submission.to_json(analyzed_exercise) == expected_output
%{
"comments" => [
%{
"comment" => "elixir.solution.compiler_warnings",
"params" => %{
"warnings" =>
"warning: HashDict.new/0 is deprecated. Use maps and the Map module instead\n lib/lasagna.ex:7\n\nwarning: HashSet.member?/2 is deprecated. Use the MapSet module instead\n lib/lasagna.ex:12\n\nwarning: HashSet.new/0 is deprecated. Use the MapSet module instead\n lib/lasagna.ex:12\n\nwarning: Behaviour.defcallback/1 is deprecated. Use the @callback module attribute instead\n lib/lasagna.ex:4\n\n"
},
type: "actionable"
},
%{"comment" => "elixir.general.feedback_request", type: "informative"}
],
"summary" => "Check the comments for some suggestions. 📣"
}

assert Submission.to_json(analyzed_exercise) |> Jason.decode!() == expected_output
end

test "solution with missing exemplar" do
Expand All @@ -160,9 +306,12 @@ defmodule ElixirAnalyzerTest do
analyzed_exercise = ElixirAnalyzer.analyze_exercise(exercise, path, path, @options)

expected_output =
"{\"comments\":[],\"summary\":\"Submission analyzed. No automated suggestions found.\"}"
%{
"comments" => [],
"summary" => "Submission analyzed. No automated suggestions found."
}

assert Submission.to_json(analyzed_exercise) == String.trim(expected_output)
assert Submission.to_json(analyzed_exercise) |> Jason.decode!() == expected_output
end) =~ "Exemploid file not found. Reason: enoent"
end

Expand All @@ -174,9 +323,12 @@ defmodule ElixirAnalyzerTest do
analyzed_exercise = ElixirAnalyzer.analyze_exercise(exercise, path, path, @options)

expected_output =
"{\"comments\":[],\"summary\":\"Submission analyzed. No automated suggestions found.\"}"
%{
"comments" => [],
"summary" => "Submission analyzed. No automated suggestions found."
}

assert Submission.to_json(analyzed_exercise) == String.trim(expected_output)
assert Submission.to_json(analyzed_exercise) |> Jason.decode!() == expected_output
end) =~ "Exemploid file could not be parsed."
end
end
Expand All @@ -196,11 +348,14 @@ defmodule ElixirAnalyzerTest do
Output written to ... a/b
"""

assert Submission.to_json(submission) ==
"{\"comments\":[],\"summary\":\"Submission analyzed. No automated suggestions found.\"}"
assert Submission.to_json(submission) |> Jason.decode!() ==
%{
"comments" => [],
"summary" => "Submission analyzed. No automated suggestions found."
}

assert Submission.to_json(%{submission | halted: true}) ==
"{\"comments\":[],\"summary\":\"Analysis was halted.\"}"
assert Submission.to_json(%{submission | halted: true}) |> Jason.decode!() ==
%{"comments" => [], "summary" => "Analysis was halted."}
end

test "solution with wrong analysis module" do
Expand Down Expand Up @@ -231,8 +386,12 @@ defmodule ElixirAnalyzerTest do
Output written to ... a/b
"""

assert Submission.to_json(analyzed_exercise) ==
"{\"comments\":[],\"summary\":\"Analysis was halted. Analysis skipped, unexpected error Elixir.ArgumentError\"}"
assert Submission.to_json(analyzed_exercise) |> Jason.decode!() ==
%{
"comments" => [],
"summary" =>
"Analysis was halted. Analysis skipped, unexpected error Elixir.ArgumentError"
}
end) =~ "[error] Loading exercise test suite 'Elixir.NonSense' failed"
end

Expand Down

0 comments on commit e186f52

Please sign in to comment.