diff --git a/lib/mix/tasks/compiler.ex b/lib/mix/tasks/compiler.ex
index 4be5725..bd62d70 100644
--- a/lib/mix/tasks/compiler.ex
+++ b/lib/mix/tasks/compiler.ex
@@ -10,30 +10,56 @@ defmodule Mix.Tasks.Compile.Speck do
 
   @impl Mix.Task.Compiler
   def run(_args) do
-    files    = Path.wildcard("#{@schema_path}/**/*.ex")
-    hashes   = hashes(files)
-    manifest = load_manifest()
+    schema_files = Path.wildcard("#{@schema_path}/**/*.ex")
+    manifest     = load_manifest()
+    hashes       = hashes(schema_files)
 
     modified_files =
-      hashes
-      |> Enum.reject(&Enum.member?(manifest, &1))
-      |> Enum.map(fn {_hash, file} -> file end)
+      hashes -- Enum.map(manifest, fn {hash, schema_file, _module} -> {hash, schema_file} end)
 
-    Enum.each(modified_files, fn file ->
-      Mix.Shell.IO.info "Compiling #{file}"
+    compiled_files =
+      Enum.map(modified_files, fn {_hash, schema_file} ->
+        Mix.Shell.IO.info "Compiling #{schema_file}"
 
-      compile(file)
-      |> Enum.each(fn {module, bytecode} ->
-        path = Path.join([Mix.Project.app_path, "ebin", "#{module}.beam"])
+        schema_file
+        |> compile()
+        |> Enum.map(fn {module, bytecode} ->
+          module_path = module_path(module)
 
-        File.mkdir_p(Path.dirname(path))
-        File.write(path, bytecode)
+          File.mkdir_p(Path.dirname(module_path))
+          File.write!(module_path, bytecode)
+
+          new_hash = hash(schema_file)
+
+          {new_hash, schema_file, module}
+        end)
       end)
-    end)
+      |> List.flatten
+
+    removed_files =
+      Enum.map(manifest, fn {_hash, schema_file, _module} -> schema_file end) -- schema_files
+
+    new_manifest =
+      manifest
+      |> Enum.reject(fn {_hash, schema_file, _module} ->
+        Enum.member?(removed_files, schema_file)
+      end)
+      |> Enum.reject(fn {_hash, schema_file, _module} ->
+        Enum.find(modified_files, fn {_h, sf} -> schema_file == sf end)
+      end)
+      |> Enum.concat(compiled_files)
 
-    files
-    |> hashes()
-    |> save_manifest()    
+    save_manifest(new_manifest)
+
+    files_to_clean =
+      (manifest -- new_manifest)
+      |> Enum.reject(fn {_hash, _schema_file, module} ->
+        Enum.find(compiled_files, fn {_h, _sf, m} -> module == m end)
+      end)
+
+    Enum.each(files_to_clean, fn {_hash, _schema_file, module} ->
+      File.rm(module_path(module))
+    end)
 
     :ok
   end
@@ -48,6 +74,10 @@ defmodule Mix.Tasks.Compile.Speck do
     File.rm_rf(@manifest_path)
   end
 
+  defp module_path(module) do
+    Path.join([Mix.Project.app_path, "ebin", "#{module}.beam"])
+  end
+
   defp compile(file) do
     file_ast =
       file
@@ -159,8 +189,8 @@ defmodule Mix.Tasks.Compile.Speck do
     end
   end
 
-  defp save_manifest(hashes) do
-    File.write!(@manifest_path, :erlang.term_to_binary(hashes))
+  defp save_manifest(manifest) do
+    File.write!(@manifest_path, :erlang.term_to_binary(manifest))
   end
 
   defp coerce(:date, value) when is_binary(value) do