Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

update(formatting): Use elixir formatting for arc #275

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .formatter.exs
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
[
import_deps: [],
inputs: ["*.{ex,exs}", "{config,lib,test}/**/*.{ex,exs}"],
subdirectories: []
]
9 changes: 5 additions & 4 deletions lib/arc/actions/delete.ex
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ defmodule Arc.Actions.Delete do
defmacro __using__(_) do
quote do
def delete(args), do: Arc.Actions.Delete.delete(__MODULE__, args)

defoverridable [{:delete, 1}]
end
end
Expand All @@ -26,12 +26,13 @@ defmodule Arc.Actions.Delete do
defp do_delete(definition, {file, scope}) do
if definition.async do
definition.__versions
|> Enum.map(fn(r) -> async_delete_version(definition, r, {file, scope}) end)
|> Enum.each(fn(task) -> Task.await(task, version_timeout()) end)
|> Enum.map(fn r -> async_delete_version(definition, r, {file, scope}) end)
|> Enum.each(fn task -> Task.await(task, version_timeout()) end)
else
definition.__versions
|> Enum.map(fn(version) -> delete_version(definition, version, {file, scope}) end)
|> Enum.map(fn version -> delete_version(definition, version, {file, scope}) end)
end

:ok
end

Expand Down
40 changes: 25 additions & 15 deletions lib/arc/actions/store.ex
Original file line number Diff line number Diff line change
Expand Up @@ -17,39 +17,43 @@ defmodule Arc.Actions.Store do
# Private
#

defp put(_definition, { error = {:error, _msg}, _scope}), do: error
defp put(definition, {%Arc.File{}=file, scope}) do
defp put(_definition, {error = {:error, _msg}, _scope}), do: error

defp put(definition, {%Arc.File{} = file, scope}) do
case definition.validate({file, scope}) do
true -> put_versions(definition, {file, scope})
_ -> {:error, :invalid_file}
_ -> {:error, :invalid_file}
end
end

defp put_versions(definition, {file, scope}) do
if definition.async do
definition.__versions
|> Enum.map(fn(r) -> async_process_version(definition, r, {file, scope}) end)
|> Enum.map(fn(task) -> Task.await(task, version_timeout()) end)
|> Enum.map(fn r -> async_process_version(definition, r, {file, scope}) end)
|> Enum.map(fn task -> Task.await(task, version_timeout()) end)
|> ensure_all_success
|> Enum.map(fn({v, r}) -> async_put_version(definition, v, {r, scope}) end)
|> Enum.map(fn(task) -> Task.await(task, version_timeout()) end)
|> Enum.map(fn {v, r} -> async_put_version(definition, v, {r, scope}) end)
|> Enum.map(fn task -> Task.await(task, version_timeout()) end)
|> handle_responses(file.file_name)
else
definition.__versions
|> Enum.map(fn(version) -> process_version(definition, version, {file, scope}) end)
|> Enum.map(fn version -> process_version(definition, version, {file, scope}) end)
|> ensure_all_success
|> Enum.map(fn({version, result}) -> put_version(definition, version, {result, scope}) end)
|> Enum.map(fn {version, result} -> put_version(definition, version, {result, scope}) end)
|> handle_responses(file.file_name)
end
end

defp ensure_all_success(responses) do
errors = Enum.filter(responses, fn({_version, resp}) -> elem(resp, 0) == :error end)
errors = Enum.filter(responses, fn {_version, resp} -> elem(resp, 0) == :error end)
if Enum.empty?(errors), do: responses, else: errors
end

defp handle_responses(responses, filename) do
errors = Enum.filter(responses, fn(resp) -> elem(resp, 0) == :error end) |> Enum.map(fn(err) -> elem(err, 1) end)
errors =
Enum.filter(responses, fn resp -> elem(resp, 0) == :error end)
|> Enum.map(fn err -> elem(err, 1) end)

if Enum.empty?(errors), do: {:ok, filename}, else: {:error, errors}
end

Expand All @@ -75,11 +79,17 @@ defmodule Arc.Actions.Store do

defp put_version(definition, version, {result, scope}) do
case result do
{:error, error} -> {:error, error}
{:ok, nil} -> {:ok, nil}
{:error, error} ->
{:error, error}

{:ok, nil} ->
{:ok, nil}

{:ok, file} ->
file_name = Arc.Definition.Versioning.resolve_file_name(definition, version, {file, scope})
file = %Arc.File{file | file_name: file_name}
file_name =
Arc.Definition.Versioning.resolve_file_name(definition, version, {file, scope})

file = %Arc.File{file | file_name: file_name}
result = definition.__storage.put(definition, version, {file, scope})
result
end
Expand Down
13 changes: 8 additions & 5 deletions lib/arc/actions/url.ex
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,9 @@ defmodule Arc.Actions.Url do
defmacro __using__(_) do
quote do
def urls(file, options \\ []) do
Enum.into __MODULE__.__versions, %{}, fn(r) ->
Enum.into(__MODULE__.__versions(), %{}, fn r ->
{r, __MODULE__.url(file, r, options)}
end
end)
end

def url(file), do: url(file, nil)
Expand All @@ -21,8 +21,9 @@ defmodule Arc.Actions.Url do
do: url(definition, file, Enum.at(definition.__versions, 0), options)

# Transform standalone file into a tuple of {file, scope}
def url(definition, file, version, options) when is_binary(file) or is_map(file) or is_nil(file),
do: url(definition, {file, nil}, version, options)
def url(definition, file, version, options)
when is_binary(file) or is_map(file) or is_nil(file),
do: url(definition, {file, nil}, version, options)

# Transform file-path into a map with a file_name key
def url(definition, {file, scope}, version, options) when is_binary(file) do
Expand All @@ -43,7 +44,9 @@ defmodule Arc.Actions.Url do

defp build(definition, version, file_and_scope, options) do
case Arc.Definition.Versioning.resolve_file_name(definition, version, file_and_scope) do
nil -> nil
nil ->
nil

_ ->
definition.__storage.url(definition, version, file_and_scope, options)
end
Expand Down
9 changes: 8 additions & 1 deletion lib/arc/definition/storage.ex
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,14 @@ defmodule Arc.Definition.Storage do
def default_url(_), do: nil
def __storage, do: Application.get_env(:arc, :storage, Arc.Storage.S3)

defoverridable [storage_dir: 2, filename: 2, validate: 1, default_url: 1, default_url: 2, __storage: 0, bucket: 0, asset_host: 0]
defoverridable storage_dir: 2,
filename: 2,
validate: 1,
default_url: 1,
default_url: 2,
__storage: 0,
bucket: 0,
asset_host: 0

@before_compile Arc.Definition.Storage
end
Expand Down
4 changes: 2 additions & 2 deletions lib/arc/definition/versioning.ex
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,9 @@ defmodule Arc.Definition.Versioning do
conversion = definition.transform(version, {file, scope})

case conversion do
:skip -> nil
:skip -> nil
{_, _, ext} -> "#{name}.#{ext}"
_ -> "#{name}#{Path.extname(file.file_name)}"
_ -> "#{name}#{Path.extname(file.file_name)}"
end
end

Expand Down
16 changes: 10 additions & 6 deletions lib/arc/file.ex
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ defmodule Arc.File do
|> Base.encode32()
|> Kernel.<>(extension)

Path.join(System.tmp_dir, file_name)
Path.join(System.tmp_dir(), file_name)
end

# Given a remote file
Expand All @@ -21,7 +21,7 @@ defmodule Arc.File do
{:ok, local_path} -> %Arc.File{path: local_path, file_name: filename}
:error -> {:error, :invalid_file_path}
end
end
end

# Accepts a path
def new(path) when is_binary(path) do
Expand Down Expand Up @@ -89,22 +89,25 @@ end
timeout: Application.get_env(:arc, :timeout, 10_000),
max_retries: Application.get_env(:arc, :max_retries, 3),
backoff_factor: Application.get_env(:arc, :backoff_factor, 1000),
backoff_max: Application.get_env(:arc, :backoff_max, 30_000),
backoff_max: Application.get_env(:arc, :backoff_max, 30_000)
]

request(remote_path, options)
end

defp request(remote_path, options, tries \\ 0) do
case :hackney.get(URI.to_string(remote_path), [], "", options) do
{:ok, 200, _headers, client_ref} -> :hackney.body(client_ref)
{:ok, 200, _headers, client_ref} ->
:hackney.body(client_ref)

{:error, %{reason: :timeout}} ->
case retry(tries, options) do
{:ok, :retry} -> request(remote_path, options, tries + 1)
{:error, :out_of_tries} -> {:error, :timeout}
end

_ -> {:error, :arc_httpoison_error}
_ ->
{:error, :arc_httpoison_error}
end
end

Expand All @@ -116,7 +119,8 @@ end
:timer.sleep(backoff)
{:ok, :retry}

true -> {:error, :out_of_tries}
true ->
{:error, :out_of_tries}
end
end
end
4 changes: 3 additions & 1 deletion lib/arc/processor.ex
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,9 @@ defmodule Arc.Processor do

defp apply_transformation(_, :skip), do: {:ok, nil}
defp apply_transformation(file, :noaction), do: {:ok, file}
defp apply_transformation(file, {:noaction}), do: {:ok, file} # Deprecated
# Deprecated
defp apply_transformation(file, {:noaction}), do: {:ok, file}

defp apply_transformation(file, {cmd, conversion, _}) do
apply_transformation(file, {cmd, conversion})
end
Expand Down
11 changes: 6 additions & 5 deletions lib/arc/storage/local.ex
Original file line number Diff line number Diff line change
Expand Up @@ -16,11 +16,12 @@ defmodule Arc.Storage.Local do
def url(definition, version, file_and_scope, _options \\ []) do
local_path = build_local_path(definition, version, file_and_scope)

url = if String.starts_with?(local_path, "/") do
local_path
else
"/" <> local_path
end
url =
if String.starts_with?(local_path, "/") do
local_path
else
"/" <> local_path
end

url |> URI.encode()
end
Expand Down
22 changes: 11 additions & 11 deletions lib/arc/storage/s3.ex
Original file line number Diff line number Diff line change
@@ -1,12 +1,11 @@
defmodule Arc.Storage.S3 do
require Logger
@default_expiry_time 60*5
@default_expiry_time 60 * 5

def put(definition, version, {file, scope}) do
destination_dir = definition.storage_dir(version, {file, scope})
s3_bucket = s3_bucket(definition)
s3_key = Path.join(destination_dir, file.file_name)
asset_host = asset_host(definition)
acl = definition.acl(version, {file, scope})

s3_options =
Expand All @@ -20,7 +19,7 @@ defmodule Arc.Storage.S3 do
def url(definition, version, file_and_scope, options \\ []) do
case Keyword.get(options, :signed, false) do
false -> build_url(definition, version, file_and_scope, options)
true -> build_signed_url(definition, version, file_and_scope, options)
true -> build_signed_url(definition, version, file_and_scope, options)
end
end

Expand All @@ -40,11 +39,12 @@ defmodule Arc.Storage.S3 do
defp ensure_keyword_list(map) when is_map(map), do: Map.to_list(map)

# If the file is stored as a binary in-memory, send to AWS in a single request
defp do_put(file=%Arc.File{binary: file_binary}, {s3_bucket, s3_key, s3_options}) when is_binary(file_binary) do
defp do_put(file = %Arc.File{binary: file_binary}, {s3_bucket, s3_key, s3_options})
when is_binary(file_binary) do
ExAws.S3.put_object(s3_bucket, s3_key, file_binary, s3_options)
|> ExAws.request()
|> case do
{:ok, _res} -> {:ok, file.file_name}
{:ok, _res} -> {:ok, file.file_name}
{:error, error} -> {:error, error}
end
end
Expand All @@ -62,23 +62,23 @@ defmodule Arc.Storage.S3 do
end
rescue
e in ExAws.Error ->
Logger.error(inspect e)
Logger.error(inspect(e))
Logger.error(e.message)
{:error, :invalid_bucket}
end

defp build_url(definition, version, file_and_scope, _options) do
url = Path.join host(definition), s3_key(definition, version, file_and_scope)
url = Path.join(host(definition), s3_key(definition, version, file_and_scope))
url |> URI.encode()
end

defp build_signed_url(definition, version, file_and_scope, options) do
# Previous arc argument was expire_in instead of expires_in
# check for expires_in, if not present, use expire_at.
options = put_in options[:expires_in], Keyword.get(options, :expires_in, options[:expire_in])
options = put_in(options[:expires_in], Keyword.get(options, :expires_in, options[:expire_in]))
# fallback to default, if neither is present.
options = put_in options[:expires_in], options[:expires_in] || @default_expiry_time
options = put_in options[:virtual_host], virtual_host()
options = put_in(options[:expires_in], options[:expires_in] || @default_expiry_time)
options = put_in(options[:virtual_host], virtual_host())
config = ExAws.Config.new(:s3, Application.get_all_env(:ex_aws))
s3_key = s3_key(definition, version, file_and_scope)
s3_bucket = s3_bucket(definition)
Expand Down Expand Up @@ -111,7 +111,7 @@ defmodule Arc.Storage.S3 do
defp default_host(definition) do
case virtual_host() do
true -> "https://#{s3_bucket(definition)}.s3.amazonaws.com"
_ -> "https://s3.amazonaws.com/#{s3_bucket(definition)}"
_ -> "https://s3.amazonaws.com/#{s3_bucket(definition)}"
end
end

Expand Down
10 changes: 8 additions & 2 deletions lib/arc/transformations/convert.ex
Original file line number Diff line number Diff line change
@@ -1,14 +1,20 @@
defmodule Arc.Transformations.Convert do
def apply(cmd, file, args) do
new_path = Arc.File.generate_temporary_path(file)
args = if is_function(args), do: args.(file.path, new_path), else: [file.path | (String.split(args, " ") ++ [new_path])]
program = to_string(cmd)

args =
if is_function(args),
do: args.(file.path, new_path),
else: [file.path | String.split(args, " ") ++ [new_path]]

program = to_string(cmd)

ensure_executable_exists!(program)

case System.cmd(program, args_list(args), stderr_to_stdout: true) do
{_, 0} ->
{:ok, %Arc.File{file | path: new_path}}

{error_message, _exit_code} ->
{:error, error_message}
end
Expand Down
Loading