From f9b734fa12f3a23e9eb9862436aa56b378b9a11d Mon Sep 17 00:00:00 2001 From: Andre Henriques Date: Tue, 2 Jan 2024 21:39:00 +0000 Subject: [PATCH] Finished the paxos --- lib/leader_elector.ex | 2 +- lib/paxos.ex | 505 ++++++++++++++++++++++++-------- lib/paxos_test.ex | 640 ++++++++++++++++++++++++++++++++++++++++ lib/test_harness.ex | 114 ++++++++ lib/test_script.exs | 128 ++++++++ lib/test_util.ex | 26 ++ lib/utils.ex | 7 +- lib/uuid.ex | 662 ++++++++++++++++++++++++++++++++++++++++++ 8 files changed, 1967 insertions(+), 117 deletions(-) create mode 100755 lib/paxos_test.ex create mode 100755 lib/test_harness.ex create mode 100755 lib/test_script.exs create mode 100755 lib/test_util.ex create mode 100755 lib/uuid.ex diff --git a/lib/leader_elector.ex b/lib/leader_elector.ex index 49437fa..338de89 100644 --- a/lib/leader_elector.ex +++ b/lib/leader_elector.ex @@ -68,7 +68,7 @@ defmodule EventualLeaderElector do if MapSet.size(active) == 0 do state else - to_trust = Enum.at(MapSet.to_list(active), 0) + to_trust = Enum.at(Enum.sort(MapSet.to_list(active)), 0) to_trust = getOriginalName(to_trust) if state.last_trust != to_trust do diff --git a/lib/paxos.ex b/lib/paxos.ex index b116b32..46faeef 100644 --- a/lib/paxos.ex +++ b/lib/paxos.ex @@ -1,23 +1,15 @@ defmodule Paxos do - def getPaxosName(name) do - String.to_atom(Atom.to_string(name) <> "_paxos") - end - - def getOriginalName(name) do - String.to_atom(String.replace(Atom.to_string(name), "_paxos", "")) - end - def start(name, processes) do - new_name = getPaxosName(name) - pid = spawn(Paxos, :init, [new_name, name, processes]) + IO.puts("Starting paxos for #{name}") - Utils.register_name(new_name, pid) + pid = spawn(Paxos, :init, [name, name, processes]) + + Utils.register_name(name, pid, false) end # Init event must be the first # one after the component is created def init(name, parent, processes) do - processes = Enum.map(processes, fn name -> getPaxosName(name) end) EventualLeaderElector.start(name, processes) EagerReliableBroadcast.start(name, processes) @@ -26,153 +18,438 @@ defmodule Paxos do parent: parent, processes: processes, leader: nil, - value: nil, - other_value: nil, - ballot: 0, - ballot_value: nil, - prepared_values: [], - accepted: 0, - running_ballot: 0, - accepted_ballot: nil, - accepted_value: nil, - decided: false, - pid_to_inform: nil + instmap: %{}, + other_values: %{}, + decided: %{}, + aborted: MapSet.new(), + timeout: MapSet.new() } run(state) end + def add_inst_map(state, inst, value, pid_to_inform) do + instmap = + Map.put(state.instmap, inst, %{ + value: value, + ballot: 0, + ballot_value: nil, + prepared_values: [], + accepted: 0, + running_ballot: 0, + accepted_ballot: nil, + accepted_value: nil, + pid_to_inform: pid_to_inform + }) + + %{state | instmap: instmap} + end + + def has_finished(state, inst) do + Map.has_key?(state.decided, inst) or inst in state.timeout or inst in state.aborted + end + def run(state) do run( receive do {:ele_trust, proc} -> - prepare(%{state | leader: proc}) + IO.puts("#{state.name} - #{proc} is leader") - {:propose, value, pid_to_inform} -> - if state.value == nil do - EagerReliableBroadcast.broadcast(state.name, {:other_propose, value}) - prepare(%{state | value: value, pid_to_inform: pid_to_inform}) - else - %{state | pid_to_inform: pid_to_inform} - end + Enum.reduce(Map.keys(state.instmap), %{state | leader: proc}, fn inst, st -> + # IO.puts("#{state.name} - looping after leader: #{inst}") + prepare(st, inst) + end) - {:rb_deliver, _proc, {:other_propose, value}} -> - %{state | other_value: value} + {:propose, inst, value, t, pid_to_inform} -> + IO.puts("#{state.name} - Propose #{inst}") - {:rb_deliver, _proc, {:prepare, proc, ballot}} -> - if ballot > state.running_ballot do - Utils.unicast({:prepared, ballot, state.accepted_ballot, state.accepted_value}, proc) - %{state | running_ballot: ballot} - else - Utils.unicast({:nack, ballot}, proc) - state - end - - {:nack, ballot} -> - if state.leader == state.name and state.ballot == ballot do - prepare(state) - else - state - end - - {:prepared, ballot, accepted_ballot, accepted_value} -> - if ballot == state.ballot do - prepared(%{ + cond do + has_finished(state, inst) -> + send(pid_to_inform, {:abort, inst}) + state + + not Map.has_key?(state.instmap, inst) -> + EagerReliableBroadcast.broadcast(state.name, {:other_propose, inst, value}) + state = add_inst_map(state, inst, value, pid_to_inform) + Process.send_after(self(), {:timeout, inst}, t) + prepare(state, inst) + + state.instmap[inst].value == nil -> + Process.send_after(self(), {:timeout, inst}, t) + + prepare( + set_instmap(state, inst, %{ + state.instmap[inst] + | value: value, + pid_to_inform: pid_to_inform + }), + inst + ) + + true -> state - | prepared_values: state.prepared_values ++ [{accepted_ballot, accepted_value}] - }) - else - state end - {:rb_deliver, proc, {:accept, ballot, value}} -> - if ballot >= state.running_ballot do - Utils.unicast({:accepted, ballot}, proc) - %{state | running_ballot: ballot, accepted_value: value, accepted_ballot: ballot} - else - Utils.unicast({:nack, ballot}, proc) - state - end - - {:accepted, ballot} -> - if state.leader == state.name and state.ballot == ballot do - accepted(%{state | accepted: state.accepted + 1}) - else - state - end - - {:rb_deliver, _, {:decide, value}} -> - if state.decided == true do - state - else - if state.pid_to_inform != nil do - send(state.pid_to_inform, {:decide, value}) + {:rb_deliver, _proc, {:other_propose, inst, value}} -> + state = + if Map.has_key?(state.instmap, inst) do + state + else + add_inst_map(state, inst, nil, nil) end - %{state | decided: true} + %{state | other_values: Map.put(state.other_values, inst, value)} + + {:rb_deliver, _proc, {:prepare, proc, inst, ballot}} -> + IO.puts("#{state.name} - prepare") + + cond do + has_finished(state, inst) -> + state + + not Map.has_key?(state.instmap, inst) -> + state + + ballot > state.instmap[inst].running_ballot -> + Utils.unicast( + {:prepared, inst, ballot, state.instmap[inst].accepted_ballot, + state.instmap[inst].accepted_value}, + proc + ) + + set_instmap(state, inst, %{ + state.instmap[inst] + | running_ballot: ballot + }) + + true -> + Utils.unicast({:nack, inst, ballot}, proc) + state + end + + {:timeout, inst} -> + EagerReliableBroadcast.broadcast(state.name, {:timeout, inst}) + state + + {:rb_deliver, _proc, {:timeout, inst}} -> + IO.puts("#{state.name}- timeout") + + if has_finished(state, inst) do + state + else + if Map.has_key?(state.instmap, inst) do + if state.instmap[inst].pid_to_inform do + send(state.instmap[inst].pid_to_inform, {:timeout, inst}) + end + end + + %{ + state + | instmap: Map.delete(state.instmap, inst), + timeout: MapSet.put(state.timeout, inst) + } + end + + # {:rb_deliver, _proc, {:abort, inst}} -> + # IO.puts("#{state.name}- abort") + + # if has_finished(state, inst) do + # state + # else + # if Map.has_key?(state.instmap, inst) and state.instmap[inst].pid_to_inform != nil do + # send(state.instmap[inst].pid_to_inform, {:abort, inst}) + # end + + # %{ + # state + # | aborted: MapSet.put(state.aborted, inst), + # instmap: Map.delete(state.instmap, inst) + # } + # end + + {:nack, inst, ballot} -> + IO.puts("#{state.name}- nack") + + if has_finished(state, inst) do + state + else + if state.leader == state.name and state.instmap[inst].ballot == ballot do + # EagerReliableBroadcast.broadcast(state.name, {:abort, inst}) + + if Map.has_key?(state.instmap, inst) and state.instmap[inst].pid_to_inform != nil do + send(state.instmap[inst].pid_to_inform, {:abort, inst}) + end + + state + else + state + end + end + + {:prepared, inst, ballot, accepted_ballot, accepted_value} -> + IO.puts("#{state.name}- prepared") + + if has_finished(state, inst) do + state + else + if ballot == state.instmap[inst].ballot do + state = + set_instmap(state, inst, %{ + state.instmap[inst] + | prepared_values: + state.instmap[inst].prepared_values ++ [{accepted_ballot, accepted_value}] + }) + + prepared(state, inst) + else + state + end + end + + {:rb_deliver, proc, {:accept, inst, ballot, value}} -> + IO.puts("#{state.name} accept") + + if has_finished(state, inst) do + state + else + if ballot >= state.instmap[inst].running_ballot do + Utils.unicast({:accepted, inst, ballot}, proc) + + set_instmap(state, inst, %{ + state.instmap[inst] + | running_ballot: ballot, + accepted_value: value, + accepted_ballot: ballot + }) + else + Utils.unicast({:nack, inst, ballot}, proc) + state + end + end + + {:accepted, inst, ballot} -> + IO.puts("#{state.name} accepted") + + if has_finished(state, inst) do + state + else + if state.leader == state.name and state.instmap[inst].ballot == ballot do + accepted( + set_instmap(state, inst, %{ + state.instmap[inst] + | accepted: state.instmap[inst].accepted + 1 + }), + inst + ) + else + state + end + end + + {:get_value, inst, pid_to_inform, t} -> + # IO.puts("#{state.name} get_value") + + cond do + t < 0 -> + send(pid_to_inform, {:get_value_res, inst}) + + has_finished(state, inst) -> + cond do + inst in state.aborted -> + send(pid_to_inform, {:get_value_res, inst}) + + inst in state.timeout -> + send(pid_to_inform, {:get_value_res, inst}) + + Map.has_key?(state.decided, inst) -> + send(pid_to_inform, {:get_value_res_actual, inst, state.decided[inst]}) + end + + true -> + Process.send_after(self(), {:get_value, inst, pid_to_inform, t - 500}, 500) + end + + state + + {:rb_deliver, _, {:decide, inst, value}} -> + IO.puts("#{state.name} decided") + + if has_finished(state, inst) do + state + else + if Map.has_key?(state.instmap, inst) != nil and + state.instmap[inst].pid_to_inform != nil do + send(state.instmap[inst].pid_to_inform, {:decision, inst, value}) + end + + %{ + state + | decided: Map.put(state.decided, inst, value), + instmap: Map.delete(state.instmap, inst) + } end end ) end + def set_instmap(state, inst, instmap) do + new_instmap = Map.put(state.instmap, inst, instmap) + %{state | instmap: new_instmap} + end + # # Puts process in the preapre state # - def prepare(state) when state.leader != state.name, do: state - def prepare(state) when state.value == nil and state.other_value == nil, do: state + def prepare(state, _) when state.leader != state.name, do: state - def prepare(state) do - ballot = state.ballot + 1 - EagerReliableBroadcast.broadcast(state.name, {:prepare, state.name, ballot}) - %{state | ballot: ballot, prepared_values: [], accepted: 0, ballot_value: nil} + def prepare(state, inst) do + if Map.get(state.instmap, inst) == nil and Map.get(state.other_values, inst) == nil do + state + else + ballot = state.instmap[inst].ballot + 1 + EagerReliableBroadcast.broadcast(state.name, {:prepare, state.name, inst, ballot}) + + set_instmap(state, inst, %{ + state.instmap[inst] + | ballot: ballot, + prepared_values: [], + accepted: 0, + ballot_value: nil + }) + end end # # Process the prepared responses # - def prepared(state) when state.leader != state.name, do: state + def prepared(state, _) when state.leader != state.name, do: state - def prepared(state) when length(state.prepared_values) > length(state.processes) / 2 + 1 do - {_, a_val} = - Enum.reduce(state.prepared_values, {0, nil}, fn {bal, val}, {a_bal, a_val} -> - if a_bal > bal do - {a_bal, a_val} + def prepared(state, inst) do + if length(state.instmap[inst].prepared_values) >= floor(length(state.processes) / 2) + 1 do + {_, a_val} = + Enum.reduce(state.instmap[inst].prepared_values, {0, nil}, fn {bal, val}, + {a_bal, a_val} -> + if a_bal > bal do + {a_bal, a_val} + else + {bal, val} + end + end) + + a_val = + if a_val == nil do + if state.instmap[inst].value == nil do + state.other_values[inst] + else + state.instmap[inst].value + end else - {bal, val} + a_val end - end) - a_val = - if a_val == nil do - if state.value == nil do - state.other_value - else - state.value - end - else - a_val - end + EagerReliableBroadcast.broadcast( + state.name, + {:accept, inst, state.instmap[inst].ballot, a_val} + ) - EagerReliableBroadcast.broadcast(state.name, {:accept, state.ballot, a_val}) - %{state | ballot_value: a_val} + set_instmap(state, inst, %{ + state.instmap[inst] + | ballot_value: a_val + }) + else + state + end end - def prepared(state), do: state - # # Process the accepted responses # - def accepted(state) when state.leader != state.name, do: state + def accepted(state, _) when state.leader != state.name, do: state - def accepted(state) when state.accepted > length(state.processes) / 2 + 1 do - EagerReliableBroadcast.broadcast(state.name, {:decide, state.ballot_value}) - state + def accepted(state, inst) do + if state.instmap[inst].accepted >= floor(length(state.processes) / 2) + 1 do + value = state.instmap[inst].ballot_value + + EagerReliableBroadcast.broadcast( + state.name, + {:decide, inst, value} + ) + + if Map.has_key?(state.instmap, inst) != nil and + state.instmap[inst].pid_to_inform != nil do + send(state.instmap[inst].pid_to_inform, {:decision, inst, value}) + end + + %{ + state + | decided: Map.put(state.decided, inst, value), + instmap: Map.delete(state.instmap, inst) + } + else + state + end end - def accepted(state), do: state + def propose(pid, inst, value, t) do + # Utils.unicast({:propose, value}, name) - def propose(name, value) do - Utils.unicast({:propose, value}, getPaxosName(name)) + send(pid, {:propose, inst, value, t, self()}) + + propose_loop(inst) + end + + def propose_loop(inInst) do + receive do + {:timeout, inst} -> + if inInst == inst do + {:timeout} + else + propose_loop(inInst) + end + + {:abort, inst} -> + if inInst == inst do + {:abort} + else + propose_loop(inInst) + end + + {:decision, inst, d} -> + if inInst == inst do + {:decision, d} + else + propose_loop(inInst) + end + + _ -> + propose_loop(inInst) + end + end + + def get_decision(pid, inst, t) do + send(pid, {:get_value, inst, self(), t}) + get_decision_loop(inst) + end + + def get_decision_loop(inInst) do + receive do + {:get_value_res, inst} -> + if inst == inInst do + nil + else + get_decision_loop(inInst) + end + + {:get_value_res_actual, inst, v} -> + if inst == inInst do + v + else + get_decision_loop(inInst) + end + + _ -> + get_decision_loop(inInst) + end end end diff --git a/lib/paxos_test.ex b/lib/paxos_test.ex new file mode 100755 index 0000000..b195d58 --- /dev/null +++ b/lib/paxos_test.ex @@ -0,0 +1,640 @@ +defmodule PaxosTest do + # The functions implement + # the module specific testing logic + defp init(name, participants, all \\ false) do + cpid = TestHarness.wait_to_register(:coord, :global.whereis_name(:coord)) + + try do + pid = Paxos.start(name, participants) + Process.sleep(100) + if not Process.alive?(pid), do: raise("no pid") + + TestHarness.wait_for( + MapSet.new(participants), + name, + if(not all, + do: length(participants) / 2, + else: length(participants) + ) + ) + + {cpid, pid} + rescue + _ -> {cpid, :c.pid(0, 2048, 0)} + end + end + + defp kill_paxos(pid, name) do + Process.exit(pid, :kill) + :global.unregister_name(name) + pid + end + + defp wait_for_decision(_, _, timeout) when timeout <= 0, do: {:none, :none} + + defp wait_for_decision(pid, inst, timeout) do + Process.sleep(100) + v = Paxos.get_decision(pid, inst, 1) + + case v do + v when v != nil -> {:decide, v} + nil -> wait_for_decision(pid, inst, timeout - 100) + end + end + + defp propose_until_commit(pid, inst, val) do + status = Paxos.propose(pid, inst, val, 10000) + + case status do + {:decision, val} -> val + {:abort, val} -> propose_until_commit(pid, inst, val) + _ -> nil + end + end + + # Test cases start from here + + # No failures, no concurrent ballots + def run_simple(name, participants, val) do + {cpid, pid} = init(name, participants) + send(cpid, :ready) + + {status, val, a} = + receive do + :start -> + IO.puts("#{inspect(name)}: started") + leader = (fn [h | _] -> h end).(participants) + + if name == leader do + Paxos.propose(pid, 1, val, 10000) + end + + {status, v} = wait_for_decision(pid, 1, 10000) + + if status != :none do + IO.puts("#{name}: decided #{inspect(v)}") + else + IO.puts("#{name}: No decision after 10 seconds") + end + + {status, v, 10} + end + + send(cpid, :done) + + receive do + :all_done -> + IO.puts("#{name}: #{inspect(ql = Process.info(pid, :message_queue_len))}") + kill_paxos(pid, name) + send(cpid, {:finished, name, pid, status, val, a, ql}) + end + end + + # No failures, 2 concurrent ballots + def run_simple_2(name, participants, val) do + {cpid, pid} = init(name, participants) + send(cpid, :ready) + + {status, val, a} = + receive do + :start -> + IO.puts("#{inspect(name)}: started") + + if name in (fn [h1, h2 | _] -> [h1, h2] end).(participants), + do: Paxos.propose(pid, 1, val, 10000) + + {status, val} = wait_for_decision(pid, 1, 10000) + + if status != :none, + do: IO.puts("#{name}: decided #{inspect(val)}"), + else: IO.puts("#{name}: No decision after 10 seconds") + + {status, val, 10} + end + + send(cpid, :done) + + receive do + :all_done -> + Process.sleep(100) + IO.puts("#{name}: #{inspect(ql = Process.info(pid, :message_queue_len))}") + kill_paxos(pid, name) + send(cpid, {:finished, name, pid, status, val, a, ql}) + end + end + + # No failures, 2 concurrent instances + def run_simple_3(name, participants, val) do + {cpid, pid} = init(name, participants) + send(cpid, :ready) + + {status, val, a} = + receive do + :start -> + IO.puts("#{inspect(name)}: started") + proposers = Enum.zip((fn [h1, h2 | _] -> [h1, h2] end).(participants), [1, 2]) + proposers = for {k, v} <- proposers, into: %{}, do: {k, v} + if proposers[name], do: Paxos.propose(pid, proposers[name], val, 10000) + + y = + List.to_integer( + (fn [_ | x] -> x end).( + Atom.to_charlist((fn [h | _] -> h end).(Map.keys(proposers))) + ) + ) + + :rand.seed(:exrop, {y * 100 + 1, y * 100 + 2, y * 100 + 3}) + inst = Enum.random(1..2) + {status, val} = wait_for_decision(pid, inst, 10000) + + if status != :none, + do: IO.puts("#{name}: decided #{inspect(val)}"), + else: IO.puts("#{name}: No decision after 10 seconds") + + {status, val, 10} + end + + send(cpid, :done) + + receive do + :all_done -> + Process.sleep(100) + IO.puts("#{name}: #{inspect(ql = Process.info(pid, :message_queue_len))}") + kill_paxos(pid, name) + send(cpid, {:finished, name, pid, status, val, a, ql}) + end + end + + # # No failures, many concurrent ballots + def run_simple_many_1(name, participants, val) do + {cpid, pid} = init(name, participants) + send(cpid, :ready) + + {status, val, a} = + receive do + :start -> + IO.puts("#{inspect(name)}: started") + Paxos.propose(pid, 2, val, 10000) + Process.sleep(Enum.random(1..10)) + {status, val} = wait_for_decision(pid, 2, 10000) + + if status != :none, + do: IO.puts("#{name}: decided #{inspect(val)}"), + else: IO.puts("#{name}: No decision after 10 seconds") + + {status, val, 10} + end + + send(cpid, :done) + + receive do + :all_done -> + Process.sleep(100) + IO.puts("#{name}: #{inspect(ql = Process.info(pid, :message_queue_len))}") + kill_paxos(pid, name) + send(cpid, {:finished, name, pid, status, val, a, ql}) + end + end + + # No failures, many concurrent ballots + def run_simple_many_2(name, participants, val) do + {cpid, pid} = init(name, participants) + send(cpid, :ready) + + {status, val, a} = + receive do + :start -> + IO.puts("#{inspect(name)}: started") + + for _ <- 1..10 do + Process.sleep(Enum.random(1..10)) + Paxos.propose(pid, 1, val, 10000) + end + + {status, val} = wait_for_decision(pid, 1, 10000) + + if status != :none, + do: IO.puts("#{name}: decided #{inspect(val)}"), + else: IO.puts("#{name}: No decision after 10 seconds") + + {status, val, 10} + end + + send(cpid, :done) + + receive do + :all_done -> + Process.sleep(100) + IO.puts("#{name}: #{inspect(ql = Process.info(pid, :message_queue_len))}") + kill_paxos(pid, name) + send(cpid, {:finished, name, pid, status, val, a, ql}) + end + end + + # # One non-leader process crashes, no concurrent ballots + def run_non_leader_crash(name, participants, val) do + {cpid, pid} = init(name, participants, true) + send(cpid, :ready) + + {status, val, a, spare} = + receive do + :start -> + IO.puts("#{inspect(name)}: started") + + [leader, kill_p | spare] = participants + + case name do + ^leader -> + Paxos.propose(pid, 1, val, 10000) + + ^kill_p -> + Process.sleep(Enum.random(1..5)) + Process.exit(pid, :kill) + + _ -> + nil + end + + spare = [leader | spare] + + if name in spare do + {status, val} = wait_for_decision(pid, 1, 10000) + + if status != :none, + do: IO.puts("#{name}: decided #{inspect(val)}"), + else: IO.puts("#{name}: No decision after 10 seconds") + + {status, val, 10, spare} + else + {:killed, :none, -1, spare} + end + end + + send(cpid, :done) + + receive do + :all_done -> + Process.sleep(100) + + ql = + if name in spare do + IO.puts("#{name}: #{inspect(ql = Process.info(pid, :message_queue_len))}") + ql + else + {:message_queue_len, -1} + end + + kill_paxos(pid, name) + send(cpid, {:finished, name, pid, status, val, a, ql}) + end + end + + # # Minority non-leader crashes, no concurrent ballots + def run_minority_non_leader_crash(name, participants, val) do + {cpid, pid} = init(name, participants, true) + send(cpid, :ready) + + {status, val, a, spare} = + try do + receive do + :start -> + IO.puts("#{inspect(name)}: started") + + [leader | rest] = participants + + to_kill = Enum.slice(rest, 0, div(length(participants), 2)) + + if name == leader do + Paxos.propose(pid, 1, val, 10000) + end + + if name in to_kill do + Process.sleep(Enum.random(1..5)) + Process.exit(pid, :kill) + end + + spare = for p <- participants, p not in to_kill, do: p + + if name in spare do + {status, val} = wait_for_decision(pid, 1, 10000) + + if status != :none, + do: IO.puts("#{name}: decided #{inspect(val)}"), + else: IO.puts("#{name}: No decision after 10 seconds") + + {status, val, 10, spare} + else + {:killed, :none, -1, spare} + end + end + rescue + _ -> {:none, :none, 10, []} + end + + send(cpid, :done) + + receive do + :all_done -> + Process.sleep(100) + + ql = + if name in spare do + IO.puts("#{name}: #{inspect(ql = Process.info(pid, :message_queue_len))}") + ql + else + {:message_queue_len, -1} + end + + kill_paxos(pid, name) + send(cpid, {:finished, name, pid, status, val, a, ql}) + end + end + + # # Leader crashes, no concurrent ballots + def run_leader_crash_simple(name, participants, val) do + {cpid, pid} = init(name, participants, true) + send(cpid, :ready) + + {status, val, a, spare} = + try do + receive do + :start -> + IO.puts("#{inspect(name)}: started") + + [leader | spare] = participants + [new_leader | _] = spare + + if name == leader do + Paxos.propose(pid, 1, val, 10000) + Process.sleep(Enum.random(1..5)) + Process.exit(pid, :kill) + end + + if name == new_leader do + Process.sleep(10) + propose_until_commit(pid, 1, val) + end + + if name in spare do + {status, val} = wait_for_decision(pid, 1, 10000) + + if status != :none, + do: IO.puts("#{name}: decided #{inspect(val)}"), + else: IO.puts("#{name}: No decision after 10 seconds") + + {status, val, 10, spare} + else + {:killed, :none, -1, spare} + end + end + rescue + _ -> {:none, :none, 10, []} + end + + send(cpid, :done) + + receive do + :all_done -> + Process.sleep(100) + + ql = + if name in spare do + IO.puts("#{name}: #{inspect(ql = Process.info(pid, :message_queue_len))}") + ql + else + {:message_queue_len, -1} + end + + kill_paxos(pid, name) + send(cpid, {:finished, name, pid, status, val, a, ql}) + end + end + + # # Leader and some non-leaders crash, no concurrent ballots + # # Needs to be run with at least 5 process config + def run_leader_crash_simple_2(name, participants, val) do + {cpid, pid} = init(name, participants, true) + send(cpid, :ready) + + {status, val, a, spare} = + receive do + :start -> + IO.puts("#{inspect(name)}: started") + leader = (fn [h | _] -> h end).(participants) + + if name == leader do + Paxos.propose(pid, 1, val, 10000) + Process.sleep(Enum.random(1..5)) + Process.exit(pid, :kill) + end + + spare = + Enum.reduce( + List.delete(participants, leader), + List.delete(participants, leader), + fn _, s -> if length(s) > length(participants) / 2 + 1, do: tl(s), else: s end + ) + + leader = hd(spare) + + if name not in spare do + Process.sleep(Enum.random(1..5)) + Process.exit(pid, :kill) + end + + if name == leader do + Process.sleep(10) + propose_until_commit(pid, 1, val) + end + + if name in spare do + {status, val} = wait_for_decision(pid, 1, 10000) + + if status != :none, + do: IO.puts("#{name}: decided #{inspect(val)}"), + else: IO.puts("#{name}: No decision after 10 seconds") + + {status, val, 10, spare} + else + {:killed, :none, -1, spare} + end + end + + send(cpid, :done) + + receive do + :all_done -> + Process.sleep(100) + + ql = + if name in spare do + IO.puts("#{name}: #{inspect(ql = Process.info(pid, :message_queue_len))}") + ql + else + {:message_queue_len, -1} + end + + kill_paxos(pid, name) + send(cpid, {:finished, name, pid, status, val, a, ql}) + end + end + + # # Cascading failures of leaders and non-leaders + def run_leader_crash_complex(name, participants, val) do + {cpid, pid} = init(name, participants, true) + send(cpid, :ready) + + {status, val, a, spare} = + receive do + :start -> + IO.puts("#{inspect(name)}: started with #{inspect(participants)}") + + {kill, spare} = + Enum.reduce(participants, {[], participants}, fn _, {k, s} -> + if length(s) > length(participants) / 2 + 1, do: {k ++ [hd(s)], tl(s)}, else: {k, s} + end) + + leaders = Enum.slice(kill, 0, div(length(kill), 2)) + followers = Enum.slice(kill, div(length(kill), 2), div(length(kill), 2) + 1) + + # IO.puts("spare = #{inspect spare}") + # IO.puts "kill: leaders, followers = #{inspect leaders}, #{inspect followers}" + + if name in leaders do + Paxos.propose(pid, 1, val, 10000) + Process.sleep(Enum.random(1..5)) + Process.exit(pid, :kill) + end + + if name in followers do + Process.sleep(Enum.random(1..5)) + Process.exit(pid, :kill) + end + + if hd(spare) == name do + Process.sleep(10) + propose_until_commit(pid, 1, val) + end + + if name in spare do + {status, val} = wait_for_decision(pid, 1, 50000) + + if status != :none, + do: IO.puts("#{name}: decided #{inspect(val)}"), + else: IO.puts("#{name}: No decision after 50 seconds") + + {status, val, 10, spare} + else + {:killed, :none, -1, spare} + end + end + + send(cpid, :done) + + receive do + :all_done -> + Process.sleep(100) + + ql = + if name in spare do + IO.puts("#{name}: #{inspect(ql = Process.info(pid, :message_queue_len))}") + ql + else + {:message_queue_len, -1} + end + + kill_paxos(pid, name) + send(cpid, {:finished, name, pid, status, val, a, ql}) + end + end + + # # Cascading failures of leaders and non-leaders, random delays + def run_leader_crash_complex_2(name, participants, val) do + {cpid, pid} = init(name, participants, true) + send(cpid, :ready) + + {status, val, a, spare} = + try do + receive do + :start -> + IO.puts("#{inspect(name)}: started") + + {kill, spare} = + Enum.reduce(participants, {[], participants}, fn _, {k, s} -> + if length(s) > length(participants) / 2 + 1, + do: {k ++ [hd(s)], tl(s)}, + else: {k, s} + end) + + leaders = Enum.slice(kill, 0, div(length(kill), 2)) + followers = Enum.slice(kill, div(length(kill), 2), div(length(kill), 2) + 1) + + IO.puts("spare = #{inspect(spare)}") + IO.puts("kill: leaders, followers = #{inspect(leaders)}, #{inspect(followers)}") + + if name in leaders do + Paxos.propose(pid, 1, val, 10000) + Process.sleep(Enum.random(1..5)) + Process.exit(pid, :kill) + end + + if name in followers do + for _ <- 1..10 do + :erlang.suspend_process(pid) + Process.sleep(Enum.random(1..5)) + :erlang.resume_process(pid) + end + + Process.exit(pid, :kill) + end + + if hd(spare) == name do + Process.sleep(10) + Paxos.propose(pid, 1, val, 10000) + end + + if name in spare do + for _ <- 1..10 do + :erlang.suspend_process(pid) + Process.sleep(Enum.random(1..5)) + :erlang.resume_process(pid) + leader = hd(Enum.reverse(spare)) + if name == leader, do: Paxos.propose(pid, 1, val, 10000) + end + + leader = hd(spare) + if name == leader, do: propose_until_commit(pid, 1, val) + {status, val} = wait_for_decision(pid, 1, 50000) + + if status != :none, + do: IO.puts("#{name}: decided #{inspect(val)}"), + else: IO.puts("#{name}: No decision after 50 seconds") + + {status, val, 10, spare} + else + {:killed, :none, -1, spare} + end + end + rescue + _ -> {:none, :none, 10, []} + end + + send(cpid, :done) + + receive do + :all_done -> + Process.sleep(100) + + ql = + if name in spare do + IO.puts("#{name}: #{inspect(ql = Process.info(pid, :message_queue_len))}") + ql + else + {:message_queue_len, -1} + end + + kill_paxos(pid, name) + send(cpid, {:finished, name, pid, status, val, a, ql}) + end + end +end diff --git a/lib/test_harness.ex b/lib/test_harness.ex new file mode 100755 index 0000000..3f2d438 --- /dev/null +++ b/lib/test_harness.ex @@ -0,0 +1,114 @@ +defmodule TestHarness do + + # @compile :nowarn_unused_vars + + # TODO: limit the number of attempts + def wait_to_register(name, :undefined) do + Process.sleep(10) + wait_to_register(name, :global.whereis_name(name)) + end + def wait_to_register(_, pid), do: pid + + defp wait_for_set(_, n, q, _, false) when n < q, do: :done + defp wait_for_set(procs, _, q, name, _) do + Process.sleep(10) + s = Enum.reduce(procs, MapSet.new, + fn p, s -> if :global.whereis_name(p) != :undefined, do: MapSet.put(s, p), else: s end) + (fn d -> wait_for_set(d, MapSet.size(d), q, name, name in d) end).(MapSet.difference(procs, s)) + end + def wait_for(proc_set, my_name, q) do + (fn d, n -> wait_for_set(d, n, q, my_name, my_name in d) end). + (proc_set, MapSet.size(proc_set)) + end + + def send_back_os_pid(pid) do + send(pid, {:os_pid, :os.getpid()}) + end + + def wait_until_up(node) do + Process.sleep(500) + status = Node.ping(node) + IO.puts("#{status}") + case status do + :pong -> :ok + _ -> wait_until_up(node) + end + end + + def get_os_pid(node) do + Process.sleep(1000) + # Node.spawn(node, TestHarness, :send_back_os_pid, [self()]) + (fn pid -> Node.spawn(node, + fn -> send(pid, {:os_pid, :os.getpid}) end) end).(self()) + receive do + {:os_pid, os_pid} -> os_pid + after 1000 -> get_os_pid(node) + end + end + + def deploy_procs(func, config) do + os_pids = for node <- MapSet.new(nodes(config)) do + cmd = "elixir --sname " <> (hd String.split(Atom.to_string(node), "@")) <> " --no-halt --erl \"-detached\" --erl \"-kernel prevent_overlapping_partitions false\"" + cmd = String.to_charlist(cmd) + # IO.puts("#{inspect cmd}") + :os.cmd(cmd) + # wait_until_up(node) + get_os_pid(node) + end + + pids = (fn participants -> + for {name, {node, param}} <- config do + case node do + :local -> Node.spawn(Node.self, fn -> func.(name, participants, param) end) + _ -> Node.spawn(node, fn -> func.(name, participants, param) end) + end + end + end).(proc_names(config)) + {pids, os_pids} + end + + def proc_names(config), do: for {name, _} <- config, do: name + def nodes(config), do: for {_, {node, _}} <- config, node != :local, do: node + + def notify_all(procs, msg) do + for p <- procs, do: send(p, msg) + end + + defp sync(msg, n) do + for _ <- 1..n do + receive do + ^msg -> :ok + end + end + end + + defp sync_and_collect(m_type, n) do + Enum.reduce(1..n, [], + fn _, res -> + [h | t] = receive do + msg -> Tuple.to_list(msg) + end + if h == m_type, do: [List.to_tuple(t) | res], else: res + end) + end + + defp kill_os_procs(os_pids) do + for os_pid <- os_pids, do: :os.cmd('kill -9 ' ++ os_pid ++ ' 2>/dev/null') + end + + # ideally should take an instance of a protocol for tested module + def test(func, config) do + :global.re_register_name(:coord, self()) + # pids = deploy_procs(&FloodingTest.run/2) + {pids, os_pids} = deploy_procs(func, config) + sync(:ready, length(config)) + notify_all(pids, :start) + sync(:done, length(config)) + notify_all(pids, :all_done) + # sync(:finished, length(config)) + res = sync_and_collect(:finished, length(config)) + :global.unregister_name(:coord) + kill_os_procs(os_pids) + res + end +end \ No newline at end of file diff --git a/lib/test_script.exs b/lib/test_script.exs new file mode 100755 index 0000000..5a7c84b --- /dev/null +++ b/lib/test_script.exs @@ -0,0 +1,128 @@ +# Replace with your own implementation source files +IEx.Helpers.c("utils.ex", ".") +IEx.Helpers.c("eager_reliable_broadcast.ex", ".") +IEx.Helpers.c("leader_elector.ex", ".") +IEx.Helpers.c("paxos.ex", ".") + +# Do not modify the following ########## +IEx.Helpers.c("test_harness.ex", ".") +IEx.Helpers.c("paxos_test.ex", ".") +IEx.Helpers.c("uuid.ex", ".") +IEx.Helpers.c("test_util.ex", ".") + +host = String.trim(to_string(:os.cmd(~c"hostname -s"))) + +# ########### + +test_suite = [ + # test case, configuration, number of times to run the case, description + # Use TestUtil.get_dist_config(host, n) to generate a multi-node configuration + # consisting of n processes, each one on a different node. + # Use TestUtil.get_local_config(n) to generate a single-node configuration + # consisting of n processes, all running on the same node. + + # {&PaxosTest.run_simple/3, TestUtil.get_local_config(3), 10, + # "No failures, no concurrent ballots, 3 local procs"}, + # {&PaxosTest.run_simple/3, TestUtil.get_dist_config(host, 3), 10, + # "No failures, no concurrent ballots, 3 nodes"}, + # {&PaxosTest.run_simple/3, TestUtil.get_local_config(5), 10, + # "No failures, no concurrent ballots, 5 local procs"}, + # {&PaxosTest.run_simple_2/3, TestUtil.get_dist_config(host, 3), 10, + # "No failures, 2 concurrent ballots, 3 nodes"}, + # {&PaxosTest.run_simple_2/3, TestUtil.get_local_config(3), 10, + # "No failures, 2 concurrent ballots, 3 local procs"}, + # {&PaxosTest.run_simple_3/3, TestUtil.get_local_config(3), 10, + # "No failures, 2 concurrent instances, 3 local procs"}, + # {&PaxosTest.run_simple_many_1/3, TestUtil.get_dist_config(host, 5), 10, + # "No failures, many concurrent ballots 1, 5 nodes"}, + # {&PaxosTest.run_simple_many_1/3, TestUtil.get_local_config(5), 10, + # "No failures, many concurrent ballots 1, 5 local procs"}, + # {&PaxosTest.run_simple_many_2/3, TestUtil.get_dist_config(host, 5), 10, + # "No failures, many concurrent ballots 2, 5 nodes"}, + # {&PaxosTest.run_simple_many_2/3, TestUtil.get_local_config(5), 10, + # "No failures, many concurrent ballots 2, 5 local procs"}, + # {&PaxosTest.run_non_leader_crash/3, TestUtil.get_dist_config(host, 3), 10, + # "One non-leader crashes, no concurrent ballots, 3 nodes"}, + # {&PaxosTest.run_non_leader_crash/3, TestUtil.get_local_config(3), 10, + # "One non-leader crashes, no concurrent ballots, 3 local procs"}, + # {&PaxosTest.run_minority_non_leader_crash/3, TestUtil.get_dist_config(host, 5), 10, + # "Minority non-leader crashes, no concurrent ballots"}, + # {&PaxosTest.run_minority_non_leader_crash/3, TestUtil.get_local_config(5), 10, + # "Minority non-leader crashes, no concurrent ballots"}, + {&PaxosTest.run_leader_crash_simple/3, TestUtil.get_dist_config(host, 5), 10, + "Leader crashes, no concurrent ballots, 5 nodes"}, + {&PaxosTest.run_leader_crash_simple/3, TestUtil.get_local_config(5), 10, + "Leader crashes, no concurrent ballots, 5 local procs"}, + {&PaxosTest.run_leader_crash_simple_2/3, TestUtil.get_dist_config(host, 7), 10, + "Leader and some non-leaders crash, no concurrent ballots, 7 nodes"}, + {&PaxosTest.run_leader_crash_simple_2/3, TestUtil.get_local_config(7), 10, + "Leader and some non-leaders crash, no concurrent ballots, 7 local procs"}, + {&PaxosTest.run_leader_crash_complex/3, TestUtil.get_dist_config(host, 11), 10, + "Cascading failures of leaders and non-leaders, 11 nodes"}, + {&PaxosTest.run_leader_crash_complex/3, TestUtil.get_local_config(11), 10, + "Cascading failures of leaders and non-leaders, 11 local procs"}, + {&PaxosTest.run_leader_crash_complex_2/3, TestUtil.get_dist_config(host, 11), 10, + "Cascading failures of leaders and non-leaders, random delays, 7 nodes"}, + {&PaxosTest.run_leader_crash_complex_2/3, TestUtil.get_local_config(11), 10, + "Cascading failures of leaders and non-leaders, random delays, 7 local procs"} +] + +Node.stop() +# Confusingly, Node.start fails if epmd is not running. +# epmd can be started manually with "epmd -daemon" or +# will start automatically whenever any Erlang VM is +# started with --sname or --name option. +Node.start(TestUtil.get_node(host), :shortnames) + +Enum.reduce(test_suite, length(test_suite), fn {func, config, n, doc}, acc -> + IO.puts(:stderr, "============") + IO.puts(:stderr, "#{inspect(doc)}, #{inspect(n)} time#{if n > 1, do: "s", else: ""}") + IO.puts(:stderr, "============") + + for _ <- 1..n do + res = TestHarness.test(func, Enum.shuffle(Map.to_list(config))) + # IO.puts("#{inspect res}") + {vl, al, ll} = + Enum.reduce(res, {[], [], []}, fn + {_, _, s, v, a, {:message_queue_len, l}}, {vl, al, ll} -> + # if s not in [:killed, :none], do: {[v | vl], [a | al], [l | ll]}, + if s not in [:killed], + do: {[v | vl], [a | al], [l | ll]}, + else: {vl, al, ll} + + {_, _, _, _, _, nil}, {vl, al, ll} -> + {vl, al, ll} + end) + + IO.puts("#{inspect(vl)}") + termination = vl != [] and :none not in vl + agreement = termination and MapSet.size(MapSet.new(vl)) == 1 + {:val, agreement_val} = if agreement, do: hd(vl), else: {:val, -1} + validity = agreement_val in 201..210 + safety = agreement and validity + TestUtil.pause_stderr(100) + + if termination and safety do + too_many_attempts = (get_att = fn a -> 10 - a + 1 end).(Enum.max(al)) > 5 + too_many_messages_left = Enum.max(ll) > 10 + warn = if too_many_attempts, do: [{:too_many_attempts, get_att.(Enum.max(al))}], else: [] + + warn = + if too_many_messages_left, + do: [{:too_many_messages_left, Enum.max(ll)} | warn], + else: warn + + IO.puts(:stderr, if(warn == [], do: "PASS", else: "PASS (#{inspect(warn)})")) + # IO.puts(:stderr, "#{inspect res}") + else + IO.puts(:stderr, "FAIL\n\t#{inspect(res)}") + end + end + + IO.puts(:stderr, "============#{if acc > 1, do: "\n", else: ""}") + acc - 1 +end) + +:os.cmd(~c"/bin/rm -f *.beam") +Node.stop() +System.halt() diff --git a/lib/test_util.ex b/lib/test_util.ex new file mode 100755 index 0000000..f9fee8c --- /dev/null +++ b/lib/test_util.ex @@ -0,0 +1,26 @@ +defmodule TestUtil do + + def get_node(host), do: String.to_atom(UUID.uuid1 <> "@" <> host) + + def get_dist_config(host, n) do + for i <- (1..n), into: %{}, do: + {String.to_atom("p"<>to_string(i)), {get_node(host), {:val, Enum.random(201..210)}}} + end + + def get_local_config(n) do + for i <- 1..n, into: %{}, do: + {String.to_atom("p"<>to_string(i)), {:local, {:val, Enum.random(201..210)}}} + end + + def pause_stderr(d) do + my_pid = self() + spawn(fn -> pid=Process.whereis(:standard_error); + :erlang.suspend_process(pid); + send(my_pid, {:suspended}) + Process.sleep(d); + :erlang.resume_process(pid) end) + receive do + {:suspended} -> :done + end + end +end \ No newline at end of file diff --git a/lib/utils.ex b/lib/utils.ex index 4635a3e..bb15cbd 100644 --- a/lib/utils.ex +++ b/lib/utils.ex @@ -8,12 +8,15 @@ defmodule Utils do def beb_broadcast(m, dest), do: for(p <- dest, do: unicast(m, p)) - def register_name(name, pid) do + def register_name(name, pid, link \\ true) do case :global.re_register_name(name, pid) do :yes -> # Note this is running on the parent so we are linking the parent to the rb # so that when we close the parent the rb also dies - Process.link(pid) + if link do + Process.link(pid) + end + pid :no -> diff --git a/lib/uuid.ex b/lib/uuid.ex new file mode 100755 index 0000000..e651e78 --- /dev/null +++ b/lib/uuid.ex @@ -0,0 +1,662 @@ +defmodule UUID do + use Bitwise, only_operators: true + @moduledoc """ + UUID generator and utilities for [Elixir](http://elixir-lang.org/). + See [RFC 4122](http://www.ietf.org/rfc/rfc4122.txt). + """ + + @nanosec_intervals_offset 122_192_928_000_000_000 # 15 Oct 1582 to 1 Jan 1970. + @nanosec_intervals_factor 10 # Microseconds to nanoseconds factor. + + @variant10 2 # Variant, corresponds to variant 1 0 of RFC 4122. + @uuid_v1 1 # UUID v1 identifier. + @uuid_v3 3 # UUID v3 identifier. + @uuid_v4 4 # UUID v4 identifier. + @uuid_v5 5 # UUID v5 identifier. + + @urn "urn:uuid:" # UUID URN prefix. + + @doc """ + Inspect a UUID and return tuple with `{:ok, result}`, where result is + information about its 128-bit binary content, type, + version and variant. + + Timestamp portion is not checked to see if it's in the future, and therefore + not yet assignable. See "Validation mechanism" in section 3 of + [RFC 4122](http://www.ietf.org/rfc/rfc4122.txt). + + Will return `{:error, message}` if the given string is not a UUID representation + in a format like: + * `"870df8e8-3107-4487-8316-81e089b8c2cf"` + * `"8ea1513df8a14dea9bea6b8f4b5b6e73"` + * `"urn:uuid:ef1b1a28-ee34-11e3-8813-14109ff1a304"` + + ## Examples + + ```elixir + iex> UUID.info("870df8e8-3107-4487-8316-81e089b8c2cf") + {:ok, [uuid: "870df8e8-3107-4487-8316-81e089b8c2cf", + binary: <<135, 13, 248, 232, 49, 7, 68, 135, 131, 22, 129, 224, 137, 184, 194, 207>>, + type: :default, + version: 4, + variant: :rfc4122]} + + iex> UUID.info("8ea1513df8a14dea9bea6b8f4b5b6e73") + {:ok, [uuid: "8ea1513df8a14dea9bea6b8f4b5b6e73", + binary: <<142, 161, 81, 61, 248, 161, 77, 234, 155, + 234, 107, 143, 75, 91, 110, 115>>, + type: :hex, + version: 4, + variant: :rfc4122]} + + iex> UUID.info("urn:uuid:ef1b1a28-ee34-11e3-8813-14109ff1a304") + {:ok, [uuid: "urn:uuid:ef1b1a28-ee34-11e3-8813-14109ff1a304", + binary: <<239, 27, 26, 40, 238, 52, 17, 227, 136, 19, 20, 16, 159, 241, 163, 4>>, + type: :urn, + version: 1, + variant: :rfc4122]} + + iex> UUID.info(<<39, 73, 196, 181, 29, 90, 74, 96, 157, 47, 171, 144, 84, 164, 155, 52>>) + {:ok, [uuid: <<39, 73, 196, 181, 29, 90, 74, 96, 157, 47, 171, 144, 84, 164, 155, 52>>, + binary: <<39, 73, 196, 181, 29, 90, 74, 96, 157, 47, 171, 144, 84, 164, 155, 52>>, + type: :raw, + version: 4, + variant: :rfc4122]} + + iex> UUID.info("12345") + {:error, "Invalid argument; Not a valid UUID: 12345"} + + ``` + + """ + def info(uuid) do + try do + {:ok, UUID.info!(uuid)} + rescue + e in ArgumentError -> {:error, e.message} + end + end + + @doc """ + Inspect a UUID and return information about its 128-bit binary content, type, + version and variant. + + Timestamp portion is not checked to see if it's in the future, and therefore + not yet assignable. See "Validation mechanism" in section 3 of + [RFC 4122](http://www.ietf.org/rfc/rfc4122.txt). + + Will raise an `ArgumentError` if the given string is not a UUID representation + in a format like: + * `"870df8e8-3107-4487-8316-81e089b8c2cf"` + * `"8ea1513df8a14dea9bea6b8f4b5b6e73"` + * `"urn:uuid:ef1b1a28-ee34-11e3-8813-14109ff1a304"` + + ## Examples + + ```elixir + iex> UUID.info!("870df8e8-3107-4487-8316-81e089b8c2cf") + [uuid: "870df8e8-3107-4487-8316-81e089b8c2cf", + binary: <<135, 13, 248, 232, 49, 7, 68, 135, 131, 22, 129, 224, 137, 184, 194, 207>>, + type: :default, + version: 4, + variant: :rfc4122] + + iex> UUID.info!("8ea1513df8a14dea9bea6b8f4b5b6e73") + [uuid: "8ea1513df8a14dea9bea6b8f4b5b6e73", + binary: <<142, 161, 81, 61, 248, 161, 77, 234, 155, + 234, 107, 143, 75, 91, 110, 115>>, + type: :hex, + version: 4, + variant: :rfc4122] + + iex> UUID.info!("urn:uuid:ef1b1a28-ee34-11e3-8813-14109ff1a304") + [uuid: "urn:uuid:ef1b1a28-ee34-11e3-8813-14109ff1a304", + binary: <<239, 27, 26, 40, 238, 52, 17, 227, 136, 19, 20, 16, 159, 241, 163, 4>>, + type: :urn, + version: 1, + variant: :rfc4122] + + iex> UUID.info!(<<39, 73, 196, 181, 29, 90, 74, 96, 157, 47, 171, 144, 84, 164, 155, 52>>) + [uuid: <<39, 73, 196, 181, 29, 90, 74, 96, 157, 47, 171, 144, 84, 164, 155, 52>>, + binary: <<39, 73, 196, 181, 29, 90, 74, 96, 157, 47, 171, 144, 84, 164, 155, 52>>, + type: :raw, + version: 4, + variant: :rfc4122] + + ``` + + """ + def info!(<> = uuid_string) do + {type, <>} = uuid_string_to_hex_pair(uuid) + <<_::48, version::4, _::12, v0::1, v1::1, v2::1, _::61>> = <> + [uuid: uuid_string, + binary: <>, + type: type, + version: version, + variant: variant(<>)] + end + def info!(_) do + raise ArgumentError, message: "Invalid argument; Expected: String" + end + + @doc """ + Convert binary UUID data to a string. + + Will raise an ArgumentError if the given binary is not valid UUID data, or + the format argument is not one of: `:default`, `:hex`, `:urn`, or `:raw`. + + ## Examples + + ```elixir + iex> UUID.binary_to_string!(<<135, 13, 248, 232, 49, 7, 68, 135, + ...> 131, 22, 129, 224, 137, 184, 194, 207>>) + "870df8e8-3107-4487-8316-81e089b8c2cf" + + iex> UUID.binary_to_string!(<<142, 161, 81, 61, 248, 161, 77, 234, 155, + ...> 234, 107, 143, 75, 91, 110, 115>>, :hex) + "8ea1513df8a14dea9bea6b8f4b5b6e73" + + iex> UUID.binary_to_string!(<<239, 27, 26, 40, 238, 52, 17, 227, 136, + ...> 19, 20, 16, 159, 241, 163, 4>>, :urn) + "urn:uuid:ef1b1a28-ee34-11e3-8813-14109ff1a304" + + iex> UUID.binary_to_string!(<<39, 73, 196, 181, 29, 90, 74, 96, 157, + ...> 47, 171, 144, 84, 164, 155, 52>>, :raw) + <<39, 73, 196, 181, 29, 90, 74, 96, 157, 47, 171, 144, 84, 164, 155, 52>> + + ``` + + """ + def binary_to_string!(uuid, format \\ :default) + def binary_to_string!(<>, format) do + uuid_to_string(<>, format) + end + def binary_to_string!(_, _) do + raise ArgumentError, message: "Invalid argument; Expected: <>" + end + + @doc """ + Convert a UUID string to its binary data equivalent. + + Will raise an ArgumentError if the given string is not a UUID representation + in a format like: + * `"870df8e8-3107-4487-8316-81e089b8c2cf"` + * `"8ea1513df8a14dea9bea6b8f4b5b6e73"` + * `"urn:uuid:ef1b1a28-ee34-11e3-8813-14109ff1a304"` + + ## Examples + + ```elixir + iex> UUID.string_to_binary!("870df8e8-3107-4487-8316-81e089b8c2cf") + <<135, 13, 248, 232, 49, 7, 68, 135, 131, 22, 129, 224, 137, 184, 194, 207>> + + iex> UUID.string_to_binary!("8ea1513df8a14dea9bea6b8f4b5b6e73") + <<142, 161, 81, 61, 248, 161, 77, 234, 155, 234, 107, 143, 75, 91, 110, 115>> + + iex> UUID.string_to_binary!("urn:uuid:ef1b1a28-ee34-11e3-8813-14109ff1a304") + <<239, 27, 26, 40, 238, 52, 17, 227, 136, 19, 20, 16, 159, 241, 163, 4>> + + iex> UUID.string_to_binary!(<<39, 73, 196, 181, 29, 90, 74, 96, 157, 47, + ...> 171, 144, 84, 164, 155, 52>>) + <<39, 73, 196, 181, 29, 90, 74, 96, 157, 47, 171, 144, 84, 164, 155, 52>> + + ``` + + """ + def string_to_binary!(<>) do + {_type, <>} = uuid_string_to_hex_pair(uuid) + <> + end + def string_to_binary!(_) do + raise ArgumentError, message: "Invalid argument; Expected: String" + end + + @doc """ + Generate a new UUID v1. This version uses a combination of one or more of: + unix epoch, random bytes, pid hash, and hardware address. + + ## Examples + + ```elixir + iex> UUID.uuid1() + "cdfdaf44-ee35-11e3-846b-14109ff1a304" + + iex> UUID.uuid1(:default) + "cdfdaf44-ee35-11e3-846b-14109ff1a304" + + iex> UUID.uuid1(:hex) + "cdfdaf44ee3511e3846b14109ff1a304" + + iex> UUID.uuid1(:urn) + "urn:uuid:cdfdaf44-ee35-11e3-846b-14109ff1a304" + + iex> UUID.uuid1(:raw) + <<205, 253, 175, 68, 238, 53, 17, 227, 132, 107, 20, 16, 159, 241, 163, 4>> + + iex> UUID.uuid1(:slug) + "zf2vRO41EeOEaxQQn_GjBA" + ``` + + """ + def uuid1(format \\ :default) do + uuid1(uuid1_clockseq(), uuid1_node(), format) + end + + @doc """ + Generate a new UUID v1, with an existing clock sequence and node address. This + version uses a combination of one or more of: unix epoch, random bytes, + pid hash, and hardware address. + + ## Examples + + ```elixir + iex> UUID.uuid1() + "cdfdaf44-ee35-11e3-846b-14109ff1a304" + + iex> UUID.uuid1(:default) + "cdfdaf44-ee35-11e3-846b-14109ff1a304" + + iex> UUID.uuid1(:hex) + "cdfdaf44ee3511e3846b14109ff1a304" + + iex> UUID.uuid1(:urn) + "urn:uuid:cdfdaf44-ee35-11e3-846b-14109ff1a304" + + iex> UUID.uuid1(:raw) + <<205, 253, 175, 68, 238, 53, 17, 227, 132, 107, 20, 16, 159, 241, 163, 4>> + + iex> UUID.uuid1(:slug) + "zf2vRO41EeOEaxQQn_GjBA" + ``` + + """ + def uuid1(clock_seq, node, format \\ :default) + def uuid1(<>, <>, format) do + <> = uuid1_time() + <> = <> + <> + |> uuid_to_string(format) + end + def uuid1(_, _, _) do + raise ArgumentError, message: + "Invalid argument; Expected: <>, <>" + end + + @doc """ + Generate a new UUID v3. This version uses an MD5 hash of fixed value (chosen + based on a namespace atom - see Appendix C of + [RFC 4122](http://www.ietf.org/rfc/rfc4122.txt) and a name value. Can also be + given an existing UUID String instead of a namespace atom. + + Accepted arguments are: `:dns`|`:url`|`:oid`|`:x500`|`:nil` OR uuid, String + + ## Examples + + ```elixir + iex> UUID.uuid3(:dns, "my.domain.com") + "03bf0706-b7e9-33b8-aee5-c6142a816478" + + iex> UUID.uuid3(:dns, "my.domain.com", :default) + "03bf0706-b7e9-33b8-aee5-c6142a816478" + + iex> UUID.uuid3(:dns, "my.domain.com", :hex) + "03bf0706b7e933b8aee5c6142a816478" + + iex> UUID.uuid3(:dns, "my.domain.com", :urn) + "urn:uuid:03bf0706-b7e9-33b8-aee5-c6142a816478" + + iex> UUID.uuid3(:dns, "my.domain.com", :raw) + <<3, 191, 7, 6, 183, 233, 51, 184, 174, 229, 198, 20, 42, 129, 100, 120>> + + iex> UUID.uuid3("cdfdaf44-ee35-11e3-846b-14109ff1a304", "my.domain.com") + "8808f33a-3e11-3708-919e-15fba88908db" + + iex> UUID.uuid3(:dns, "my.domain.com", :slug) + "A78HBrfpM7iu5cYUKoFkeA" + ``` + + """ + def uuid3(namespace_or_uuid, name, format \\ :default) + def uuid3(:dns, <>, format) do + namebased_uuid(:md5, <<0x6ba7b8109dad11d180b400c04fd430c8::128, name::binary>>) + |> uuid_to_string(format) + end + def uuid3(:url, <>, format) do + namebased_uuid(:md5, <<0x6ba7b8119dad11d180b400c04fd430c8::128, name::binary>>) + |> uuid_to_string(format) + end + def uuid3(:oid, <>, format) do + namebased_uuid(:md5, <<0x6ba7b8129dad11d180b400c04fd430c8::128, name::binary>>) + |> uuid_to_string(format) + end + def uuid3(:x500, <>, format) do + namebased_uuid(:md5, <<0x6ba7b8149dad11d180b400c04fd430c8::128, name::binary>>) + |> uuid_to_string(format) + end + def uuid3(:nil, <>, format) do + namebased_uuid(:md5, <<0::128, name::binary>>) + |> uuid_to_string(format) + end + def uuid3(<>, <>, format) do + {_type, <>} = uuid_string_to_hex_pair(uuid) + namebased_uuid(:md5, <>) + |> uuid_to_string(format) + end + def uuid3(_, _, _) do + raise ArgumentError, message: + "Invalid argument; Expected: :dns|:url|:oid|:x500|:nil OR String, String" + end + + @doc """ + Generate a new UUID v4. This version uses pseudo-random bytes generated by + the `crypto` module. + + ## Examples + + ```elixir + iex> UUID.uuid4() + "fb49a0ec-d60c-4d20-9264-3b4cfe272106" + + iex> UUID.uuid4(:default) + "fb49a0ec-d60c-4d20-9264-3b4cfe272106" + + iex> UUID.uuid4(:hex) + "fb49a0ecd60c4d2092643b4cfe272106" + + iex> UUID.uuid4(:urn) + "urn:uuid:fb49a0ec-d60c-4d20-9264-3b4cfe272106" + + iex> UUID.uuid4(:raw) + <<251, 73, 160, 236, 214, 12, 77, 32, 146, 100, 59, 76, 254, 39, 33, 6>> + + iex> UUID.uuid4(:slug) + "-0mg7NYMTSCSZDtM_ichBg" + ``` + + """ + def uuid4(), do: uuid4(:default) + + def uuid4(:strong), do: uuid4(:default) # For backwards compatibility. + def uuid4(:weak), do: uuid4(:default) # For backwards compatibility. + def uuid4(format) do + <> = :crypto.strong_rand_bytes(16) + <> + |> uuid_to_string(format) + end + + @doc """ + Generate a new UUID v5. This version uses an SHA1 hash of fixed value (chosen + based on a namespace atom - see Appendix C of + [RFC 4122](http://www.ietf.org/rfc/rfc4122.txt) and a name value. Can also be + given an existing UUID String instead of a namespace atom. + + Accepted arguments are: `:dns`|`:url`|`:oid`|`:x500`|`:nil` OR uuid, String + + ## Examples + + ```elixir + iex> UUID.uuid5(:dns, "my.domain.com") + "016c25fd-70e0-56fe-9d1a-56e80fa20b82" + + iex> UUID.uuid5(:dns, "my.domain.com", :default) + "016c25fd-70e0-56fe-9d1a-56e80fa20b82" + + iex> UUID.uuid5(:dns, "my.domain.com", :hex) + "016c25fd70e056fe9d1a56e80fa20b82" + + iex> UUID.uuid5(:dns, "my.domain.com", :urn) + "urn:uuid:016c25fd-70e0-56fe-9d1a-56e80fa20b82" + + iex> UUID.uuid5(:dns, "my.domain.com", :raw) + <<1, 108, 37, 253, 112, 224, 86, 254, 157, 26, 86, 232, 15, 162, 11, 130>> + + iex> UUID.uuid5("fb49a0ec-d60c-4d20-9264-3b4cfe272106", "my.domain.com") + "822cab19-df58-5eb4-98b5-c96c15c76d32" + + iex> UUID.uuid5("fb49a0ec-d60c-4d20-9264-3b4cfe272106", "my.domain.com", :slug) + "giyrGd9YXrSYtclsFcdtMg" + ``` + + """ + def uuid5(namespace_or_uuid, name, format \\ :default) + def uuid5(:dns, <>, format) do + namebased_uuid(:sha1, <<0x6ba7b8109dad11d180b400c04fd430c8::128, name::binary>>) + |> uuid_to_string(format) + end + def uuid5(:url, <>, format) do + namebased_uuid(:sha1, <<0x6ba7b8119dad11d180b400c04fd430c8::128, name::binary>>) + |> uuid_to_string(format) + end + def uuid5(:oid, <>, format) do + namebased_uuid(:sha1, <<0x6ba7b8129dad11d180b400c04fd430c8::128, name::binary>>) + |> uuid_to_string(format) + end + def uuid5(:x500, <>, format) do + namebased_uuid(:sha1, <<0x6ba7b8149dad11d180b400c04fd430c8::128, name::binary>>) + |> uuid_to_string(format) + end + def uuid5(:nil, <>, format) do + namebased_uuid(:sha1, <<0::128, name::binary>>) + |> uuid_to_string(format) + end + def uuid5(<>, <>, format) do + {_type, <>} = uuid_string_to_hex_pair(uuid) + namebased_uuid(:sha1, <>) + |> uuid_to_string(format) + end + def uuid5(_, _, _) do + raise ArgumentError, message: + "Invalid argument; Expected: :dns|:url|:oid|:x500|:nil OR String, String" + end + + # + # Internal utility functions. + # + + # Convert UUID bytes to String. + defp uuid_to_string(<<_::128>> = u, :default) do + uuid_to_string_default(u) + end + defp uuid_to_string(<<_::128>> = u, :hex) do + IO.iodata_to_binary(for <>, do: e(part)) + end + defp uuid_to_string(<<_::128>> = u, :urn) do + @urn <> uuid_to_string(u, :default) + end + defp uuid_to_string(<<_::128>> = u, :raw) do + u + end + defp uuid_to_string(<<_::128>> = u, :slug) do + Base.url_encode64(u, [padding: false]) + end + defp uuid_to_string(_u, format) when format in [:default, :hex, :urn, :slug] do + raise ArgumentError, message: + "Invalid binary data; Expected: <>" + end + defp uuid_to_string(_u, format) do + raise ArgumentError, message: + "Invalid format #{format}; Expected: :default|:hex|:urn|:slug" + end + + defp uuid_to_string_default(<< + a1::4, a2::4, a3::4, a4::4, + a5::4, a6::4, a7::4, a8::4, + b1::4, b2::4, b3::4, b4::4, + c1::4, c2::4, c3::4, c4::4, + d1::4, d2::4, d3::4, d4::4, + e1::4, e2::4, e3::4, e4::4, + e5::4, e6::4, e7::4, e8::4, + e9::4, e10::4, e11::4, e12::4 >>) do + << e(a1), e(a2), e(a3), e(a4), e(a5), e(a6), e(a7), e(a8), ?-, + e(b1), e(b2), e(b3), e(b4), ?-, + e(c1), e(c2), e(c3), e(c4), ?-, + e(d1), e(d2), e(d3), e(d4), ?-, + e(e1), e(e2), e(e3), e(e4), e(e5), e(e6), e(e7), e(e8), e(e9), e(e10), e(e11), e(e12) >> + end + + @compile {:inline, e: 1} + + defp e(0), do: ?0 + defp e(1), do: ?1 + defp e(2), do: ?2 + defp e(3), do: ?3 + defp e(4), do: ?4 + defp e(5), do: ?5 + defp e(6), do: ?6 + defp e(7), do: ?7 + defp e(8), do: ?8 + defp e(9), do: ?9 + defp e(10), do: ?a + defp e(11), do: ?b + defp e(12), do: ?c + defp e(13), do: ?d + defp e(14), do: ?e + defp e(15), do: ?f + + # Extract the type (:default etc) and pure byte value from a UUID String. + defp uuid_string_to_hex_pair(<<_::128>> = uuid) do + {:raw, uuid} + end + defp uuid_string_to_hex_pair(<>) do + uuid = String.downcase(uuid_in) + {type, hex_str} = case uuid do + <> -> + {:default, <>} + <> -> + {:hex, <>} + <<@urn, u0::64, ?-, u1::32, ?-, u2::32, ?-, u3::32, ?-, u4::96>> -> + {:urn, <>} + _ -> + case uuid_in do + _ when byte_size(uuid_in) == 22 -> + case Base.url_decode64(uuid_in <> "==") do + {:ok, decoded} -> {:slug, Base.encode16(decoded)} + _ -> raise ArgumentError, message: "Invalid argument; Not a valid UUID: #{uuid}" + end + _ -> raise ArgumentError, message: "Invalid argument; Not a valid UUID: #{uuid}" + end + end + + try do + {type, hex_str_to_binary(hex_str)} + catch + _, _ -> + raise ArgumentError, message: + "Invalid argument; Not a valid UUID: #{uuid}" + end + end + + # Get unix epoch as a 60-bit timestamp. + defp uuid1_time() do + {mega_sec, sec, micro_sec} = :os.timestamp() + epoch = (mega_sec * 1_000_000_000_000 + sec * 1_000_000 + micro_sec) + timestamp = @nanosec_intervals_offset + @nanosec_intervals_factor * epoch + <> + end + + # Generate random clock sequence. + defp uuid1_clockseq() do + <> = :crypto.strong_rand_bytes(2) + <> + end + + # Get local IEEE 802 (MAC) address, or a random node id if it can't be found. + defp uuid1_node() do + {:ok, ifs0} = :inet.getifaddrs() + uuid1_node(ifs0) + end + + defp uuid1_node([{_if_name, if_config} | rest]) do + case :lists.keyfind(:hwaddr, 1, if_config) do + :false -> + uuid1_node(rest) + {:hwaddr, hw_addr} -> + if length(hw_addr) != 6 or Enum.all?(hw_addr, fn(n) -> n == 0 end) do + uuid1_node(rest) + else + :erlang.list_to_binary(hw_addr) + end + end + end + defp uuid1_node(_) do + <> = :crypto.strong_rand_bytes(6) + <> + end + + # Generate a hash of the given data. + defp namebased_uuid(:md5, data) do + md5 = :crypto.hash(:md5, data) + compose_namebased_uuid(@uuid_v3, md5) + end + defp namebased_uuid(:sha1, data) do + <> = :crypto.hash(:sha, data) + compose_namebased_uuid(@uuid_v5, <>) + end + + # Format the given hash as a UUID. + defp compose_namebased_uuid(version, hash) do + <> = hash + <> + end + + # Identify the UUID variant according to section 4.1.1 of RFC 4122. + defp variant(<<1, 1, 1>>) do + :reserved_future + end + defp variant(<<1, 1, _v>>) do + :reserved_microsoft + end + defp variant(<<1, 0, _v>>) do + :rfc4122 + end + defp variant(<<0, _v::2-binary>>) do + :reserved_ncs + end + defp variant(_) do + raise ArgumentError, message: "Invalid argument; Not valid variant bits" + end + + defp hex_str_to_binary(<< a1, a2, a3, a4, a5, a6, a7, a8, + b1, b2, b3, b4, + c1, c2, c3, c4, + d1, d2, d3, d4, + e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11, e12 >>) do + << d(a1)::4, d(a2)::4, d(a3)::4, d(a4)::4, + d(a5)::4, d(a6)::4, d(a7)::4, d(a8)::4, + d(b1)::4, d(b2)::4, d(b3)::4, d(b4)::4, + d(c1)::4, d(c2)::4, d(c3)::4, d(c4)::4, + d(d1)::4, d(d2)::4, d(d3)::4, d(d4)::4, + d(e1)::4, d(e2)::4, d(e3)::4, d(e4)::4, + d(e5)::4, d(e6)::4, d(e7)::4, d(e8)::4, + d(e9)::4, d(e10)::4, d(e11)::4, d(e12)::4 >> + end + + @compile {:inline, d: 1} + + defp d(?0), do: 0 + defp d(?1), do: 1 + defp d(?2), do: 2 + defp d(?3), do: 3 + defp d(?4), do: 4 + defp d(?5), do: 5 + defp d(?6), do: 6 + defp d(?7), do: 7 + defp d(?8), do: 8 + defp d(?9), do: 9 + defp d(?A), do: 10 + defp d(?B), do: 11 + defp d(?C), do: 12 + defp d(?D), do: 13 + defp d(?E), do: 14 + defp d(?F), do: 15 + defp d(?a), do: 10 + defp d(?b), do: 11 + defp d(?c), do: 12 + defp d(?d), do: 13 + defp d(?e), do: 14 + defp d(?f), do: 15 + defp d(_), do: throw(:error) +end