Merge branch 'ei' into develop

lib/gen_magic/helpers.ex

@@ -5,7 +5,9 @@ defmodule GenMagic.Helpers do
 
   alias GenMagic.Result
   alias GenMagic.Server
-  @spec perform_once(Path.t(), [Server.option()]) :: {:ok, Result.t()} | {:error, term()}
+
+  @spec perform_once(Path.t() | {:bytes, binary}, [Server.option()]) ::
+          {:ok, Result.t()} | {:error, term()}
 
   @doc """
   Runs a one-shot process without supervision.

lib/gen_magic/server.ex

@@ -81,7 +81,8 @@ defmodule GenMagic.Server do
 
   @spec child_spec([option()]) :: Supervisor.child_spec()
   @spec start_link([option()]) :: :gen_statem.start_ret()
-  @spec perform(t(), Path.t(), timeout()) :: {:ok, Result.t()} | {:error, term() | String.t()}
+  @spec perform(t(), Path.t() | {:bytes, binary()}, timeout()) ::
+          {:ok, Result.t()} | {:error, term() | String.t()}
   @spec status(t(), timeout()) :: {:ok, Status.t()} | {:error, term()}
   @spec stop(t(), term(), timeout()) :: :ok
 
@@ -185,10 +186,9 @@ defmodule GenMagic.Server do
   end
 
   @doc false
-  def starting(:info, {port, {:data, binary}}, %{port: port} = data) do
-    case :erlang.binary_to_term(binary) do
-      :ready ->
-        {:next_state, :available, data}
+  def starting(:info, {port, {:data, ready}}, %{port: port} = data) do
+    case :erlang.binary_to_term(ready) do
+      :ready -> {:next_state, :available, data}
     end
   end
 
@@ -198,6 +198,7 @@ defmodule GenMagic.Server do
         1 -> :no_database
         2 -> :no_argument
         3 -> :missing_database
+        code -> {:unexpected_error, code}
       end
 
     {:stop, {:error, error}, data}
@@ -243,12 +244,10 @@ defmodule GenMagic.Server do
   end
 
   @doc false
-  def processing(:info, {port, {:data, response}}, %{port: port} = data) do
-    {_, from, _} = data.request
-    data = %{data | request: nil}
+  def processing(:info, {port, {:data, response}}, %{port: port, request: {_, from, _}} = data) do
     response = {:reply, from, handle_response(response)}
     next_state = (data.cycles >= data.recycle_threshold && :recycling) || :available
-    {:next_state, next_state, data, [response, :hibernate]}
+    {:next_state, next_state, %{data | request: nil}, [response, :hibernate]}
   end
 
   @doc false
@@ -279,11 +278,10 @@ defmodule GenMagic.Server do
   @errnos %{
     2 => :enoent,
     13 => :eaccess,
-    21 => :eisdir,
     20 => :enotdir,
     12 => :enomem,
     24 => :emfile,
-    36 => :enametoolong,
+    36 => :enametoolong
   }
   @errno Map.keys(@errnos)
 
@@ -292,6 +290,7 @@ defmodule GenMagic.Server do
       {:ok, {mime_type, encoding, content}} -> {:ok, Result.build(mime_type, encoding, content)}
       {:error, {errno, _}} when errno in @errno -> {:error, @errnos[errno]}
       {:error, {errno, string}} -> {:error, "#{errno}: #{string}"}
+      {:error, _} = error -> error
     end
   end
 

src/apprentice.c

@@ -2,7 +2,7 @@
 // The Sorcerer’s Apprentice
 //
 // To use this program, compile it with dynamically linked libmagic, as mirrored
-// at https://github.com/threatstack/libmagic. You may install it with apt-get,
+// at https://github.com/file/file. You may install it with apt-get,
 // yum or brew. Refer to the Makefile for further reference.
 //
 // This program is designed to run interactively as a backend daemon to the
@@ -12,18 +12,34 @@
 //
 // Where each argument either refers to a compiled or uncompiled magic database,
 // or the default database. They will be loaded in the sequence that they were
-// specified. Note that you must specify at least one database. Erlang Term
+// specified. Note that you must specify at least one database.
 //
-// -- main: send atom ready
-//          enter loop
+// Communication is done over STDIN/STDOUT as binary packets of 2 bytes length
+// plus X bytes payload, where the payload is an erlang term encoded with
+// :erlang.term_to_binary/1 and decoded with :erlang.binary_to_term/1.
 //
-// -- while
-//      get {:file, path} -> process_file   -> ok | error
-//          {:bytes, path} -> process_bytes -> ok | error
-//            ok: {:ok, {type, encoding, name}}
-//            error: {:error, :badarg} | {:error, {errno, String.t()}}
-//          {:stop, _} -> exit(ERROR_OK)    -> exit 0
+// Once the program is ready, it sends the `:ready` atom. The startup can fail
+// for multiples reasons, and the program will exit accordingly:
+// - 1: No database
+// - 2: Missing/Bad argument
+// - 3: Missing database
 //
+// Commands are sent to the program STDIN as an erlang term of `{Operation,
+// Argument}`, and response of `{:ok | :error, Response}`.
+//
+// Invalid packets will cause the program to exit (exit code 4). This will
+// happen if your Erlang Term format doesn't match the version the program has
+// been compiled with, or if you send a command too huge.
+//
+// The program may exit with error codes 5 or 255 if something went wrong (such
+// as error allocating terms, or if stdin is lost).
+//
+// Commands:
+// {:file, path :: String.t()} :: {:ok, {type, encoding, name}} | {:error,
+// :badarg} | {:error, {errno :: integer(), String.t()}}
+// {:bytes, binary()} :: same as :file
+// {:stop, reason :: atom()} :: exit 0
+
 #include <ei.h>
 #include <errno.h>
 #include <getopt.h>
@@ -36,8 +52,6 @@
 #include <sys/stat.h>
 #include <sys/types.h>
 #include <unistd.h>
-#define USAGE "[--database-file <path/to/magic.mgc> | --database-default, ...]"
-#define DELIMITER "\t"
 
 #define ERROR_OK 0
 #define ERROR_NO_DATABASE 1
@@ -46,11 +60,10 @@
 #define ERROR_BAD_TERM 4
 #define ERROR_EI 5
 
-#define ANSI_INFO "\x1b[37m"   // gray
-#define ANSI_OK "\x1b[32m"     // green
-#define ANSI_ERROR "\x1b[31m"  // red
-#define ANSI_IGNORE "\x1b[90m" // red
-#define ANSI_RESET "\x1b[0m"
+// We use a bigger than possible valid command length (around 4111 bytes) to
+// allow more precise errors when using too long paths.
+#define COMMAND_LEN 8000
+#define COMMAND_BUFFER_SIZE COMMAND_LEN + 1
 
 #define MAGIC_FLAGS_COMMON (MAGIC_CHECK | MAGIC_ERROR)
 magic_t magic_setup(int flags);
@@ -62,14 +75,14 @@ void setup_options(int argc, char **argv);
 void setup_options_file(char *optarg);
 void setup_options_default();
 void setup_system();
-int process_command(byte *buf);
-void process_line(char *line);
+int process_command(uint16_t len, byte *buf);
 void process_file(char *path, ei_x_buff *result);
 void process_bytes(char *bytes, int size, ei_x_buff *result);
 size_t read_cmd(byte *buf);
 size_t write_cmd(byte *buf, size_t len);
 void error(ei_x_buff *result, const char *error);
 void handle_magic_error(magic_t handle, int errn, ei_x_buff *result);
+void fdseek(uint16_t count);
 
 struct magic_file {
   struct magic_file *prev;
@@ -95,29 +108,35 @@ int main(int argc, char **argv) {
   if (ei_x_free(&ok_buf) != 0)
     exit(ERROR_EI);
 
-  byte buf[4112];
-  while (read_cmd(buf) > 0) {
-    process_command(buf);
+  byte buf[COMMAND_BUFFER_SIZE];
+  uint16_t len;
+  while ((len = read_cmd(buf)) > 0) {
+    process_command(len, buf);
   }
 
   return 255;
 }
 
-int process_command(byte *buf) {
+int process_command(uint16_t len, byte *buf) {
   ei_x_buff result;
   char atom[128];
   int index, version, arity, termtype, termsize;
   index = 0;
 
-  if (ei_decode_version(buf, &index, &version) != 0) {
-    exit(ERROR_BAD_TERM);
-  }
-
   // Initialize result
   if (ei_x_new_with_version(&result) || ei_x_encode_tuple_header(&result, 2)) {
     exit(ERROR_EI);
   }
 
+  if (len >= COMMAND_LEN) {
+    error(&result, "badarg");
+    return 1;
+  }
+
+  if (ei_decode_version(buf, &index, &version) != 0) {
+    exit(ERROR_BAD_TERM);
+  }
+
   if (ei_decode_tuple_header(buf, &index, &arity) != 0) {
     error(&result, "badarg");
     return 1;
@@ -137,11 +156,16 @@ int process_command(byte *buf) {
     char path[4097];
     ei_get_type(buf, &index, &termtype, &termsize);
 
-    if (termtype == ERL_BINARY_EXT && termsize < 4096) {
-      long bin_length;
-      ei_decode_binary(buf, &index, path, &bin_length);
-      path[termsize] = '\0';
-      process_file(path, &result);
+    if (termtype == ERL_BINARY_EXT) {
+      if (termsize < 4096) {
+        long bin_length;
+        ei_decode_binary(buf, &index, path, &bin_length);
+        path[termsize] = '\0';
+        process_file(path, &result);
+      } else {
+        error(&result, "enametoolong");
+        return 1;
+      }
     } else {
       error(&result, "badarg");
       return 1;
@@ -378,9 +402,15 @@ size_t read_cmd(byte *buf) {
   }
   uint16_t len16 = *(uint16_t *)buf;
   len16 = ntohs(len16);
-  if (len16 > 4111) {
-    exit(ERROR_BAD_TERM);
+
+  // Buffer isn't large enough: just return possible len, without reading.
+  // Up to the caller of verifying the size again and return an error.
+  // buf left unchanged, stdin emptied of X bytes.
+  if (len16 > COMMAND_LEN) {
+    fdseek(len16);
+    return len16;
   }
+
   return read_exact(buf, len16);
 }
 
@@ -404,3 +434,11 @@ void error(ei_x_buff *result, const char *error) {
   if (ei_x_free(result) != 0)
     exit(ERROR_EI);
 }
+
+void fdseek(uint16_t count) {
+  int i = 0;
+  while (i < count) {
+    getchar();
+    i += 1;
+  }
+}

test/gen_magic/apprentice_test.exs

 defmodule GenMagic.ApprenticeTest do
   use GenMagic.MagicCase
 
+  @tmp_path "/tmp/testgenmagicx"
+
   test "sends ready" do
     port = Port.open(GenMagic.Config.get_port_name(), GenMagic.Config.get_port_options([]))
     assert_ready(port)
@@ -85,19 +87,39 @@ defmodule GenMagic.ApprenticeTest do
     end
 
     test "works with big file path", %{port: port} do
-      file = too_big() <> "/a"
-      File.mkdir_p!(too_big())
-      File.touch!(file)
-      on_exit(fn -> File.rm_rf!("/tmp/testmagicex/") end)
-      send(port, {self(), {:command, :erlang.term_to_binary({:file, file})}})
+      # Test with longest valid path.
+      {dir, bigfile} = too_big(@tmp_path, "/a")
+      File.mkdir_p!(dir)
+      File.touch!(bigfile)
+      on_exit(fn -> File.rm_rf!(@tmp_path) end)
+      send(port, {self(), {:command, :erlang.term_to_binary({:file, bigfile})}})
       assert_receive {^port, {:data, data}}
       assert {:ok, _} = :erlang.binary_to_term(data)
       refute_receive _
-      file = too_big() <> "/aaaaaaaaaa"
+
+      # This path should be long enough for buffers, but larger than a valid path name. Magic will return an errno 36.
+      file = @tmp_path <> String.duplicate("a", 256)
+      send(port, {self(), {:command, :erlang.term_to_binary({:file, file})}})
+      assert_receive {^port, {:data, data}}
+      assert {:error, {36, _}} = :erlang.binary_to_term(data)
+      refute_receive _
+      # Theses filename should be too big for the path buffer.
+      file = bigfile <> "aaaaaaaaaa"
+      send(port, {self(), {:command, :erlang.term_to_binary({:file, file})}})
+      assert_receive {^port, {:data, data}}
+      assert {:error, :enametoolong} = :erlang.binary_to_term(data)
+      refute_receive _
+      # This call should be larger than the COMMAND_BUFFER_SIZE. Ensure nothing bad happens!
+      file = String.duplicate(bigfile, 4)
       send(port, {self(), {:command, :erlang.term_to_binary({:file, file})}})
       assert_receive {^port, {:data, data}}
       assert {:error, :badarg} = :erlang.binary_to_term(data)
       refute_receive _
+      # We re-run a valid call to ensure the buffer/... haven't been corrupted in port land.
+      send(port, {self(), {:command, :erlang.term_to_binary({:file, bigfile})}})
+      assert_receive {^port, {:data, data}}
+      assert {:ok, _} = :erlang.binary_to_term(data)
+      refute_receive _
     end
   end
 
@@ -106,7 +128,20 @@ defmodule GenMagic.ApprenticeTest do
     assert :ready == :erlang.binary_to_term(data)
   end
 
-  def too_big do
-    "/tmp/testmagicex/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+  def too_big(path, filename, limit \\ 4095) do
+    last_len = byte_size(filename)
+    path_len = byte_size(path)
+    needed = limit - (last_len + path_len)
+    extra = make_too_big(needed, "")
+    {path <> extra, path <> extra <> filename}
+  end
+
+  def make_too_big(needed, acc) when needed <= 255 do
+    acc <> "/" <> String.duplicate("a", needed - 1)
+  end
+
+  def make_too_big(needed, acc) do
+    acc = acc <> "/" <> String.duplicate("a", 254)
+    make_too_big(needed - 255, acc)
   end
 end