add an endpoint to receive the chunk of the file

add chunk.sh shell script that returns cut of the file
based on starting line, number of lines to read
and direction (up or down)

add ChunkExtractor module that sanitizes params and handles response of
the chunk.sh script

add a json endpoint to be able to request chunks

Author: RudolfMan

lib/diff/hex/chunk_extractor.ex

@@ -0,0 +1,112 @@
+defmodule Diff.Hex.ChunkExtractor do
+  @enforce_keys [:file_path, :from_line, :lines_to_read, :direction]
+  defstruct Enum.map(@enforce_keys, &{&1, nil}) ++
+              [
+                errors: [],
+                raw: nil,
+                parsed: nil
+              ]
+
+  def run(params) do
+    __MODULE__
+    |> struct!(params)
+    |> parse_integers([:from_line, :lines_to_read])
+    |> validate_direction()
+    |> system_read_raw_chunk()
+    |> parse_chunk()
+    |> remove_trailing_newline()
+  end
+
+  defp parse_integers(chunk, fields) do
+    Enum.reduce(fields, chunk, &parse_integer/2)
+  end
+
+  defp parse_integer(field, chunk) do
+    value = chunk |> Map.get(field) |> parse_value()
+
+    case value do
+      :error -> %{chunk | errors: {:parse_integer, "#{field} must be a number"}}
+      integer -> Map.put(chunk, field, integer)
+    end
+  end
+
+  defp parse_value(number) when is_integer(number), do: number
+
+  defp parse_value(number) when is_binary(number) do
+    with {int, _} <- Integer.parse(number), do: int
+  end
+
+  defp validate_direction(%{direction: direction} = chunk) when direction in ["up", "down"] do
+    chunk
+  end
+
+  defp validate_direction(chunk) do
+    error = {:direction, "direction must be either \"up\" or \"down\""}
+    %{chunk | errors: [error | chunk.errors]}
+  end
+
+  defp system_read_raw_chunk(%{errors: [_ | _]} = chunk), do: chunk
+
+  defp system_read_raw_chunk(chunk) do
+    chunk_sh = Application.app_dir(:diff, ["priv", "chunk.sh"])
+
+    path = chunk.file_path
+    from_line = to_string(chunk.from_line)
+    lines_to_read = to_string(chunk.lines_to_read)
+    direction = chunk.direction
+
+    case System.cmd(chunk_sh, [path, from_line, lines_to_read, direction], stderr_to_stdout: true) do
+      {raw_chunk, 0} ->
+        %{chunk | raw: raw_chunk}
+
+      {error, code} ->
+        error = {:system, "System command exited with a non-zero status #{code}: #{error}"}
+        %{chunk | errors: [error | chunk.errors]}
+    end
+  end
+
+  defp parse_chunk(%{errors: [_ | _]} = chunk), do: chunk
+
+  defp parse_chunk(chunk) do
+    parsed =
+      chunk.raw
+      |> String.split("\n")
+      |> Enum.map(fn line -> %{line_text: line} end)
+
+    set_line_numbers(%{chunk | parsed: parsed})
+  end
+
+  defp set_line_numbers(%{direction: "down"} = chunk) do
+    %{chunk | parsed: parsed_with_line_numbers(chunk.parsed, chunk.from_line)}
+  end
+
+  defp set_line_numbers(%{direction: "up"} = chunk) do
+    offset = chunk.from_line - length(chunk.parsed) + 1
+
+    %{chunk | parsed: parsed_with_line_numbers(chunk.parsed, offset)}
+  end
+
+  defp parsed_with_line_numbers(parsed_chunk, starting_number) when is_binary(starting_number) do
+    parsed_with_line_numbers(parsed_chunk, starting_number)
+  end
+
+  defp parsed_with_line_numbers(parsed_chunk, starting_number) do
+    parsed_chunk
+    |> Enum.with_index(starting_number)
+    |> Enum.map(fn {line, line_number} -> Map.put_new(line, :line_number, line_number) end)
+  end
+
+  defp remove_trailing_newline(%{errors: [_ | _]} = chunk), do: {:error, chunk}
+
+  defp remove_trailing_newline(chunk) do
+    [trailing_line | reversed_tail] = Enum.reverse(chunk.parsed)
+
+    chunk =
+      case trailing_line do
+        %{line_text: ""} -> %{chunk | parsed: Enum.reverse(reversed_tail)}
+        %{line_text: _text} -> chunk
+      end
+
+    {:ok, chunk}
+  end
+end

lib/diff/hex/hex.ex

@@ -38,10 +38,11 @@ defmodule Diff.Hex do
     end
   end
 
-  def unpack_tarball(tarball, path) when is_binary(path) do
+  def unpack_tarball(tarball, file_list \\ [], path) when is_binary(path) do
     path = to_charlist(path)
+    file_list = Enum.map(file_list, &to_charlist/1)
 
-    with {:ok, _} <- :hex_tarball.unpack(tarball, path) do
+    with {:ok, _} <- :hex_tarball.unpack(tarball, file_list, path) do
       :ok
     end
   end
@@ -100,6 +101,35 @@ defmodule Diff.Hex do
     end
   end
 
+  def get_chunk(package, version, file_name, from_line, direction) do
+    path = tmp_path("package-#{package}-#{version}-")
+
+    chunk_extractor_params = %{
+      file_path: Path.join(path, file_name),
+      from_line: from_line,
+      direction: direction,
+      lines_to_read: 20
+    }
+
+    try do
+      with {:ok, tarball} <- get_tarball(package, version),
+           :ok <- unpack_tarball(tarball, [file_name], path),
+           {:ok, %{parsed: parsed_chunk}} <- Diff.Hex.ChunkExtractor.run(chunk_extractor_params) do
+        {:ok, parsed_chunk}
+      else
+        {:error, %Diff.Hex.ChunkExtractor{errors: errors} = chunk} ->
+          Logger.error(inspect(errors))
+          {:error, chunk}
+
+        error ->
+          Logger.error(inspect(error))
+          {:error, error}
+      end
+    after
+      File.rm_rf(path)
+    end
+  end
+
   defp git_diff(path_from, path_to, path_out) do
     case System.cmd("git", [
            "-c",

lib/diff_web/controllers/page_controller.ex

@@ -28,6 +28,32 @@ defmodule DiffWeb.PageController do
     end
   end
 
+  def expand_context(conn, %{"file_name" => file_name} = params) do
+    %{
+      "version" => version,
+      "package" => package,
+      "from_line" => from_line,
+      "direction" => direction
+    } = params
+
+    case parse_version(version) do
+      {:ok, version} ->
+        version = to_string(version)
+        do_expand_context(conn, package, version, file_name, from_line, direction)
+
+      :error ->
+        conn
+        |> put_status(400)
+        |> json(%{error: "Bad Request"})
+    end
+  end
+
+  def expand_context(conn, _params) do
+    conn
+    |> put_status(400)
+    |> json(%{error: "missing query parameter: file_name"})
+  end
+
   defp maybe_cached_diff(conn, _package, version, version) do
     render_error(conn, 400)
   end
@@ -70,6 +96,18 @@ defmodule DiffWeb.PageController do
     end
   end
 
+  defp do_expand_context(conn, package, version, file_name, from_line, direction) do
+    case Diff.Hex.get_chunk(package, version, file_name, from_line, direction) do
+      {:ok, chunk} ->
+        json(conn, %{chunk: chunk})
+
+      {:error, %{errors: errors}} ->
+        conn
+        |> put_status(400)
+        |> json(%{errors: Enum.into(errors, %{})})
+    end
+  end
+
   defp do_diff(conn, package, from, to) do
     case Diff.Hex.diff(package, from, to) do
       {:ok, stream} ->

lib/diff_web/router.ex

@@ -15,6 +15,7 @@ defmodule DiffWeb.Router do
     pipe_through :browser
 
     live "/", SearchLiveView
+    get "/diff/:package/:version/expand/:from_line/:direction", PageController, :expand_context
     get "/diff/:package/:versions", PageController, :diff
     get "/diffs", PageController, :diffs
   end

mix.exs

@@ -43,7 +43,7 @@ defmodule Diff.MixProject do
       {:jason, "~> 1.0"},
       {:plug_cowboy, "~> 2.0"},
       {:phoenix_live_view, "~> 0.6"},
-      {:hex_core, "~> 0.6.1"},
+      {:hex_core, github: "RudolfMan/hex_core", branch: "unpack-list-of-files"},
       {:rollbax, "~> 0.11.0"},
       {:logster, "~> 1.0"},
       {:git_diff, github: "hexpm/git_diff", branch: "emj/stream"},

mix.lock

@@ -9,7 +9,7 @@
   "git_diff": {:git, "https://github.com/hexpm/git_diff.git", "453ed1933e87e47c2debef1b5b77a0dc4a1c3fb7", [branch: "emj/stream"]},
   "goth": {:hex, :goth, "1.2.0", "92d6d926065a72a7e0da8818cc3a133229b56edf378022c00d9886c4125ce769", [:mix], [{:httpoison, "~> 0.11 or ~> 1.0", [hex: :httpoison, repo: "hexpm", optional: false]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:joken, "~> 2.0", [hex: :joken, repo: "hexpm", optional: false]}], "hexpm", "4974932ab3b782c99a6fdeb0b968ddd61436ef14de5862bd6bb0227386c63b26"},
   "hackney": {:hex, :hackney, "1.15.2", "07e33c794f8f8964ee86cebec1a8ed88db5070e52e904b8f12209773c1036085", [:rebar3], [{:certifi, "2.5.1", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "6.0.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.5", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm", "e0100f8ef7d1124222c11ad362c857d3df7cb5f4204054f9f0f4a728666591fc"},
-  "hex_core": {:hex, :hex_core, "0.6.6", "a253f0abd41e10bc33cb73cdb1261a33a710559243ef3d6ca4d7ade0462a3f2c", [:rebar3], [], "hexpm", "f1aa2bf3a27520055d94e7c03880314f77c46ab19e1b280e8b8e46981ae92286"},
+  "hex_core": {:git, "https://github.com/RudolfMan/hex_core.git", "8295d4b67f7d461a991fa34507b0e2c74a3201ea", [branch: "unpack-list-of-files"]},
   "html_entities": {:hex, :html_entities, "0.5.0", "40f5c5b9cbe23073b48a4e69c67b6c11974f623a76165e2b92d098c0e88ccb1d", [:mix], [], "hexpm", "8e9186e1873bea1067895f6a542b59df6c9fcf3b516ba272eeff3ea0c7b755cd"},
   "httpoison": {:hex, :httpoison, "1.6.2", "ace7c8d3a361cebccbed19c283c349b3d26991eff73a1eaaa8abae2e3c8089b6", [:mix], [{:hackney, "~> 1.15 and >= 1.15.2", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "aa2c74bd271af34239a3948779612f87df2422c2fdcfdbcec28d9c105f0773fe"},
   "idna": {:hex, :idna, "6.0.0", "689c46cbcdf3524c44d5f3dde8001f364cd7608a99556d8fbd8239a5798d4c10", [:rebar3], [{:unicode_util_compat, "0.4.1", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "4bdd305eb64e18b0273864920695cb18d7a2021f31a11b9c5fbcd9a253f936e2"},

priv/chunk.sh

@@ -0,0 +1,21 @@
+#!/bin/sh
+
+file=$1
+from_line=$2
+lines_to_read=$3
+direction=$4
+
+if [ $direction = "down" ]
+then
+  TAIL="$(tail -n+$from_line $file)"
+  TMP_STATUS=$?
+  printf '%s' "$TAIL" | head -n$lines_to_read
+  FINAL_STATUS=$?
+else
+  HEAD="$(head -n$from_line $file)"
+  TMP_STATUS=$?
+  printf '%s' "$HEAD" | tail -n$lines_to_read
+  FINAL_STATUS=$?
+fi
+
+[ $TMP_STATUS -gt 0 ] && exit $TMP_STATUS || exit $FINAL_STATUS

test/diff/hex/chunk_extractor_test.exs

@@ -0,0 +1,88 @@
+defmodule Diff.Hex.ChunkExtractorTest do
+  use ExUnit.Case, async: true
+
+  alias Diff.Hex.ChunkExtractor
+
+  setup do
+    content = """
+    foo 1
+    bar 2
+    baz 3
+    baf 4
+    """
+
+    path = System.tmp_dir!() |> Path.join("test_file")
+    File.write!(path, content)
+
+    on_exit(fn -> File.rm!(path) end)
+
+    # some deafult params
+    %{params: %{file_path: path, lines_to_read: 2, from_line: 1, direction: "down"}}
+  end
+
+  describe "validates direction" do
+    test "down", %{params: params} do
+      {:ok, %{errors: errors}} = ChunkExtractor.run(%{params | direction: "down"})
+      assert [] = errors
+    end
+
+    test "up", %{params: params} do
+      {:ok, %{errors: errors}} = ChunkExtractor.run(%{params | direction: "up"})
+      assert [] = errors
+    end
+
+    test "error when direction is neither up nor down", %{params: params} do
+      {:error, %{errors: errors}} = ChunkExtractor.run(%{params | direction: "left"})
+      assert "direction must be either \"up\" or \"down\"" = Keyword.get(errors, :direction)
+    end
+  end
+
+  describe "reads raw chunk from the file_path" do
+    test "reads first 2 lines down", %{params: params} do
+      {:ok, %{raw: raw}} = ChunkExtractor.run(%{params | direction: "down"})
+      assert "foo 1\nbar 2\n" = raw
+    end
+
+    test "reads first 2 lines up", %{params: params} do
+      {:ok, %{raw: raw}} = ChunkExtractor.run(%{params | direction: "up", from_line: 2})
+      assert "foo 1\nbar 2" = raw
+    end
+
+    test "error when file doesn't exist", %{params: params} do
+      {:error, %{errors: errors}} = ChunkExtractor.run(%{params | file_path: "non_existent"})
+      assert Keyword.get(errors, :system) =~ ~r/non_existent: No such file/
+    end
+
+    test "error when arguments are not valid", %{params: params} do
+      {:error, %{errors: errors}} = ChunkExtractor.run(%{params | from_line: -1})
+      assert Keyword.get(errors, :system) =~ ~r/illegal offset/
+    end
+
+    test "reads 2 lines up from the middle", %{params: params} do
+      {:ok, %{raw: raw}} = ChunkExtractor.run(%{params | direction: "up", from_line: 3})
+      assert "bar 2\nbaz 3" = raw
+    end
+
+    test "reads 2 lines down from the middle", %{params: params} do
+      {:ok, %{raw: raw}} = ChunkExtractor.run(%{params | direction: "down", from_line: 2})
+      assert "bar 2\nbaz 3\n" = raw
+    end
+  end
+
+  describe "parse_chunk" do
+    test "parses raw chunk into list of structs", %{params: params} do
+      {:ok, %{parsed: actual}} = ChunkExtractor.run(params)
+      assert [%{line_text: "foo 1"}, %{line_text: "bar 2"}] = actual
+    end
+
+    test "sets line_numbers when direction is down", %{params: params} do
+      {:ok, %{parsed: actual}} = ChunkExtractor.run(%{params | direction: "down", from_line: 2})
+      assert [%{line_number: 2}, %{line_number: 3}] = actual
+    end
+
+    test "sets line_numbers when direction is up", %{params: params} do
+      {:ok, %{parsed: actual}} = ChunkExtractor.run(%{params | direction: "up", from_line: 3})
+      assert [%{line_number: 2}, %{line_number: 3}] = actual
+    end
+  end
+end