From d3b02a418717612b459e9cc8d7484345ff0a3236 Mon Sep 17 00:00:00 2001 From: Rossukhon Leagmongkol Date: Mon, 30 Mar 2020 16:19:17 +0700 Subject: [PATCH 01/12] Add the crawler and migrate the db to store the result --- lib/google_crawler/application.ex | 6 +++--- lib/google_crawler/errors.ex | 4 ++++ lib/google_crawler/search/crawler.ex | 13 +++++++++++++ lib/google_crawler/search/keyword.ex | 7 +++++++ lib/google_crawler/search/scrapper.ex | 5 +++++ .../search/search_keyword_task.ex | 5 +++++ mix.exs | 5 ++++- mix.lock | 17 +++++++++++++++++ ...d_status_and_raw_html_result_to_keywords.exs | 10 ++++++++++ 9 files changed, 68 insertions(+), 4 deletions(-) create mode 100644 lib/google_crawler/search/crawler.ex create mode 100644 lib/google_crawler/search/scrapper.ex create mode 100644 lib/google_crawler/search/search_keyword_task.ex create mode 100644 priv/repo/migrations/20200331103751_add_status_and_raw_html_result_to_keywords.exs diff --git a/lib/google_crawler/application.ex b/lib/google_crawler/application.ex index 9f95610..332b486 100644 --- a/lib/google_crawler/application.ex +++ b/lib/google_crawler/application.ex @@ -11,9 +11,9 @@ defmodule GoogleCrawler.Application do # Start the Ecto repository GoogleCrawler.Repo, # Start the endpoint when the application starts - GoogleCrawlerWeb.Endpoint - # Starts a worker by calling: GoogleCrawler.Worker.start_link(arg) - # {GoogleCrawler.Worker, arg}, + GoogleCrawlerWeb.Endpoint, + # The supervisor for crawler background job + {Task.Supervisor, name: GoogleCrawler.TaskSupervisor} ] # See https://hexdocs.pm/elixir/Supervisor.html diff --git a/lib/google_crawler/errors.ex b/lib/google_crawler/errors.ex index 0521f3a..3f637c8 100644 --- a/lib/google_crawler/errors.ex +++ b/lib/google_crawler/errors.ex @@ -1,3 +1,7 @@ defmodule GoogleCrawler.Errors.FileNotSupportedError do defexception message: "File is not supported" end + +defmodule GoogleCrawler.Errors.FetchError do + defexception message: "Fails to fetch the keyword result" +end diff --git a/lib/google_crawler/search/crawler.ex b/lib/google_crawler/search/crawler.ex new file mode 100644 index 0000000..a276088 --- /dev/null +++ b/lib/google_crawler/search/crawler.ex @@ -0,0 +1,13 @@ +defmodule GoogleCrawler.Crawler do + @url "https://www.google.com/search?q=" + + def fetch(keyword) do + case HTTPoison.get(@url <> keyword) do + {:ok, %HTTPoison.Response{status_code: 200, body: body}} -> + body + + {:error, %HTTPoison.Error{reason: reason}} -> + raise GoogleCrawler.Errors.FetchError, message: reason + end + end +end diff --git a/lib/google_crawler/search/keyword.ex b/lib/google_crawler/search/keyword.ex index d87ca11..49bc007 100644 --- a/lib/google_crawler/search/keyword.ex +++ b/lib/google_crawler/search/keyword.ex @@ -1,9 +1,16 @@ +import EctoEnum + +defenum GoogleCrawler.Search.Keyword.Status, + in_queue: 0, in_progress: 1, completed: 2 + defmodule GoogleCrawler.Search.Keyword do use Ecto.Schema import Ecto.Changeset schema "keywords" do field :keyword, :string + field :status, GoogleCrawler.Search.Keyword.Status + field :raw_html_result, :string belongs_to :user, GoogleCrawler.Accounts.User diff --git a/lib/google_crawler/search/scrapper.ex b/lib/google_crawler/search/scrapper.ex new file mode 100644 index 0000000..582c4ff --- /dev/null +++ b/lib/google_crawler/search/scrapper.ex @@ -0,0 +1,5 @@ +defmodule GoogleCrawler.Scapper do + def scrap(html) do + + end +end diff --git a/lib/google_crawler/search/search_keyword_task.ex b/lib/google_crawler/search/search_keyword_task.ex new file mode 100644 index 0000000..c883899 --- /dev/null +++ b/lib/google_crawler/search/search_keyword_task.ex @@ -0,0 +1,5 @@ +defmodule GoogleCrawler.SearchKeywordTask do + def perform(%{keyword: keyword}) do + + end +end diff --git a/mix.exs b/mix.exs index 5bb4cfb..d8a21d0 100644 --- a/mix.exs +++ b/mix.exs @@ -45,7 +45,10 @@ defmodule GoogleCrawler.MixProject do {:plug_cowboy, "~> 2.0"}, {:bcrypt_elixir, "~> 2.0"}, {:faker_elixir_octopus, "~> 1.0.0", only: [:dev, :test]}, - {:csv, "~> 2.3"} + {:csv, "~> 2.3"}, + {:httpoison, "~> 1.6"}, + {:ecto_enum, "~> 1.4"}, + {:floki, "~> 0.26.0"} ] end diff --git a/mix.lock b/mix.lock index 6489f6e..bb3b2e3 100644 --- a/mix.lock +++ b/mix.lock @@ -1,5 +1,6 @@ %{ "bcrypt_elixir": {:hex, :bcrypt_elixir, "2.2.0", "3df902b81ce7fa8867a2ae30d20a1da6877a2c056bfb116fd0bc8a5f0190cea4", [:make, :mix], [{:comeonin, "~> 5.3", [hex: :comeonin, repo: "hexpm", optional: false]}, {:elixir_make, "~> 0.6", [hex: :elixir_make, repo: "hexpm", optional: false]}], "hexpm", "762be3fcb779f08207531bc6612cca480a338e4b4357abb49f5ce00240a77d1e"}, + "certifi": {:hex, :certifi, "2.5.1", "867ce347f7c7d78563450a18a6a28a8090331e77fa02380b4a21962a65d36ee5", [:rebar3], [{:parse_trans, "~>3.3", [hex: :parse_trans, repo: "hexpm", optional: false]}], "hexpm", "805abd97539caf89ec6d4732c91e62ba9da0cda51ac462380bbd28ee697a8c42"}, "comeonin": {:hex, :comeonin, "5.3.1", "7fe612b739c78c9c1a75186ef2d322ce4d25032d119823269d0aa1e2f1e20025", [:mix], [], "hexpm", "d6222483060c17f0977fad1b7401ef0c5863c985a64352755f366aee3799c245"}, "connection": {:hex, :connection, "1.0.4", "a1cae72211f0eef17705aaededacac3eb30e6625b04a6117c1b2db6ace7d5976", [:mix], [], "hexpm", "4a0850c9be22a43af9920a71ab17c051f5f7d45c209e40269a1938832510e4d9"}, "cowboy": {:hex, :cowboy, "2.7.0", "91ed100138a764355f43316b1d23d7ff6bdb0de4ea618cb5d8677c93a7a2f115", [:rebar3], [{:cowlib, "~> 2.8.0", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "~> 1.7.1", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "04fd8c6a39edc6aaa9c26123009200fc61f92a3a94f3178c527b70b767c6e605"}, @@ -8,15 +9,26 @@ "db_connection": {:hex, :db_connection, "2.2.1", "caee17725495f5129cb7faebde001dc4406796f12a62b8949f4ac69315080566", [:mix], [{:connection, "~> 1.0.2", [hex: :connection, repo: "hexpm", optional: false]}], "hexpm", "2b02ece62d9f983fcd40954e443b7d9e6589664380e5546b2b9b523cd0fb59e1"}, "decimal": {:hex, :decimal, "1.8.1", "a4ef3f5f3428bdbc0d35374029ffcf4ede8533536fa79896dd450168d9acdf3c", [:mix], [], "hexpm", "3cb154b00225ac687f6cbd4acc4b7960027c757a5152b369923ead9ddbca7aec"}, "ecto": {:hex, :ecto, "3.3.4", "95b05c82ae91361475e5491c9f3ac47632f940b3f92ae3988ac1aad04989c5bb", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "9b96cbb83a94713731461ea48521b178b0e3863d310a39a3948c807266eebd69"}, + "ecto_enum": {:hex, :ecto_enum, "1.4.0", "d14b00e04b974afc69c251632d1e49594d899067ee2b376277efd8233027aec8", [:mix], [{:ecto, ">= 3.0.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:ecto_sql, "> 3.0.0", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:mariaex, ">= 0.0.0", [hex: :mariaex, repo: "hexpm", optional: true]}, {:postgrex, ">= 0.0.0", [hex: :postgrex, repo: "hexpm", optional: true]}], "hexpm", "8fb55c087181c2b15eee406519dc22578fa60dd82c088be376d0010172764ee4"}, "ecto_sql": {:hex, :ecto_sql, "3.3.4", "aa18af12eb875fbcda2f75e608b3bd534ebf020fc4f6448e4672fcdcbb081244", [:mix], [{:db_connection, "~> 2.2", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.4 or ~> 3.3.3", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.3.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.15.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "5eccbdbf92e3c6f213007a82d5dbba4cd9bb659d1a21331f89f408e4c0efd7a8"}, "elixir_make": {:hex, :elixir_make, "0.6.0", "38349f3e29aff4864352084fc736fa7fa0f2995a819a737554f7ebd28b85aaab", [:mix], [], "hexpm", "d522695b93b7f0b4c0fcb2dfe73a6b905b1c301226a5a55cb42e5b14d509e050"}, "faker": {:hex, :faker, "0.13.0", "8abcb996f010ccd6c85588c89fc047f11134e04da019b70252f95431d721a3dc", [:mix], [], "hexpm", "b0016680cae6776e3d1caa34d70438acc09c11c003e80fd3d44f79ec7370be00"}, "faker_elixir_octopus": {:hex, :faker_elixir_octopus, "1.0.2", "9895a5cbd08ba47ec70aa04dd91077ff57abe6e49239e853c5f6969e106ce997", [:mix], [], "hexpm", "a332fd3c5c633c87e15f262109a5f141b3e2dd08324df71d2591ef918f20789d"}, "file_system": {:hex, :file_system, "0.2.8", "f632bd287927a1eed2b718f22af727c5aeaccc9a98d8c2bd7bff709e851dc986", [:mix], [], "hexpm", "97a3b6f8d63ef53bd0113070102db2ce05352ecf0d25390eb8d747c2bde98bca"}, + "floki": {:hex, :floki, "0.26.0", "4df88977e2e357c6720e1b650f613444bfb48c5acfc6a0c646ab007d08ad13bf", [:mix], [{:html_entities, "~> 0.5.0", [hex: :html_entities, repo: "hexpm", optional: false]}], "hexpm", "e7b66ce7feef5518a9cd9fc7b52dd62a64028bd9cb6d6ad282a0f0fc90a4ae52"}, "gettext": {:hex, :gettext, "0.17.4", "f13088e1ec10ce01665cf25f5ff779e7df3f2dc71b37084976cf89d1aa124d5c", [:mix], [], "hexpm", "3c75b5ea8288e2ee7ea503ff9e30dfe4d07ad3c054576a6e60040e79a801e14d"}, + "hackney": {:hex, :hackney, "1.15.2", "07e33c794f8f8964ee86cebec1a8ed88db5070e52e904b8f12209773c1036085", [:rebar3], [{:certifi, "2.5.1", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "6.0.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.5", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm", "e0100f8ef7d1124222c11ad362c857d3df7cb5f4204054f9f0f4a728666591fc"}, + "html_entities": {:hex, :html_entities, "0.5.1", "1c9715058b42c35a2ab65edc5b36d0ea66dd083767bef6e3edb57870ef556549", [:mix], [], "hexpm", "30efab070904eb897ff05cd52fa61c1025d7f8ef3a9ca250bc4e6513d16c32de"}, + "httpoison": {:hex, :httpoison, "1.6.2", "ace7c8d3a361cebccbed19c283c349b3d26991eff73a1eaaa8abae2e3c8089b6", [:mix], [{:hackney, "~> 1.15 and >= 1.15.2", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "aa2c74bd271af34239a3948779612f87df2422c2fdcfdbcec28d9c105f0773fe"}, + "idna": {:hex, :idna, "6.0.0", "689c46cbcdf3524c44d5f3dde8001f364cd7608a99556d8fbd8239a5798d4c10", [:rebar3], [{:unicode_util_compat, "0.4.1", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "4bdd305eb64e18b0273864920695cb18d7a2021f31a11b9c5fbcd9a253f936e2"}, "jason": {:hex, :jason, "1.2.0", "10043418c42d2493d0ee212d3fddd25d7ffe484380afad769a0a38795938e448", [:mix], [{:decimal, "~> 1.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "116747dbe057794c3a3e4e143b7c8390b29f634e16c78a7f59ba75bfa6852e7f"}, + "meeseeks": {:hex, :meeseeks, "0.15.0", "8ea95378a7cbaf8f17b7369c150bae6a8dc11c9793d9932deff0ce06c6a7e11a", [:mix], [{:meeseeks_html5ever, "~> 0.12.1", [hex: :meeseeks_html5ever, repo: "hexpm", optional: false]}], "hexpm", "3d3644ef0fd0eb4828f5d76fc8be413c1fa35ddbc22f2d424e99eea48a9b6013"}, + "meeseeks_html5ever": {:hex, :meeseeks_html5ever, "0.12.1", "718fab10d05b83204524a518b2b88caa37ba6a6e02f82e80d6a7bc47552fb54a", [:mix], [{:rustler, "~> 0.21.0", [hex: :rustler, repo: "hexpm", optional: false]}], "hexpm", "11489094637f49a26bad4610a9138352c8d229339d888169cb35b08cdfd8861a"}, + "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm", "69b09adddc4f74a40716ae54d140f93beb0fb8978d8636eaded0c31b6f099f16"}, "mime": {:hex, :mime, "1.3.1", "30ce04ab3175b6ad0bdce0035cba77bba68b813d523d1aac73d9781b4d193cf8", [:mix], [], "hexpm", "6cbe761d6a0ca5a31a0931bf4c63204bceb64538e664a8ecf784a9a6f3b875f1"}, + "mimerl": {:hex, :mimerl, "1.2.0", "67e2d3f571088d5cfd3e550c383094b47159f3eee8ffa08e64106cdf5e981be3", [:rebar3], [], "hexpm", "f278585650aa581986264638ebf698f8bb19df297f66ad91b18910dfc6e19323"}, "parallel_stream": {:hex, :parallel_stream, "1.0.6", "b967be2b23f0f6787fab7ed681b4c45a215a81481fb62b01a5b750fa8f30f76c", [:mix], [], "hexpm", "639b2e8749e11b87b9eb42f2ad325d161c170b39b288ac8d04c4f31f8f0823eb"}, + "parse_trans": {:hex, :parse_trans, "3.3.0", "09765507a3c7590a784615cfd421d101aec25098d50b89d7aa1d66646bc571c1", [:rebar3], [], "hexpm", "17ef63abde837ad30680ea7f857dd9e7ced9476cdd7b0394432af4bfc241b960"}, "phoenix": {:hex, :phoenix, "1.4.16", "2cbbe0c81e6601567c44cc380c33aa42a1372ac1426e3de3d93ac448a7ec4308", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 1.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:plug, "~> 1.8.1 or ~> 1.9", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 1.0 or ~> 2.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "856cc1a032fa53822737413cf51aa60e750525d7ece7d1c0576d90d7c0f05c24"}, "phoenix_ecto": {:hex, :phoenix_ecto, "4.1.0", "a044d0756d0464c5a541b4a0bf4bcaf89bffcaf92468862408290682c73ae50d", [:mix], [{:ecto, "~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.9", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "c5e666a341ff104d0399d8f0e4ff094559b2fde13a5985d4cb5023b2c2ac558b"}, "phoenix_html": {:hex, :phoenix_html, "2.14.1", "7dabafadedb552db142aacbd1f11de1c0bbaa247f90c449ca549d5e30bbc66b4", [:mix], [{:plug, "~> 1.5", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "536d5200ad37fecfe55b3241d90b7a8c3a2ca60cd012fc065f776324fa9ab0a9"}, @@ -25,7 +37,12 @@ "plug": {:hex, :plug, "1.9.0", "8d7c4e26962283ff9f8f3347bd73838e2413fbc38b7bb5467d5924f68f3a5a4a", [:mix], [{:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm", "9902eda2c52ada2a096434682e99a2493f5d06a94d6ac6bcfff9805f952350f1"}, "plug_cowboy": {:hex, :plug_cowboy, "2.1.2", "8b0addb5908c5238fac38e442e81b6fcd32788eaa03246b4d55d147c47c5805e", [:mix], [{:cowboy, "~> 2.5", [hex: :cowboy, repo: "hexpm", optional: false]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "7d722581ce865a237e14da6d946f92704101740a256bd13ec91e63c0b122fc70"}, "plug_crypto": {:hex, :plug_crypto, "1.1.2", "bdd187572cc26dbd95b87136290425f2b580a116d3fb1f564216918c9730d227", [:mix], [], "hexpm", "6b8b608f895b6ffcfad49c37c7883e8df98ae19c6a28113b02aa1e9c5b22d6b5"}, + "poolboy": {:hex, :poolboy, "1.5.2", "392b007a1693a64540cead79830443abf5762f5d30cf50bc95cb2c1aaafa006b", [:rebar3], [], "hexpm", "dad79704ce5440f3d5a3681c8590b9dc25d1a561e8f5a9c995281012860901e3"}, "postgrex": {:hex, :postgrex, "0.15.3", "5806baa8a19a68c4d07c7a624ccdb9b57e89cbc573f1b98099e3741214746ae4", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "4737ce62a31747b4c63c12b20c62307e51bb4fcd730ca0c32c280991e0606c90"}, "ranch": {:hex, :ranch, "1.7.1", "6b1fab51b49196860b733a49c07604465a47bdb78aa10c1c16a3d199f7f8c881", [:rebar3], [], "hexpm", "451d8527787df716d99dc36162fca05934915db0b6141bbdac2ea8d3c7afc7d7"}, + "rustler": {:hex, :rustler, "0.21.0", "68cc4fc015d0b9541865ea78e78e9ef2dd91ee4be80bf543fd15791102a45aca", [:mix], [{:toml, "~> 0.5.2", [hex: :toml, repo: "hexpm", optional: false]}], "hexpm", "e5429378c397f37f1091a35593b153aee1925e197c6842d04648d802edb52f80"}, + "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.5", "6eaf7ad16cb568bb01753dbbd7a95ff8b91c7979482b95f38443fe2c8852a79b", [:make, :mix, :rebar3], [], "hexpm", "13104d7897e38ed7f044c4de953a6c28597d1c952075eb2e328bc6d6f2bfc496"}, "telemetry": {:hex, :telemetry, "0.4.1", "ae2718484892448a24470e6aa341bc847c3277bfb8d4e9289f7474d752c09c7f", [:rebar3], [], "hexpm", "4738382e36a0a9a2b6e25d67c960e40e1a2c95560b9f936d8e29de8cd858480f"}, + "toml": {:hex, :toml, "0.5.2", "e471388a8726d1ce51a6b32f864b8228a1eb8edc907a0edf2bb50eab9321b526", [:mix], [], "hexpm", "f1e3dabef71fb510d015fad18c0e05e7c57281001141504c6b69d94e99750a07"}, + "unicode_util_compat": {:hex, :unicode_util_compat, "0.4.1", "d869e4c68901dd9531385bb0c8c40444ebf624e60b6962d95952775cac5e90cd", [:rebar3], [], "hexpm", "1d1848c40487cdb0b30e8ed975e34e025860c02e419cb615d255849f3427439d"}, } diff --git a/priv/repo/migrations/20200331103751_add_status_and_raw_html_result_to_keywords.exs b/priv/repo/migrations/20200331103751_add_status_and_raw_html_result_to_keywords.exs new file mode 100644 index 0000000..7cf2ec5 --- /dev/null +++ b/priv/repo/migrations/20200331103751_add_status_and_raw_html_result_to_keywords.exs @@ -0,0 +1,10 @@ +defmodule GoogleCrawler.Repo.Migrations.AddStatusAndRawHtmlResultToKeywords do + use Ecto.Migration + + def change do + alter table(:keywords) do + add :status, :integer + add :raw_html_result, :text + end + end +end From ae8b735e17c837cd6c3a1082b4f9b7d402d5d77e Mon Sep 17 00:00:00 2001 From: Rossukhon Leagmongkol Date: Tue, 31 Mar 2020 18:26:36 +0700 Subject: [PATCH 02/12] Add search keyword task --- lib/google_crawler/search/crawler.ex | 9 ++++++--- lib/google_crawler/search/keyword.ex | 2 +- lib/google_crawler/search/scrapper.ex | 2 +- lib/google_crawler/search/search_keyword_task.ex | 14 +++++++++++++- 4 files changed, 21 insertions(+), 6 deletions(-) diff --git a/lib/google_crawler/search/crawler.ex b/lib/google_crawler/search/crawler.ex index a276088..c8d726c 100644 --- a/lib/google_crawler/search/crawler.ex +++ b/lib/google_crawler/search/crawler.ex @@ -1,13 +1,16 @@ -defmodule GoogleCrawler.Crawler do +defmodule GoogleCrawler.Search.Crawler do @url "https://www.google.com/search?q=" def fetch(keyword) do case HTTPoison.get(@url <> keyword) do {:ok, %HTTPoison.Response{status_code: 200, body: body}} -> - body + {:ok, body} + + {:ok, %HTTPoison.Response{body: body}} -> + {:error, body} {:error, %HTTPoison.Error{reason: reason}} -> - raise GoogleCrawler.Errors.FetchError, message: reason + {:error, reason} end end end diff --git a/lib/google_crawler/search/keyword.ex b/lib/google_crawler/search/keyword.ex index 49bc007..bd8e7ee 100644 --- a/lib/google_crawler/search/keyword.ex +++ b/lib/google_crawler/search/keyword.ex @@ -9,7 +9,7 @@ defmodule GoogleCrawler.Search.Keyword do schema "keywords" do field :keyword, :string - field :status, GoogleCrawler.Search.Keyword.Status + field :status, GoogleCrawler.Search.Keyword.Status, default: :in_queue field :raw_html_result, :string belongs_to :user, GoogleCrawler.Accounts.User diff --git a/lib/google_crawler/search/scrapper.ex b/lib/google_crawler/search/scrapper.ex index 582c4ff..a607bfb 100644 --- a/lib/google_crawler/search/scrapper.ex +++ b/lib/google_crawler/search/scrapper.ex @@ -1,4 +1,4 @@ -defmodule GoogleCrawler.Scapper do +defmodule GoogleCrawler.Search.Scapper do def scrap(html) do end diff --git a/lib/google_crawler/search/search_keyword_task.ex b/lib/google_crawler/search/search_keyword_task.ex index c883899..139d5da 100644 --- a/lib/google_crawler/search/search_keyword_task.ex +++ b/lib/google_crawler/search/search_keyword_task.ex @@ -1,5 +1,17 @@ -defmodule GoogleCrawler.SearchKeywordTask do +defmodule GoogleCrawler.Search.SearchKeywordTask do + alias GoogleCrawler.Search.Crawler + alias GoogleCrawler.Search.Scapper + def perform(%{keyword: keyword}) do + # Update the status to in progress + + case Crawler.fetch(keyword) do + {:ok, body} -> + Scapper.scrap(body) + # Store the result to the db + {:error, reason} -> + # TODO: Retry the task + end end end From 360c9edf5044e774dfcbbdd2f9157e1ea54d106a Mon Sep 17 00:00:00 2001 From: Rossukhon Leagmongkol Date: Wed, 1 Apr 2020 11:21:03 +0700 Subject: [PATCH 03/12] Add update keyword method, remove unrelated code and update tests --- lib/google_crawler/errors.ex | 4 ---- lib/google_crawler/search.ex | 18 ++++++++++++++++++ lib/google_crawler/search/keyword.ex | 7 +++---- lib/google_crawler/search/scrapper.ex | 5 ----- .../search/search_keyword_task.ex | 9 +++++---- mix.exs | 3 +-- test/factories/keyword_factory.ex | 3 ++- test/google_crawler/search/keyword_test.exs | 17 +++++++++++++++++ test/google_crawler/search_test.exs | 19 +++++++++++++++++++ 9 files changed, 65 insertions(+), 20 deletions(-) delete mode 100644 lib/google_crawler/search/scrapper.ex diff --git a/lib/google_crawler/errors.ex b/lib/google_crawler/errors.ex index 3f637c8..0521f3a 100644 --- a/lib/google_crawler/errors.ex +++ b/lib/google_crawler/errors.ex @@ -1,7 +1,3 @@ defmodule GoogleCrawler.Errors.FileNotSupportedError do defexception message: "File is not supported" end - -defmodule GoogleCrawler.Errors.FetchError do - defexception message: "Fails to fetch the keyword result" -end diff --git a/lib/google_crawler/search.ex b/lib/google_crawler/search.ex index 9c261e0..75cddc5 100644 --- a/lib/google_crawler/search.ex +++ b/lib/google_crawler/search.ex @@ -58,6 +58,24 @@ defmodule GoogleCrawler.Search do |> Repo.insert() end + @doc """ + Updates a keyword. + + ## Examples + + iex> update_keyword(keyword, %{field: new_value}) + {:ok, %Keyword{}} + + iex> update_keyword(keyword, %{field: bad_value}) + {:error, %Ecto.Changeset{}} + + """ + def update_keyword(%Keyword{} = keyword, attrs \\ %{}) do + keyword + |> Keyword.changeset(attrs) + |> Repo.update() + end + @doc """ Parses the keyword from the given file. Returns the stream for each line in the csv file as [line_result]. diff --git a/lib/google_crawler/search/keyword.ex b/lib/google_crawler/search/keyword.ex index bd8e7ee..17b20cc 100644 --- a/lib/google_crawler/search/keyword.ex +++ b/lib/google_crawler/search/keyword.ex @@ -1,7 +1,6 @@ import EctoEnum -defenum GoogleCrawler.Search.Keyword.Status, - in_queue: 0, in_progress: 1, completed: 2 +defenum(GoogleCrawler.Search.Keyword.Status, in_queue: 0, in_progress: 1, completed: 2) defmodule GoogleCrawler.Search.Keyword do use Ecto.Schema @@ -19,7 +18,7 @@ defmodule GoogleCrawler.Search.Keyword do def changeset(keyword, attrs \\ %{}) do keyword - |> cast(attrs, [:keyword, :user_id]) - |> validate_required([:keyword, :user_id]) + |> cast(attrs, [:keyword, :user_id, :status]) + |> validate_required([:keyword, :user_id, :status]) end end diff --git a/lib/google_crawler/search/scrapper.ex b/lib/google_crawler/search/scrapper.ex deleted file mode 100644 index a607bfb..0000000 --- a/lib/google_crawler/search/scrapper.ex +++ /dev/null @@ -1,5 +0,0 @@ -defmodule GoogleCrawler.Search.Scapper do - def scrap(html) do - - end -end diff --git a/lib/google_crawler/search/search_keyword_task.ex b/lib/google_crawler/search/search_keyword_task.ex index 139d5da..6eb2420 100644 --- a/lib/google_crawler/search/search_keyword_task.ex +++ b/lib/google_crawler/search/search_keyword_task.ex @@ -1,17 +1,18 @@ defmodule GoogleCrawler.Search.SearchKeywordTask do alias GoogleCrawler.Search.Crawler - alias GoogleCrawler.Search.Scapper + alias GoogleCrawler.Search.Keyword - def perform(%{keyword: keyword}) do + def perform(%Keyword{} = keyword) do # Update the status to in progress - case Crawler.fetch(keyword) do + case Crawler.fetch(keyword.keyword) do {:ok, body} -> - Scapper.scrap(body) # Store the result to the db + IO.inspect(body) {:error, reason} -> # TODO: Retry the task + IO.inspect(reason) end end end diff --git a/mix.exs b/mix.exs index d8a21d0..84cd248 100644 --- a/mix.exs +++ b/mix.exs @@ -47,8 +47,7 @@ defmodule GoogleCrawler.MixProject do {:faker_elixir_octopus, "~> 1.0.0", only: [:dev, :test]}, {:csv, "~> 2.3"}, {:httpoison, "~> 1.6"}, - {:ecto_enum, "~> 1.4"}, - {:floki, "~> 0.26.0"} + {:ecto_enum, "~> 1.4"} ] end diff --git a/test/factories/keyword_factory.ex b/test/factories/keyword_factory.ex index f8170fa..6a0a1df 100644 --- a/test/factories/keyword_factory.ex +++ b/test/factories/keyword_factory.ex @@ -4,7 +4,8 @@ defmodule GoogleCrawler.KeywordFactory do def default_attrs do %{ - keyword: FakerElixir.Lorem.word() + keyword: FakerElixir.Lorem.word(), + status: :in_queue } end diff --git a/test/google_crawler/search/keyword_test.exs b/test/google_crawler/search/keyword_test.exs index d18e444..7e4b4b9 100644 --- a/test/google_crawler/search/keyword_test.exs +++ b/test/google_crawler/search/keyword_test.exs @@ -22,5 +22,22 @@ defmodule Googlecrawler.Search.KeywordTest do refute changeset.valid? assert %{user_id: ["can't be blank"]} = errors_on(changeset) end + + test "status is required" do + user = UserFactory.create() + attrs = KeywordFactory.build_attrs(%{status: nil, user: user}) + changeset = Keyword.changeset(%Keyword{}, attrs) + + refute changeset.valid? + assert %{status: ["can't be blank"]} = errors_on(changeset) + end + + test "status is valid" do + attrs = KeywordFactory.build_attrs(%{status: :invalid}) + changeset = Keyword.changeset(%Keyword{}, attrs) + + refute changeset.valid? + assert %{status: ["is invalid"]} = errors_on(changeset) + end end end diff --git a/test/google_crawler/search_test.exs b/test/google_crawler/search_test.exs index e3f567b..8856ecb 100644 --- a/test/google_crawler/search_test.exs +++ b/test/google_crawler/search_test.exs @@ -30,6 +30,8 @@ defmodule GoogleCrawler.SearchTest do assert {:ok, %Keyword{} = keyword} = Search.create_keyword(keyword_attrs, user) assert keyword.keyword == "elixir" + assert keyword.status == :in_queue + assert keyword.user_id == user.id end test "create_keyword/1 with invalid data returns error changeset" do @@ -38,6 +40,23 @@ defmodule GoogleCrawler.SearchTest do assert {:error, %Ecto.Changeset{}} = Search.create_keyword(keyword_attrs, user) end + + test "update_keyword/2 with valid data updates the keyword" do + keyword = KeywordFactory.create() + keyword_attrs = %{keyword: "new", status: :in_progress} + + assert {:ok, %Keyword{} = keyword} = Search.update_keyword(keyword, keyword_attrs) + assert keyword.keyword == "new" + assert keyword.status == :in_progress + end + + test "update_keyword/2 with invalid data returns error changeset" do + keyword = KeywordFactory.create() + keyword_attrs = %{keyword: "", status: :invalid} + + assert {:error, %Ecto.Changeset{}} = Search.update_keyword(keyword, keyword_attrs) + assert Repo.get_by(Keyword, keyword: keyword.keyword) != nil + end end describe "keyword file" do From aa2ad06e8eb910185a5568a552a022d493847b1d Mon Sep 17 00:00:00 2001 From: Rossukhon Leagmongkol Date: Wed, 1 Apr 2020 14:02:59 +0700 Subject: [PATCH 04/12] Implement the background task to search and scrap the page --- lib/google_crawler/search.ex | 23 +++++++++++++++++++ lib/google_crawler/search/keyword.ex | 4 +++- .../search/{crawler.ex => page_fetcher.ex} | 2 +- lib/google_crawler/search/page_scrapper.ex | 15 ++++++++++++ .../search/search_keyword_task.ex | 13 ++++++++--- .../controllers/upload_controller.ex | 2 +- 6 files changed, 53 insertions(+), 6 deletions(-) rename lib/google_crawler/search/{crawler.ex => page_fetcher.ex} (88%) create mode 100644 lib/google_crawler/search/page_scrapper.ex diff --git a/lib/google_crawler/search.ex b/lib/google_crawler/search.ex index 75cddc5..fb7d471 100644 --- a/lib/google_crawler/search.ex +++ b/lib/google_crawler/search.ex @@ -58,6 +58,29 @@ defmodule GoogleCrawler.Search do |> Repo.insert() end + @doc """ + Save the keyword and perform the keyword search + + ## Examples + + iex> create_and_search_keyword(%{field: value}, %User{}) + {:ok, %Keyword{}} + + iex> create_and_search_keyword(%{field: bad_value}, %User{}) + {:error, %Ecto.Changeset{}} + """ + def create_and_search_keyword(attrs \\ %{}, user) do + case create_keyword(attrs, user) do + {:ok, %Keyword{} = keyword} -> + search_task = fn -> GoogleCrawler.Search.SearchKeywordTask.perform(keyword) end + Task.Supervisor.start_child(GoogleCrawler.TaskSupervisor, search_task) + {:ok, %Keyword{}} + + {:error, %Ecto.Changeset{} = changeset} -> + {:error, changeset} + end + end + @doc """ Updates a keyword. diff --git a/lib/google_crawler/search/keyword.ex b/lib/google_crawler/search/keyword.ex index 17b20cc..85f6c69 100644 --- a/lib/google_crawler/search/keyword.ex +++ b/lib/google_crawler/search/keyword.ex @@ -16,9 +16,11 @@ defmodule GoogleCrawler.Search.Keyword do timestamps() end + @fields ~w(keyword user_id status raw_html_result)a + def changeset(keyword, attrs \\ %{}) do keyword - |> cast(attrs, [:keyword, :user_id, :status]) + |> cast(attrs, @fields) |> validate_required([:keyword, :user_id, :status]) end end diff --git a/lib/google_crawler/search/crawler.ex b/lib/google_crawler/search/page_fetcher.ex similarity index 88% rename from lib/google_crawler/search/crawler.ex rename to lib/google_crawler/search/page_fetcher.ex index c8d726c..a41394a 100644 --- a/lib/google_crawler/search/crawler.ex +++ b/lib/google_crawler/search/page_fetcher.ex @@ -1,4 +1,4 @@ -defmodule GoogleCrawler.Search.Crawler do +defmodule GoogleCrawler.Search.PageFetcher do @url "https://www.google.com/search?q=" def fetch(keyword) do diff --git a/lib/google_crawler/search/page_scrapper.ex b/lib/google_crawler/search/page_scrapper.ex new file mode 100644 index 0000000..083d1dc --- /dev/null +++ b/lib/google_crawler/search/page_scrapper.ex @@ -0,0 +1,15 @@ +defmodule GoogleCrawler.Search.PageScrapper do + def scrap(html) do + # TODO: Scrap the page content + %{ + raw_html_result: cleanup_html(html) + } + end + + def cleanup_html(html) do + html + |> String.chunk(:printable) + |> Enum.filter(&String.printable?/1) + |> Enum.join + end +end diff --git a/lib/google_crawler/search/search_keyword_task.ex b/lib/google_crawler/search/search_keyword_task.ex index 6eb2420..dac4023 100644 --- a/lib/google_crawler/search/search_keyword_task.ex +++ b/lib/google_crawler/search/search_keyword_task.ex @@ -1,14 +1,21 @@ defmodule GoogleCrawler.Search.SearchKeywordTask do - alias GoogleCrawler.Search.Crawler + alias GoogleCrawler.Search alias GoogleCrawler.Search.Keyword + alias GoogleCrawler.Search.PageFetcher + alias GoogleCrawler.Search.PageScrapper def perform(%Keyword{} = keyword) do # Update the status to in progress + Search.update_keyword(keyword, %{status: :in_progress}) - case Crawler.fetch(keyword.keyword) do + case PageFetcher.fetch(keyword.keyword) do {:ok, body} -> # Store the result to the db - IO.inspect(body) + result = PageScrapper.scrap(body) + Search.update_keyword(keyword, %{ + status: :completed, + raw_html_result: result.raw_html_result + }) {:error, reason} -> # TODO: Retry the task diff --git a/lib/google_crawler_web/controllers/upload_controller.ex b/lib/google_crawler_web/controllers/upload_controller.ex index e7ddf8b..d1a6fb2 100644 --- a/lib/google_crawler_web/controllers/upload_controller.ex +++ b/lib/google_crawler_web/controllers/upload_controller.ex @@ -27,7 +27,7 @@ defmodule GoogleCrawlerWeb.UploadController do csv_result |> Stream.map(fn keyword_row -> List.first(keyword_row) end) |> Stream.map(fn keyword -> %{keyword: keyword} end) - |> Enum.map(&Search.create_keyword(&1, conn.assigns.current_user)) + |> Enum.map(&Search.create_and_search_keyword(&1, conn.assigns.current_user)) end defp put_error_flash_for_failed_keywords(create_result, conn) do From 4f85743fee9a4e942e318211518a7c395097e230 Mon Sep 17 00:00:00 2001 From: Rossukhon Leagmongkol Date: Wed, 1 Apr 2020 15:02:54 +0700 Subject: [PATCH 05/12] Add keyword show page --- assets/css/{app.css => app.scss} | 2 + assets/css/screens/dashboard.scss | 34 + assets/js/app.js | 2 +- assets/package-lock.json | 1261 ++++++++++++++++- assets/package.json | 2 + assets/webpack.config.js | 8 +- .../controllers/keyword_controller.ex | 11 + lib/google_crawler_web/router.ex | 1 + .../templates/dashboard/index.html.eex | 16 +- .../templates/keyword/show.html.eex | 6 + 10 files changed, 1333 insertions(+), 10 deletions(-) rename assets/css/{app.css => app.scss} (67%) create mode 100644 assets/css/screens/dashboard.scss create mode 100644 lib/google_crawler_web/controllers/keyword_controller.ex create mode 100644 lib/google_crawler_web/templates/keyword/show.html.eex diff --git a/assets/css/app.css b/assets/css/app.scss similarity index 67% rename from assets/css/app.css rename to assets/css/app.scss index fec0b3f..486fc72 100644 --- a/assets/css/app.css +++ b/assets/css/app.scss @@ -1,3 +1,5 @@ /* This file is for your main application css. */ @import "./phoenix.css"; + +@import "./screens/dashboard.scss" diff --git a/assets/css/screens/dashboard.scss b/assets/css/screens/dashboard.scss new file mode 100644 index 0000000..ec3d46a --- /dev/null +++ b/assets/css/screens/dashboard.scss @@ -0,0 +1,34 @@ +body.dashboard.index { + .keyword-list-item { + display: flex; + flex-wrap: wrap; + } + + .keyword-list-item:not(:last-child)::after { + display: block; + + width: 100%; + height: 1px; + + background-color: #eeeeee; + + content: ""; + } + + .keyword-col { + padding: 10px; + } + + .keyword-col.keyword-name { + flex: 0 0 40%; + } + + .keyword-col.keyword-status { + flex: 0 0 30%; + } + + .keyword-col.keyword-action { + flex: 0 0 30%; + text-align: right; + } +} diff --git a/assets/js/app.js b/assets/js/app.js index 8a5d386..1c025d6 100644 --- a/assets/js/app.js +++ b/assets/js/app.js @@ -1,7 +1,7 @@ // We need to import the CSS so that webpack will load it. // The MiniCssExtractPlugin is used to separate it out into // its own CSS file. -import css from "../css/app.css" +import css from "../css/app.scss" // webpack automatically bundles all modules in your // entry points. Those entry points can be configured diff --git a/assets/package-lock.json b/assets/package-lock.json index e87a496..3d40f3e 100644 --- a/assets/package-lock.json +++ b/assets/package-lock.json @@ -1099,6 +1099,12 @@ "integrity": "sha512-NuHqBY1PB/D8xU6s/thBgOAiAP7HOYDQ32+BFZILJ8ivkUkAHQnWfn6WhL79Owj1qmUnoN/YPhktdIoucipkAQ==", "dev": true }, + "abbrev": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", + "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==", + "dev": true + }, "acorn": { "version": "6.4.1", "resolved": "https://registry.npmjs.org/acorn/-/acorn-6.4.1.tgz", @@ -1145,6 +1151,12 @@ "integrity": "sha1-l6ERlkmyEa0zaR2fn0hqjsn74KM=", "dev": true }, + "amdefine": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/amdefine/-/amdefine-1.0.1.tgz", + "integrity": "sha1-SlKCrBZHKek2Gbz9OtFR+BfOkfU=", + "dev": true + }, "ansi-colors": { "version": "3.2.4", "resolved": "https://registry.npmjs.org/ansi-colors/-/ansi-colors-3.2.4.tgz", @@ -1193,6 +1205,16 @@ "integrity": "sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw==", "dev": true }, + "are-we-there-yet": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-1.1.5.tgz", + "integrity": "sha512-5hYdAkZlcG8tOLujVDTgCT+uPX0VnpAH28gWsLfzpXYm7wP6mp5Q/gYyR7YQ0cKVJcXJnl3j2kpBan13PtQf6w==", + "dev": true, + "requires": { + "delegates": "1.0.0", + "readable-stream": "2.3.7" + } + }, "argparse": { "version": "1.0.10", "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", @@ -1220,6 +1242,12 @@ "integrity": "sha1-45sJrqne+Gao8gbiiK9jkZuuOcQ=", "dev": true }, + "array-find-index": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/array-find-index/-/array-find-index-1.0.2.tgz", + "integrity": "sha1-3wEKoSh+Fku9pvlyOwqWoexBh6E=", + "dev": true + }, "array-union": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/array-union/-/array-union-1.0.2.tgz", @@ -1241,6 +1269,15 @@ "integrity": "sha1-qJS3XUvE9s1nnvMkSp/Y9Gri1Cg=", "dev": true }, + "asn1": { + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.4.tgz", + "integrity": "sha512-jxwzQpLQjSmWXgwaCZE9Nz+glAG01yF1QnWgbhGwHI5A6FRIEY6IVqtHhIepHqI7/kyEyQEagBC5mBEFlIYvdg==", + "dev": true, + "requires": { + "safer-buffer": "2.1.2" + } + }, "asn1.js": { "version": "4.10.1", "resolved": "https://registry.npmjs.org/asn1.js/-/asn1.js-4.10.1.tgz", @@ -1279,6 +1316,12 @@ } } }, + "assert-plus": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", + "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=", + "dev": true + }, "assign-symbols": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/assign-symbols/-/assign-symbols-1.0.0.tgz", @@ -1291,12 +1334,36 @@ "integrity": "sha512-z/WhQ5FPySLdvREByI2vZiTWwCnF0moMJ1hK9YQwDTHKh6I7/uSckMetoRGb5UBZPC1z0jlw+n/XCgjeH7y1AQ==", "dev": true }, + "async-foreach": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/async-foreach/-/async-foreach-0.1.3.tgz", + "integrity": "sha1-NhIfhFwFeBct5Bmpfb6x0W7DRUI=", + "dev": true + }, + "asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=", + "dev": true + }, "atob": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/atob/-/atob-2.1.2.tgz", "integrity": "sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg==", "dev": true }, + "aws-sign2": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz", + "integrity": "sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg=", + "dev": true + }, + "aws4": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.9.1.tgz", + "integrity": "sha512-wMHVg2EOHaMRxbzgFJ9gtjOOCrI80OHLG14rxi28XwOW8ux6IiEbRCGGGqCtdAIg4FQCbW20k9RsT4y3gJlFug==", + "dev": true + }, "babel-loader": { "version": "8.1.0", "resolved": "https://registry.npmjs.org/babel-loader/-/babel-loader-8.1.0.tgz", @@ -1386,6 +1453,15 @@ "integrity": "sha512-mLQ4i2QO1ytvGWFWmcngKO//JXAQueZvwEKtjgQFM4jIK0kU+ytMfplL8j+n5mspOfjHwoAg+9yhb7BwAHm36g==", "dev": true }, + "bcrypt-pbkdf": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz", + "integrity": "sha1-pDAdOJtqQ/m2f/PKEaP2Y342Dp4=", + "dev": true, + "requires": { + "tweetnacl": "0.14.5" + } + }, "big.js": { "version": "5.2.2", "resolved": "https://registry.npmjs.org/big.js/-/big.js-5.2.2.tgz", @@ -1408,6 +1484,15 @@ "file-uri-to-path": "1.0.0" } }, + "block-stream": { + "version": "0.0.9", + "resolved": "https://registry.npmjs.org/block-stream/-/block-stream-0.0.9.tgz", + "integrity": "sha1-E+v+d4oDIFz+A3UUgeu0szAMEmo=", + "dev": true, + "requires": { + "inherits": "2.0.4" + } + }, "bluebird": { "version": "3.7.2", "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz", @@ -1653,6 +1738,24 @@ "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==", "dev": true }, + "camelcase-keys": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/camelcase-keys/-/camelcase-keys-2.1.0.tgz", + "integrity": "sha1-MIvur/3ygRkFHvodkyITyRuPkuc=", + "dev": true, + "requires": { + "camelcase": "2.1.1", + "map-obj": "1.0.1" + }, + "dependencies": { + "camelcase": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-2.1.1.tgz", + "integrity": "sha1-fB0W1nmhu+WcoCys7PsBHiAfWh8=", + "dev": true + } + } + }, "caniuse-api": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/caniuse-api/-/caniuse-api-3.0.0.tgz", @@ -1671,6 +1774,12 @@ "integrity": "sha512-jU8CIFIj2oR7r4W+5AKcsvWNVIb6Q6OZE3UsrXrZBHFtreT4YgTeOJtTucp+zSedEpTi3L5wASSP0LYIE3if6w==", "dev": true }, + "caseless": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz", + "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=", + "dev": true + }, "chalk": { "version": "2.4.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", @@ -1767,6 +1876,17 @@ "wrap-ansi": "5.1.0" } }, + "clone-deep": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/clone-deep/-/clone-deep-4.0.1.tgz", + "integrity": "sha512-neHB9xuzh/wk0dIHweyAXv2aPGZIVk3pLMe+/RNzINf17fe0OG96QroktYAUm7SM1PBnzTabaLboqqxDyMU+SQ==", + "dev": true, + "requires": { + "is-plain-object": "2.0.4", + "kind-of": "6.0.3", + "shallow-clone": "3.0.1" + } + }, "coa": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/coa/-/coa-2.0.2.tgz", @@ -1778,6 +1898,12 @@ "q": "1.5.1" } }, + "code-point-at": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz", + "integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c=", + "dev": true + }, "collection-visit": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/collection-visit/-/collection-visit-1.0.0.tgz", @@ -1823,6 +1949,15 @@ "simple-swizzle": "0.2.2" } }, + "combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "dev": true, + "requires": { + "delayed-stream": "1.0.0" + } + }, "commander": { "version": "2.20.3", "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", @@ -1865,6 +2000,12 @@ "integrity": "sha512-ZMkYO/LkF17QvCPqM0gxw8yUzigAOZOSWSHg91FH6orS7vcEj5dVZTidN2fQ14yBSdg97RqhSNwLUXInd52OTA==", "dev": true }, + "console-control-strings": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz", + "integrity": "sha1-PXz0Rk22RG6mRL9LOVB/mFEAjo4=", + "dev": true + }, "constants-browserify": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/constants-browserify/-/constants-browserify-1.0.0.tgz", @@ -2208,12 +2349,30 @@ "css-tree": "1.0.0-alpha.37" } }, + "currently-unhandled": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/currently-unhandled/-/currently-unhandled-0.4.1.tgz", + "integrity": "sha1-mI3zP+qxke95mmE2nddsF635V+o=", + "dev": true, + "requires": { + "array-find-index": "1.0.2" + } + }, "cyclist": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/cyclist/-/cyclist-1.0.1.tgz", "integrity": "sha1-WW6WmP0MgOEgOMK4LW6xs1tiJNk=", "dev": true }, + "dashdash": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz", + "integrity": "sha1-hTz6D3y+L+1d4gMmuN1YEDX24vA=", + "dev": true, + "requires": { + "assert-plus": "1.0.0" + } + }, "debug": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", @@ -2285,6 +2444,18 @@ } } }, + "delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=", + "dev": true + }, + "delegates": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz", + "integrity": "sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o=", + "dev": true + }, "des.js": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/des.js/-/des.js-1.0.1.tgz", @@ -2382,6 +2553,16 @@ "stream-shift": "1.0.1" } }, + "ecc-jsbn": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz", + "integrity": "sha1-OoOpBOVDUyh4dMVkt1SThoSamMk=", + "dev": true, + "requires": { + "jsbn": "0.1.1", + "safer-buffer": "2.1.2" + } + }, "electron-to-chromium": { "version": "1.3.382", "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.3.382.tgz", @@ -2634,6 +2815,12 @@ "homedir-polyfill": "1.0.3" } }, + "extend": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==", + "dev": true + }, "extend-shallow": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-3.0.2.tgz", @@ -2720,6 +2907,12 @@ } } }, + "extsprintf": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz", + "integrity": "sha1-lpGEQOMEGnpBT4xS48V06zw+HgU=", + "dev": true + }, "fast-deep-equal": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.1.tgz", @@ -2816,6 +3009,23 @@ "integrity": "sha1-gQaNKVqBQuwKxybG4iAMMPttXoA=", "dev": true }, + "forever-agent": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz", + "integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE=", + "dev": true + }, + "form-data": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.3.tgz", + "integrity": "sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==", + "dev": true, + "requires": { + "asynckit": "0.4.0", + "combined-stream": "1.0.8", + "mime-types": "2.1.26" + } + }, "fragment-cache": { "version": "0.2.1", "resolved": "https://registry.npmjs.org/fragment-cache/-/fragment-cache-0.2.1.tgz", @@ -3393,12 +3603,86 @@ } } }, + "fstream": { + "version": "1.0.12", + "resolved": "https://registry.npmjs.org/fstream/-/fstream-1.0.12.tgz", + "integrity": "sha512-WvJ193OHa0GHPEL+AycEJgxvBEwyfRkN1vhjca23OaPVMCaLCXTd5qAu82AjTcgP1UJmytkOKb63Ypde7raDIg==", + "dev": true, + "requires": { + "graceful-fs": "4.2.3", + "inherits": "2.0.4", + "mkdirp": "0.5.4", + "rimraf": "2.7.1" + } + }, "function-bind": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==", "dev": true }, + "gauge": { + "version": "2.7.4", + "resolved": "https://registry.npmjs.org/gauge/-/gauge-2.7.4.tgz", + "integrity": "sha1-LANAXHU4w51+s3sxcCLjJfsBi/c=", + "dev": true, + "requires": { + "aproba": "1.2.0", + "console-control-strings": "1.1.0", + "has-unicode": "2.0.1", + "object-assign": "4.1.1", + "signal-exit": "3.0.2", + "string-width": "1.0.2", + "strip-ansi": "3.0.1", + "wide-align": "1.1.3" + }, + "dependencies": { + "ansi-regex": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", + "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=", + "dev": true + }, + "is-fullwidth-code-point": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz", + "integrity": "sha1-754xOG8DGn8NZDr4L95QxFfvAMs=", + "dev": true, + "requires": { + "number-is-nan": "1.0.1" + } + }, + "string-width": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", + "integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=", + "dev": true, + "requires": { + "code-point-at": "1.1.0", + "is-fullwidth-code-point": "1.0.0", + "strip-ansi": "3.0.1" + } + }, + "strip-ansi": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", + "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", + "dev": true, + "requires": { + "ansi-regex": "2.1.1" + } + } + } + }, + "gaze": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/gaze/-/gaze-1.1.3.tgz", + "integrity": "sha512-BRdNm8hbWzFzWHERTrejLqwHDfS4GibPoq5wjTPIoJHoBtKGPg3xAFfxmM+9ztbXelxcf2hwQcaz1PtmFeue8g==", + "dev": true, + "requires": { + "globule": "1.3.1" + } + }, "gensync": { "version": "1.0.0-beta.1", "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.1.tgz", @@ -3411,6 +3695,12 @@ "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", "dev": true }, + "get-stdin": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/get-stdin/-/get-stdin-4.0.1.tgz", + "integrity": "sha1-uWjGsKBDhDJJAui/Gl3zJXmkUP4=", + "dev": true + }, "get-stream": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-4.1.0.tgz", @@ -3426,6 +3716,15 @@ "integrity": "sha1-3BXKHGcjh8p2vTesCjlbogQqLCg=", "dev": true }, + "getpass": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz", + "integrity": "sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo=", + "dev": true, + "requires": { + "assert-plus": "1.0.0" + } + }, "glob": { "version": "7.1.6", "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz", @@ -3524,12 +3823,39 @@ } } }, + "globule": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/globule/-/globule-1.3.1.tgz", + "integrity": "sha512-OVyWOHgw29yosRHCHo7NncwR1hW5ew0W/UrvtwvjefVJeQ26q4/8r8FmPsSF1hJ93IgWkyv16pCTz6WblMzm/g==", + "dev": true, + "requires": { + "glob": "7.1.6", + "lodash": "4.17.15", + "minimatch": "3.0.4" + } + }, "graceful-fs": { "version": "4.2.3", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.3.tgz", "integrity": "sha512-a30VEBm4PEdx1dRB7MFK7BejejvCvBronbLjht+sHuGYj8PHs7M/5Z+rt5lw551vZ7yfTCj4Vuyy3mSJytDWRQ==", "dev": true }, + "har-schema": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz", + "integrity": "sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI=", + "dev": true + }, + "har-validator": { + "version": "5.1.3", + "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.1.3.tgz", + "integrity": "sha512-sNvOCzEQNr/qrvJgc3UG/kD4QtlHycrzwS+6mfTrrSq97BvaYcPZZI1ZSqGSPR73Cxn4LKTD4PttRwfU7jWq5g==", + "dev": true, + "requires": { + "ajv": "6.12.0", + "har-schema": "2.0.0" + } + }, "has": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", @@ -3539,6 +3865,23 @@ "function-bind": "1.1.1" } }, + "has-ansi": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz", + "integrity": "sha1-NPUEnOHs3ysGSa8+8k5F7TVBbZE=", + "dev": true, + "requires": { + "ansi-regex": "2.1.1" + }, + "dependencies": { + "ansi-regex": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", + "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=", + "dev": true + } + } + }, "has-flag": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", @@ -3551,6 +3894,12 @@ "integrity": "sha512-PLcsoqu++dmEIZB+6totNFKq/7Do+Z0u4oT0zKOJNl3lYK6vGwwu2hjHs+68OEZbTjiUE9bgOABXbP/GvrS0Kg==", "dev": true }, + "has-unicode": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.1.tgz", + "integrity": "sha1-4Ob+aijPUROIVeCG0Wkedx3iqLk=", + "dev": true + }, "has-value": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/has-value/-/has-value-1.0.0.tgz", @@ -3629,6 +3978,12 @@ "parse-passwd": "1.0.0" } }, + "hosted-git-info": { + "version": "2.8.8", + "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.8.8.tgz", + "integrity": "sha512-f/wzC2QaWBs7t9IYqB4T3sR1xviIViXJRJTWBlx2Gf3g0Xi5vI7Yy4koXQ1c9OYDGHN9sBy1DQ2AB8fqZBWhUg==", + "dev": true + }, "hsl-regex": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/hsl-regex/-/hsl-regex-1.0.0.tgz", @@ -3647,6 +4002,17 @@ "integrity": "sha512-P+M65QY2JQ5Y0G9KKdlDpo0zK+/OHptU5AaBwUfAIDJZk1MYf32Frm84EcOytfJE0t5JvkAnKlmjsXDnWzCJmQ==", "dev": true }, + "http-signature": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz", + "integrity": "sha1-muzZJRFHcvPZW2WmCruPfBj7rOE=", + "dev": true, + "requires": { + "assert-plus": "1.0.0", + "jsprim": "1.4.1", + "sshpk": "1.16.1" + } + }, "https-browserify": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/https-browserify/-/https-browserify-1.0.0.tgz", @@ -3706,6 +4072,12 @@ "integrity": "sha1-khi5srkoojixPcT7a21XbyMUU+o=", "dev": true }, + "in-publish": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/in-publish/-/in-publish-2.0.1.tgz", + "integrity": "sha512-oDM0kUSNFC31ShNxHKUyfZKy8ZeXZBWMjMdZHKLOk13uvT27VTL/QzRGfRUcevJhpkZAvlhPYuXkF7eNWrtyxQ==", + "dev": true + }, "indent-string": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-4.0.0.tgz", @@ -3897,6 +4269,12 @@ "integrity": "sha1-qIwCU1eR8C7TfHahueqXc8gz+MI=", "dev": true }, + "is-finite": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/is-finite/-/is-finite-1.1.0.tgz", + "integrity": "sha512-cdyMtqX/BOqqNBBiKlIVkytNHm49MtMlYyn1zxzvJKWmFMlGzm+ry5BBfYyeY9YmNKbRSo/o7OX9w9ale0wg3w==", + "dev": true + }, "is-fullwidth-code-point": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", @@ -3992,6 +4370,18 @@ "has-symbols": "1.0.1" } }, + "is-typedarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", + "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=", + "dev": true + }, + "is-utf8": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/is-utf8/-/is-utf8-0.2.1.tgz", + "integrity": "sha1-Sw2hRCEE0bM2NA6AeX6GXPOffXI=", + "dev": true + }, "is-windows": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/is-windows/-/is-windows-1.0.2.tgz", @@ -4022,6 +4412,12 @@ "integrity": "sha1-TkMekrEalzFjaqH5yNHMvP2reN8=", "dev": true }, + "isstream": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", + "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=", + "dev": true + }, "jest-worker": { "version": "25.1.0", "resolved": "https://registry.npmjs.org/jest-worker/-/jest-worker-25.1.0.tgz", @@ -4049,6 +4445,12 @@ } } }, + "js-base64": { + "version": "2.5.2", + "resolved": "https://registry.npmjs.org/js-base64/-/js-base64-2.5.2.tgz", + "integrity": "sha512-Vg8czh0Q7sFBSUMWWArX/miJeBWYBPpdU/3M/DKSaekLMqrqVPaedp+5mZhie/r0lgrcaYBfwXatEew6gwgiQQ==", + "dev": true + }, "js-tokens": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", @@ -4065,6 +4467,12 @@ "esprima": "4.0.1" } }, + "jsbn": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz", + "integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM=", + "dev": true + }, "jsesc": { "version": "2.5.2", "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-2.5.2.tgz", @@ -4077,12 +4485,24 @@ "integrity": "sha512-mrqyZKfX5EhL7hvqcV6WG1yYjnjeuYDzDhhcAAUrq8Po85NBQBJP+ZDUT75qZQ98IkUoBqdkExkukOU7Ts2wrw==", "dev": true }, + "json-schema": { + "version": "0.2.3", + "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz", + "integrity": "sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM=", + "dev": true + }, "json-schema-traverse": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", "dev": true }, + "json-stringify-safe": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", + "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=", + "dev": true + }, "json5": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/json5/-/json5-2.1.2.tgz", @@ -4092,6 +4512,18 @@ "minimist": "1.2.5" } }, + "jsprim": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz", + "integrity": "sha1-MT5mvB5cwG5Di8G3SZwuXFastqI=", + "dev": true, + "requires": { + "assert-plus": "1.0.0", + "extsprintf": "1.3.0", + "json-schema": "0.2.3", + "verror": "1.10.0" + } + }, "kind-of": { "version": "6.0.3", "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", @@ -4132,6 +4564,36 @@ "leven": "3.1.0" } }, + "load-json-file": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-1.1.0.tgz", + "integrity": "sha1-lWkFcI1YtLq0wiYbBPWfMcmTdMA=", + "dev": true, + "requires": { + "graceful-fs": "4.2.3", + "parse-json": "2.2.0", + "pify": "2.3.0", + "pinkie-promise": "2.0.1", + "strip-bom": "2.0.0" + }, + "dependencies": { + "parse-json": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-2.2.0.tgz", + "integrity": "sha1-9ID0BDTvgHQfhGkJn43qGPVaTck=", + "dev": true, + "requires": { + "error-ex": "1.3.2" + } + }, + "pify": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", + "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=", + "dev": true + } + } + }, "loader-runner": { "version": "2.4.0", "resolved": "https://registry.npmjs.org/loader-runner/-/loader-runner-2.4.0.tgz", @@ -4197,6 +4659,16 @@ "js-tokens": "4.0.0" } }, + "loud-rejection": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/loud-rejection/-/loud-rejection-1.6.0.tgz", + "integrity": "sha1-W0b4AUft7leIcPCG0Eghz5mOVR8=", + "dev": true, + "requires": { + "currently-unhandled": "0.4.1", + "signal-exit": "3.0.2" + } + }, "lru-cache": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", @@ -4237,6 +4709,12 @@ "integrity": "sha1-wyq9C9ZSXZsFFkW7TyasXcmKDb8=", "dev": true }, + "map-obj": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/map-obj/-/map-obj-1.0.1.tgz", + "integrity": "sha1-2TPOuSBdgr3PSIb2dCvcK03qFG0=", + "dev": true + }, "map-visit": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/map-visit/-/map-visit-1.0.0.tgz", @@ -4284,6 +4762,24 @@ "readable-stream": "2.3.7" } }, + "meow": { + "version": "3.7.0", + "resolved": "https://registry.npmjs.org/meow/-/meow-3.7.0.tgz", + "integrity": "sha1-cstmi0JSKCkKu/qFaJJYcwioAfs=", + "dev": true, + "requires": { + "camelcase-keys": "2.1.0", + "decamelize": "1.2.0", + "loud-rejection": "1.6.0", + "map-obj": "1.0.1", + "minimist": "1.2.5", + "normalize-package-data": "2.5.0", + "object-assign": "4.1.1", + "read-pkg-up": "1.0.1", + "redent": "1.0.0", + "trim-newlines": "1.0.0" + } + }, "merge-stream": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz", @@ -4321,9 +4817,24 @@ "brorand": "1.1.0" } }, - "mimic-fn": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz", + "mime-db": { + "version": "1.43.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.43.0.tgz", + "integrity": "sha512-+5dsGEEovYbT8UY9yD7eE4XTc4UwJ1jBYlgaQQF38ENsKR3wj/8q8RFZrF9WIZpB2V1ArTVFUva8sAul1NzRzQ==", + "dev": true + }, + "mime-types": { + "version": "2.1.26", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.26.tgz", + "integrity": "sha512-01paPWYgLrkqAyrlDorC1uDwl2p3qZT7yl806vW7DvDoxwXi46jsjFbg+WdwotBIk6/MbEhO/dh5aZ5sNj/dWQ==", + "dev": true, + "requires": { + "mime-db": "1.43.0" + } + }, + "mimic-fn": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz", "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==", "dev": true }, @@ -4495,8 +5006,7 @@ "version": "2.14.0", "resolved": "https://registry.npmjs.org/nan/-/nan-2.14.0.tgz", "integrity": "sha512-INOFj37C7k3AfaNTtX8RhsTw7qRy7eLET14cROi9+5HAVbbHuIWUHEauBv5qT4Av2tWasiTY1Jw6puUNqRJXQg==", - "dev": true, - "optional": true + "dev": true }, "nanomatch": { "version": "1.2.13", @@ -4529,6 +5039,34 @@ "integrity": "sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ==", "dev": true }, + "node-gyp": { + "version": "3.8.0", + "resolved": "https://registry.npmjs.org/node-gyp/-/node-gyp-3.8.0.tgz", + "integrity": "sha512-3g8lYefrRRzvGeSowdJKAKyks8oUpLEd/DyPV4eMhVlhJ0aNaZqIrNUIPuEWWTAoPqyFkfGrM67MC69baqn6vA==", + "dev": true, + "requires": { + "fstream": "1.0.12", + "glob": "7.1.6", + "graceful-fs": "4.2.3", + "mkdirp": "0.5.4", + "nopt": "3.0.6", + "npmlog": "4.1.2", + "osenv": "0.1.5", + "request": "2.88.2", + "rimraf": "2.7.1", + "semver": "5.3.0", + "tar": "2.2.2", + "which": "1.3.1" + }, + "dependencies": { + "semver": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.3.0.tgz", + "integrity": "sha1-myzl094C0XxgEq0yaqa00M9U+U8=", + "dev": true + } + } + }, "node-libs-browser": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/node-libs-browser/-/node-libs-browser-2.2.1.tgz", @@ -4585,6 +5123,120 @@ } } }, + "node-sass": { + "version": "4.13.1", + "resolved": "https://registry.npmjs.org/node-sass/-/node-sass-4.13.1.tgz", + "integrity": "sha512-TTWFx+ZhyDx1Biiez2nB0L3YrCZ/8oHagaDalbuBSlqXgUPsdkUSzJsVxeDO9LtPB49+Fh3WQl3slABo6AotNw==", + "dev": true, + "requires": { + "async-foreach": "0.1.3", + "chalk": "1.1.3", + "cross-spawn": "3.0.1", + "gaze": "1.1.3", + "get-stdin": "4.0.1", + "glob": "7.1.6", + "in-publish": "2.0.1", + "lodash": "4.17.15", + "meow": "3.7.0", + "mkdirp": "0.5.4", + "nan": "2.14.0", + "node-gyp": "3.8.0", + "npmlog": "4.1.2", + "request": "2.88.2", + "sass-graph": "2.2.4", + "stdout-stream": "1.4.1", + "true-case-path": "1.0.3" + }, + "dependencies": { + "ansi-regex": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", + "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=", + "dev": true + }, + "ansi-styles": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz", + "integrity": "sha1-tDLdM1i2NM914eRmQ2gkBTPB3b4=", + "dev": true + }, + "chalk": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", + "integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=", + "dev": true, + "requires": { + "ansi-styles": "2.2.1", + "escape-string-regexp": "1.0.5", + "has-ansi": "2.0.0", + "strip-ansi": "3.0.1", + "supports-color": "2.0.0" + } + }, + "cross-spawn": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-3.0.1.tgz", + "integrity": "sha1-ElYDfsufDF9549bvE14wdwGEuYI=", + "dev": true, + "requires": { + "lru-cache": "4.1.5", + "which": "1.3.1" + } + }, + "lru-cache": { + "version": "4.1.5", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.1.5.tgz", + "integrity": "sha512-sWZlbEP2OsHNkXrMl5GYk/jKk70MBng6UU4YI/qGDYbgf6YbP4EvmqISbXCoJiRKs+1bSpFHVgQxvJ17F2li5g==", + "dev": true, + "requires": { + "pseudomap": "1.0.2", + "yallist": "2.1.2" + } + }, + "strip-ansi": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", + "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", + "dev": true, + "requires": { + "ansi-regex": "2.1.1" + } + }, + "supports-color": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz", + "integrity": "sha1-U10EXOa2Nj+kARcIRimZXp3zJMc=", + "dev": true + }, + "yallist": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz", + "integrity": "sha1-HBH5IY8HYImkfdUS+TxmmaaoHVI=", + "dev": true + } + } + }, + "nopt": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/nopt/-/nopt-3.0.6.tgz", + "integrity": "sha1-xkZdvwirzU2zWTF/eaxopkayj/k=", + "dev": true, + "requires": { + "abbrev": "1.1.1" + } + }, + "normalize-package-data": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/normalize-package-data/-/normalize-package-data-2.5.0.tgz", + "integrity": "sha512-/5CMN3T0R4XTj4DcGaexo+roZSdSFW/0AOOTROrjxzCG1wrWXEsGbRKevjlIL+ZDE4sZlJr5ED4YW0yqmkK+eA==", + "dev": true, + "requires": { + "hosted-git-info": "2.8.8", + "resolve": "1.15.1", + "semver": "5.7.1", + "validate-npm-package-license": "3.0.4" + } + }, "normalize-path": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", @@ -4612,6 +5264,18 @@ "path-key": "2.0.1" } }, + "npmlog": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/npmlog/-/npmlog-4.1.2.tgz", + "integrity": "sha512-2uUqazuKlTaSI/dC8AzicUck7+IrEaOnN/e0jd3Xtt1KcGpwx30v50mL7oPyr/h9bL3E4aZccVwpwP+5W9Vjkg==", + "dev": true, + "requires": { + "are-we-there-yet": "1.1.5", + "console-control-strings": "1.1.0", + "gauge": "2.7.4", + "set-blocking": "2.0.0" + } + }, "nth-check": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-1.0.2.tgz", @@ -4621,6 +5285,18 @@ "boolbase": "1.0.0" } }, + "number-is-nan": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz", + "integrity": "sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0=", + "dev": true + }, + "oauth-sign": { + "version": "0.9.0", + "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz", + "integrity": "sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==", + "dev": true + }, "object-assign": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", @@ -4747,6 +5423,12 @@ "integrity": "sha1-hUNzx/XCMVkU/Jv8a9gjj92h7Cc=", "dev": true }, + "os-homedir": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/os-homedir/-/os-homedir-1.0.2.tgz", + "integrity": "sha1-/7xJiDNuDoM94MFox+8VISGqf7M=", + "dev": true + }, "os-locale": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/os-locale/-/os-locale-3.1.0.tgz", @@ -4758,6 +5440,22 @@ "mem": "4.3.0" } }, + "os-tmpdir": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz", + "integrity": "sha1-u+Z0BseaqFxc/sdm/lc0VV36EnQ=", + "dev": true + }, + "osenv": { + "version": "0.1.5", + "resolved": "https://registry.npmjs.org/osenv/-/osenv-0.1.5.tgz", + "integrity": "sha512-0CWcCECdMVc2Rw3U5w9ZjqX6ga6ubk1xDVKxtBQPK7wis/0F2r9T6k4ydGYhecl7YUBxBVxhL5oisPsNxAPe2g==", + "dev": true, + "requires": { + "os-homedir": "1.0.2", + "os-tmpdir": "1.0.2" + } + }, "p-defer": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/p-defer/-/p-defer-1.0.0.tgz", @@ -4928,6 +5626,12 @@ "sha.js": "2.4.11" } }, + "performance-now": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", + "integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=", + "dev": true + }, "phoenix": { "version": "file:../deps/phoenix" }, @@ -4940,6 +5644,21 @@ "integrity": "sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g==", "dev": true }, + "pinkie": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/pinkie/-/pinkie-2.0.4.tgz", + "integrity": "sha1-clVrgM+g1IqXToDnckjoDtT3+HA=", + "dev": true + }, + "pinkie-promise": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/pinkie-promise/-/pinkie-promise-2.0.1.tgz", + "integrity": "sha1-ITXW36ejWMBprJsXh3YogihFD/o=", + "dev": true, + "requires": { + "pinkie": "2.0.4" + } + }, "pkg-dir": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-3.0.0.tgz", @@ -5555,6 +6274,18 @@ "integrity": "sha1-0/wRS6BplaRexok/SEzrHXj19HY=", "dev": true }, + "pseudomap": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz", + "integrity": "sha1-8FKijacOYYkX7wqKw0wa5aaChrM=", + "dev": true + }, + "psl": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/psl/-/psl-1.8.0.tgz", + "integrity": "sha512-RIdOzyoavK+hA18OGGWDqUTsCLhtA7IcZ/6NCs4fFJaHBDab+pDDmDIByWFRQJq2Cd7r1OoQxBGKOaztq+hjIQ==", + "dev": true + }, "public-encrypt": { "version": "4.0.3", "resolved": "https://registry.npmjs.org/public-encrypt/-/public-encrypt-4.0.3.tgz", @@ -5614,6 +6345,12 @@ "integrity": "sha1-fjL3W0E4EpHQRhHxvxQQmsAGUdc=", "dev": true }, + "qs": { + "version": "6.5.2", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz", + "integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==", + "dev": true + }, "query-string": { "version": "4.3.4", "resolved": "https://registry.npmjs.org/query-string/-/query-string-4.3.4.tgz", @@ -5655,6 +6392,67 @@ "safe-buffer": "5.1.2" } }, + "read-pkg": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-1.1.0.tgz", + "integrity": "sha1-9f+qXs0pyzHAR0vKfXVra7KePyg=", + "dev": true, + "requires": { + "load-json-file": "1.1.0", + "normalize-package-data": "2.5.0", + "path-type": "1.1.0" + }, + "dependencies": { + "path-type": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/path-type/-/path-type-1.1.0.tgz", + "integrity": "sha1-WcRPfuSR2nBNpBXaWkBwuk+P5EE=", + "dev": true, + "requires": { + "graceful-fs": "4.2.3", + "pify": "2.3.0", + "pinkie-promise": "2.0.1" + } + }, + "pify": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", + "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=", + "dev": true + } + } + }, + "read-pkg-up": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/read-pkg-up/-/read-pkg-up-1.0.1.tgz", + "integrity": "sha1-nWPBMnbAZZGNV/ACpX9AobZD+wI=", + "dev": true, + "requires": { + "find-up": "1.1.2", + "read-pkg": "1.1.0" + }, + "dependencies": { + "find-up": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-1.1.2.tgz", + "integrity": "sha1-ay6YIrGizgpgq2TWEOzK1TyyTQ8=", + "dev": true, + "requires": { + "path-exists": "2.1.0", + "pinkie-promise": "2.0.1" + } + }, + "path-exists": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-2.1.0.tgz", + "integrity": "sha1-D+tsZPD8UY2adU3V77YscCJ2H0s=", + "dev": true, + "requires": { + "pinkie-promise": "2.0.1" + } + } + } + }, "readable-stream": { "version": "2.3.7", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", @@ -5681,6 +6479,27 @@ "readable-stream": "2.3.7" } }, + "redent": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/redent/-/redent-1.0.0.tgz", + "integrity": "sha1-z5Fqsf1fHxbfsggi3W7H9zDCr94=", + "dev": true, + "requires": { + "indent-string": "2.1.0", + "strip-indent": "1.0.1" + }, + "dependencies": { + "indent-string": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-2.1.0.tgz", + "integrity": "sha1-ji1INIdCEhtKghi3oTfppSBJ3IA=", + "dev": true, + "requires": { + "repeating": "2.0.1" + } + } + } + }, "regenerate": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/regenerate/-/regenerate-1.4.0.tgz", @@ -5777,6 +6596,43 @@ "integrity": "sha1-jcrkcOHIirwtYA//Sndihtp15jc=", "dev": true }, + "repeating": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/repeating/-/repeating-2.0.1.tgz", + "integrity": "sha1-UhTFOpJtNVJwdSf7q0FdvAjQbdo=", + "dev": true, + "requires": { + "is-finite": "1.1.0" + } + }, + "request": { + "version": "2.88.2", + "resolved": "https://registry.npmjs.org/request/-/request-2.88.2.tgz", + "integrity": "sha512-MsvtOrfG9ZcrOwAW+Qi+F6HbD0CWXEh9ou77uOb7FM2WPhwT7smM833PzanhJLsgXjN89Ir6V2PczXNnMpwKhw==", + "dev": true, + "requires": { + "aws-sign2": "0.7.0", + "aws4": "1.9.1", + "caseless": "0.12.0", + "combined-stream": "1.0.8", + "extend": "3.0.2", + "forever-agent": "0.6.1", + "form-data": "2.3.3", + "har-validator": "5.1.3", + "http-signature": "1.2.0", + "is-typedarray": "1.0.0", + "isstream": "0.1.2", + "json-stringify-safe": "5.0.1", + "mime-types": "2.1.26", + "oauth-sign": "0.9.0", + "performance-now": "2.1.0", + "qs": "6.5.2", + "safe-buffer": "5.1.2", + "tough-cookie": "2.5.0", + "tunnel-agent": "0.6.0", + "uuid": "3.4.0" + } + }, "require-directory": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", @@ -5903,6 +6759,187 @@ "ret": "0.1.15" } }, + "safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "dev": true + }, + "sass-graph": { + "version": "2.2.4", + "resolved": "https://registry.npmjs.org/sass-graph/-/sass-graph-2.2.4.tgz", + "integrity": "sha1-E/vWPNHK8JCLn9k0dq1DpR0eC0k=", + "dev": true, + "requires": { + "glob": "7.1.6", + "lodash": "4.17.15", + "scss-tokenizer": "0.2.3", + "yargs": "7.1.0" + }, + "dependencies": { + "ansi-regex": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", + "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=", + "dev": true + }, + "camelcase": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-3.0.0.tgz", + "integrity": "sha1-MvxLn82vhF/N9+c7uXysImHwqwo=", + "dev": true + }, + "cliui": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-3.2.0.tgz", + "integrity": "sha1-EgYBU3qRbSmUD5NNo7SNWFo5IT0=", + "dev": true, + "requires": { + "string-width": "1.0.2", + "strip-ansi": "3.0.1", + "wrap-ansi": "2.1.0" + } + }, + "get-caller-file": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-1.0.3.tgz", + "integrity": "sha512-3t6rVToeoZfYSGd8YoLFR2DJkiQrIiUrGcjvFX2mDw3bn6k2OtwHN0TNCLbBO+w8qTvimhDkv+LSscbJY1vE6w==", + "dev": true + }, + "invert-kv": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/invert-kv/-/invert-kv-1.0.0.tgz", + "integrity": "sha1-EEqOSqym09jNFXqO+L+rLXo//bY=", + "dev": true + }, + "is-fullwidth-code-point": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz", + "integrity": "sha1-754xOG8DGn8NZDr4L95QxFfvAMs=", + "dev": true, + "requires": { + "number-is-nan": "1.0.1" + } + }, + "lcid": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/lcid/-/lcid-1.0.0.tgz", + "integrity": "sha1-MIrMr6C8SDo4Z7S28rlQYlHRuDU=", + "dev": true, + "requires": { + "invert-kv": "1.0.0" + } + }, + "os-locale": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/os-locale/-/os-locale-1.4.0.tgz", + "integrity": "sha1-IPnxeuKe00XoveWDsT0gCYA8FNk=", + "dev": true, + "requires": { + "lcid": "1.0.0" + } + }, + "require-main-filename": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-1.0.1.tgz", + "integrity": "sha1-l/cXtp1IeE9fUmpsWqj/3aBVpNE=", + "dev": true + }, + "string-width": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", + "integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=", + "dev": true, + "requires": { + "code-point-at": "1.1.0", + "is-fullwidth-code-point": "1.0.0", + "strip-ansi": "3.0.1" + } + }, + "strip-ansi": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", + "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", + "dev": true, + "requires": { + "ansi-regex": "2.1.1" + } + }, + "which-module": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/which-module/-/which-module-1.0.0.tgz", + "integrity": "sha1-u6Y8qGGUiZT/MHc2CJ47lgJsKk8=", + "dev": true + }, + "wrap-ansi": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-2.1.0.tgz", + "integrity": "sha1-2Pw9KE3QV5T+hJc8rs3Rz4JP3YU=", + "dev": true, + "requires": { + "string-width": "1.0.2", + "strip-ansi": "3.0.1" + } + }, + "y18n": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-3.2.1.tgz", + "integrity": "sha1-bRX7qITAhnnA136I53WegR4H+kE=", + "dev": true + }, + "yargs": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-7.1.0.tgz", + "integrity": "sha1-a6MY6xaWFyf10oT46gA+jWFU0Mg=", + "dev": true, + "requires": { + "camelcase": "3.0.0", + "cliui": "3.2.0", + "decamelize": "1.2.0", + "get-caller-file": "1.0.3", + "os-locale": "1.4.0", + "read-pkg-up": "1.0.1", + "require-directory": "2.1.1", + "require-main-filename": "1.0.1", + "set-blocking": "2.0.0", + "string-width": "1.0.2", + "which-module": "1.0.0", + "y18n": "3.2.1", + "yargs-parser": "5.0.0" + } + }, + "yargs-parser": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-5.0.0.tgz", + "integrity": "sha1-J17PDX/+Bcd+ZOfIbkzZS/DhIoo=", + "dev": true, + "requires": { + "camelcase": "3.0.0" + } + } + } + }, + "sass-loader": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/sass-loader/-/sass-loader-8.0.2.tgz", + "integrity": "sha512-7o4dbSK8/Ol2KflEmSco4jTjQoV988bM82P9CZdmo9hR3RLnvNc0ufMNdMrB0caq38JQ/FgF4/7RcbcfKzxoFQ==", + "dev": true, + "requires": { + "clone-deep": "4.0.1", + "loader-utils": "1.4.0", + "neo-async": "2.6.1", + "schema-utils": "2.6.5", + "semver": "6.3.0" + }, + "dependencies": { + "semver": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "dev": true + } + } + }, "sax": { "version": "1.2.4", "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz", @@ -5919,6 +6956,27 @@ "ajv-keywords": "3.4.1" } }, + "scss-tokenizer": { + "version": "0.2.3", + "resolved": "https://registry.npmjs.org/scss-tokenizer/-/scss-tokenizer-0.2.3.tgz", + "integrity": "sha1-jrBtualyMzOCTT9VMGQRSYR85dE=", + "dev": true, + "requires": { + "js-base64": "2.5.2", + "source-map": "0.4.4" + }, + "dependencies": { + "source-map": { + "version": "0.4.4", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.4.4.tgz", + "integrity": "sha1-66T12pwNyZneaAMti092FzZSA2s=", + "dev": true, + "requires": { + "amdefine": "1.0.1" + } + } + } + }, "semver": { "version": "5.7.1", "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", @@ -5976,6 +7034,15 @@ "safe-buffer": "5.1.2" } }, + "shallow-clone": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/shallow-clone/-/shallow-clone-3.0.1.tgz", + "integrity": "sha512-/6KqX+GVUdqPuPPd2LxDDxzX6CAbjJehAAOKlNpqqUpAqPM6HeL8f+o3a+JsyGjn2lv0WY8UsTgUJjU9Ok55NA==", + "dev": true, + "requires": { + "kind-of": "6.0.3" + } + }, "shebang-command": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-1.2.0.tgz", @@ -6200,6 +7267,38 @@ "integrity": "sha1-PpNdfd1zYxuXZZlW1VEo6HtQhKM=", "dev": true }, + "spdx-correct": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/spdx-correct/-/spdx-correct-3.1.0.tgz", + "integrity": "sha512-lr2EZCctC2BNR7j7WzJ2FpDznxky1sjfxvvYEyzxNyb6lZXHODmEoJeFu4JupYlkfha1KZpJyoqiJ7pgA1qq8Q==", + "dev": true, + "requires": { + "spdx-expression-parse": "3.0.0", + "spdx-license-ids": "3.0.5" + } + }, + "spdx-exceptions": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/spdx-exceptions/-/spdx-exceptions-2.2.0.tgz", + "integrity": "sha512-2XQACfElKi9SlVb1CYadKDXvoajPgBVPn/gOQLrTvHdElaVhr7ZEbqJaRnJLVNeaI4cMEAgVCeBMKF6MWRDCRA==", + "dev": true + }, + "spdx-expression-parse": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/spdx-expression-parse/-/spdx-expression-parse-3.0.0.tgz", + "integrity": "sha512-Yg6D3XpRD4kkOmTpdgbUiEJFKghJH03fiC1OPll5h/0sO6neh2jqRDVHOQ4o/LMea0tgCkbMgea5ip/e+MkWyg==", + "dev": true, + "requires": { + "spdx-exceptions": "2.2.0", + "spdx-license-ids": "3.0.5" + } + }, + "spdx-license-ids": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/spdx-license-ids/-/spdx-license-ids-3.0.5.tgz", + "integrity": "sha512-J+FWzZoynJEXGphVIS+XEh3kFSjZX/1i9gFBaWQcB+/tmpe2qUsSBABpcxqxnAxFdiUFEgAX1bjYGQvIZmoz9Q==", + "dev": true + }, "split-string": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/split-string/-/split-string-3.1.0.tgz", @@ -6215,6 +7314,23 @@ "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=", "dev": true }, + "sshpk": { + "version": "1.16.1", + "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.16.1.tgz", + "integrity": "sha512-HXXqVUq7+pcKeLqqZj6mHFUMvXtOJt1uoUx09pFW6011inTMxqI8BA8PM95myrIyyKwdnzjdFjLiE6KBPVtJIg==", + "dev": true, + "requires": { + "asn1": "0.2.4", + "assert-plus": "1.0.0", + "bcrypt-pbkdf": "1.0.2", + "dashdash": "1.14.1", + "ecc-jsbn": "0.1.2", + "getpass": "0.1.7", + "jsbn": "0.1.1", + "safer-buffer": "2.1.2", + "tweetnacl": "0.14.5" + } + }, "ssri": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/ssri/-/ssri-6.0.1.tgz", @@ -6251,6 +7367,15 @@ } } }, + "stdout-stream": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/stdout-stream/-/stdout-stream-1.4.1.tgz", + "integrity": "sha512-j4emi03KXqJWcIeF8eIXkjMFN1Cmb8gUlDYGeBALLPo5qdyTfA9bOtl8m33lRoC+vFMkP3gl0WsDr6+gzxbbTA==", + "dev": true, + "requires": { + "readable-stream": "2.3.7" + } + }, "stream-browserify": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/stream-browserify/-/stream-browserify-2.0.2.tgz", @@ -6345,12 +7470,30 @@ "ansi-regex": "4.1.0" } }, + "strip-bom": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-2.0.0.tgz", + "integrity": "sha1-YhmoVhZSBJHzV4i9vxRHqZx+aw4=", + "dev": true, + "requires": { + "is-utf8": "0.2.1" + } + }, "strip-eof": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/strip-eof/-/strip-eof-1.0.0.tgz", "integrity": "sha1-u0P/VZim6wXYm1n80SnJgzE2Br8=", "dev": true }, + "strip-indent": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/strip-indent/-/strip-indent-1.0.1.tgz", + "integrity": "sha1-DHlipq3vp7vUrDZkYKY4VSrhoKI=", + "dev": true, + "requires": { + "get-stdin": "4.0.1" + } + }, "stylehacks": { "version": "4.0.3", "resolved": "https://registry.npmjs.org/stylehacks/-/stylehacks-4.0.3.tgz", @@ -6411,6 +7554,17 @@ "integrity": "sha512-4WK/bYZmj8xLr+HUCODHGF1ZFzsYffasLUgEiMBY4fgtltdO6B4WJtlSbPaDTLpYTcGVwM2qLnFTICEcNxs3kA==", "dev": true }, + "tar": { + "version": "2.2.2", + "resolved": "https://registry.npmjs.org/tar/-/tar-2.2.2.tgz", + "integrity": "sha512-FCEhQ/4rE1zYv9rYXJw/msRqsnmlje5jHP6huWeBZ704jUTy02c5AZyWujpMR1ax6mVw9NyJMfuK2CMDWVIfgA==", + "dev": true, + "requires": { + "block-stream": "0.0.9", + "fstream": "1.0.12", + "inherits": "2.0.4" + } + }, "terser": { "version": "4.6.7", "resolved": "https://registry.npmjs.org/terser/-/terser-4.6.7.tgz", @@ -6639,6 +7793,31 @@ "repeat-string": "1.6.1" } }, + "tough-cookie": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.5.0.tgz", + "integrity": "sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g==", + "dev": true, + "requires": { + "psl": "1.8.0", + "punycode": "2.1.1" + } + }, + "trim-newlines": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/trim-newlines/-/trim-newlines-1.0.0.tgz", + "integrity": "sha1-WIeWa7WCpFA6QetST301ARgVphM=", + "dev": true + }, + "true-case-path": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/true-case-path/-/true-case-path-1.0.3.tgz", + "integrity": "sha512-m6s2OdQe5wgpFMC+pAJ+q9djG82O2jcHPOI6RNg1yy9rCYR+WD6Nbpl32fDpfC56nirdRy+opFa/Vk7HYhqaew==", + "dev": true, + "requires": { + "glob": "7.1.6" + } + }, "tslib": { "version": "1.11.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.11.1.tgz", @@ -6651,6 +7830,21 @@ "integrity": "sha1-oVe6QC2iTpv5V/mqadUk7tQpAaY=", "dev": true }, + "tunnel-agent": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", + "integrity": "sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=", + "dev": true, + "requires": { + "safe-buffer": "5.1.2" + } + }, + "tweetnacl": { + "version": "0.14.5", + "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", + "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=", + "dev": true + }, "typedarray": { "version": "0.0.6", "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz", @@ -6865,12 +8059,33 @@ "integrity": "sha512-CNmdbwQMBjwr9Gsmohvm0pbL954tJrNzf6gWL3K+QMQf00PF7ERGrEiLgjuU3mKreLC2MeGhUsNV9ybTbLgd3w==", "dev": true }, + "validate-npm-package-license": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/validate-npm-package-license/-/validate-npm-package-license-3.0.4.tgz", + "integrity": "sha512-DpKm2Ui/xN7/HQKCtpZxoRWBhZ9Z0kqtygG8XCgNQ8ZlDnxuQmWhj566j8fN4Cu3/JmbhsDo7fcAJq4s9h27Ew==", + "dev": true, + "requires": { + "spdx-correct": "3.1.0", + "spdx-expression-parse": "3.0.0" + } + }, "vendors": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/vendors/-/vendors-1.0.4.tgz", "integrity": "sha512-/juG65kTL4Cy2su4P8HjtkTxk6VmJDiOPBufWniqQ6wknac6jNiXS9vU+hO3wgusiyqWlzTbVHi0dyJqRONg3w==", "dev": true }, + "verror": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz", + "integrity": "sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA=", + "dev": true, + "requires": { + "assert-plus": "1.0.0", + "core-util-is": "1.0.2", + "extsprintf": "1.3.0" + } + }, "vm-browserify": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/vm-browserify/-/vm-browserify-1.1.2.tgz", @@ -7065,6 +8280,42 @@ "integrity": "sha1-2e8H3Od7mQK4o6j6SzHD4/fm6Ho=", "dev": true }, + "wide-align": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.3.tgz", + "integrity": "sha512-QGkOQc8XL6Bt5PwnsExKBPuMKBxnGxWWW3fU55Xt4feHozMUhdUMaBCk290qpm/wG5u/RSKzwdAC4i51YigihA==", + "dev": true, + "requires": { + "string-width": "2.1.1" + }, + "dependencies": { + "ansi-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.0.tgz", + "integrity": "sha1-7QMXwyIGT3lGbAKWa922Bas32Zg=", + "dev": true + }, + "string-width": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz", + "integrity": "sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw==", + "dev": true, + "requires": { + "is-fullwidth-code-point": "2.0.0", + "strip-ansi": "4.0.0" + } + }, + "strip-ansi": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz", + "integrity": "sha1-qEeQIusaw2iocTibY1JixQXuNo8=", + "dev": true, + "requires": { + "ansi-regex": "3.0.0" + } + } + } + }, "worker-farm": { "version": "1.7.0", "resolved": "https://registry.npmjs.org/worker-farm/-/worker-farm-1.7.0.tgz", diff --git a/assets/package.json b/assets/package.json index cd2bb9a..d4a6e27 100644 --- a/assets/package.json +++ b/assets/package.json @@ -16,7 +16,9 @@ "copy-webpack-plugin": "^5.1.1", "css-loader": "^3.4.2", "mini-css-extract-plugin": "^0.9.0", + "node-sass": "^4.13.1", "optimize-css-assets-webpack-plugin": "^5.0.1", + "sass-loader": "^8.0.2", "terser-webpack-plugin": "^2.3.2", "webpack": "4.41.5", "webpack-cli": "^3.3.2" diff --git a/assets/webpack.config.js b/assets/webpack.config.js index 4569a84..3cde0cb 100644 --- a/assets/webpack.config.js +++ b/assets/webpack.config.js @@ -29,8 +29,12 @@ module.exports = (env, options) => ({ } }, { - test: /\.css$/, - use: [MiniCssExtractPlugin.loader, 'css-loader'] + test: /\.scss$/, + use: [ + MiniCssExtractPlugin.loader, + 'css-loader', + 'sass-loader' + ] } ] }, diff --git a/lib/google_crawler_web/controllers/keyword_controller.ex b/lib/google_crawler_web/controllers/keyword_controller.ex new file mode 100644 index 0000000..7343d22 --- /dev/null +++ b/lib/google_crawler_web/controllers/keyword_controller.ex @@ -0,0 +1,11 @@ +defmodule GoogleCrawlerWeb.KeywordController do + use GoogleCrawlerWeb, :controller + + alias GoogleCrawler.Search + + def show(conn, %{"id" => id}) do + keyword = Search.get_keyword(id) + + render conn, "show.html", keyword: keyword + end +end diff --git a/lib/google_crawler_web/router.ex b/lib/google_crawler_web/router.ex index 7145612..50be996 100644 --- a/lib/google_crawler_web/router.ex +++ b/lib/google_crawler_web/router.ex @@ -28,6 +28,7 @@ defmodule GoogleCrawlerWeb.Router do resources "/sessions", SessionController, only: [:delete] resources "/upload", UploadController, only: [:create] + resources "/keywords", KeywordController, only: [:show] get "/", DashboardController, :index end diff --git a/lib/google_crawler_web/templates/dashboard/index.html.eex b/lib/google_crawler_web/templates/dashboard/index.html.eex index 16d85d2..47849ae 100644 --- a/lib/google_crawler_web/templates/dashboard/index.html.eex +++ b/lib/google_crawler_web/templates/dashboard/index.html.eex @@ -7,9 +7,21 @@ <%= if length(@keywords) == 0 do %>

<%= gettext("You don't have any keywords.") %>

<% else %> -
    +
      <%= for keyword <- @keywords do %> -
    • <%= keyword.keyword %>
      • +
    • +
      + <%= keyword.keyword %> +
      +
      + <%= keyword.status %> +
      +
      + <%= if keyword.status == :completed do %> + <%= link(gettext("View result"), to: Routes.keyword_path(@conn, :show, keyword)) %> + <% end %> +
      +
      • <% end %>
    <% end %> diff --git a/lib/google_crawler_web/templates/keyword/show.html.eex b/lib/google_crawler_web/templates/keyword/show.html.eex new file mode 100644 index 0000000..f297b04 --- /dev/null +++ b/lib/google_crawler_web/templates/keyword/show.html.eex @@ -0,0 +1,6 @@ +

    <%= gettext("Keyword: %{keyword}", keyword: @keyword.keyword) %>

    + +
    +

    Result:

    + +
    From 768c5dcc983c10e6932eab13a3f87bcba22f92b9 Mon Sep 17 00:00:00 2001 From: Rossukhon Leagmongkol Date: Wed, 1 Apr 2020 15:23:24 +0700 Subject: [PATCH 06/12] Display the raw result on the show page --- assets/css/app.scss | 3 ++- assets/css/phoenix.css | 3 +++ assets/css/screens/keyword.scss | 6 ++++++ lib/google_crawler/search/search_keyword_task.ex | 1 + lib/google_crawler_web/templates/layout/app.html.eex | 2 +- 5 files changed, 13 insertions(+), 2 deletions(-) create mode 100644 assets/css/screens/keyword.scss diff --git a/assets/css/app.scss b/assets/css/app.scss index 486fc72..8551a62 100644 --- a/assets/css/app.scss +++ b/assets/css/app.scss @@ -2,4 +2,5 @@ @import "./phoenix.css"; -@import "./screens/dashboard.scss" +@import "./screens/dashboard.scss"; +@import "./screens/keyword.scss"; diff --git a/assets/css/phoenix.css b/assets/css/phoenix.css index b8e45ec..9dbf047 100644 --- a/assets/css/phoenix.css +++ b/assets/css/phoenix.css @@ -98,6 +98,9 @@ header section { flex-direction: column; justify-content: space-between; } +header section a { + color: black; +} header nav ul, header nav li { margin: 0; diff --git a/assets/css/screens/keyword.scss b/assets/css/screens/keyword.scss new file mode 100644 index 0000000..ea55079 --- /dev/null +++ b/assets/css/screens/keyword.scss @@ -0,0 +1,6 @@ +body.keyword.show { + iframe { + width: 100%; + height: 400px; + } +} diff --git a/lib/google_crawler/search/search_keyword_task.ex b/lib/google_crawler/search/search_keyword_task.ex index dac4023..a6e21ba 100644 --- a/lib/google_crawler/search/search_keyword_task.ex +++ b/lib/google_crawler/search/search_keyword_task.ex @@ -12,6 +12,7 @@ defmodule GoogleCrawler.Search.SearchKeywordTask do {:ok, body} -> # Store the result to the db result = PageScrapper.scrap(body) + IO.inspect result Search.update_keyword(keyword, %{ status: :completed, raw_html_result: result.raw_html_result diff --git a/lib/google_crawler_web/templates/layout/app.html.eex b/lib/google_crawler_web/templates/layout/app.html.eex index e51913e..3b161ec 100644 --- a/lib/google_crawler_web/templates/layout/app.html.eex +++ b/lib/google_crawler_web/templates/layout/app.html.eex @@ -11,7 +11,7 @@
    -

    <%= gettext("Google Crawler") %>

    +

    <%= link gettext("Google Crawler"), to: "/" %>

      <%= if @conn.assigns.user_signed_in? do %> From 8e91089588e543d1890d18e3e1f4fbc1ca9d5b25 Mon Sep 17 00:00:00 2001 From: Rossukhon Leagmongkol Date: Wed, 1 Apr 2020 16:05:40 +0700 Subject: [PATCH 07/12] Add failed status and retry the job --- lib/google_crawler/search.ex | 3 +-- lib/google_crawler/search/keyword.ex | 2 +- lib/google_crawler/search/page_fetcher.ex | 3 ++- lib/google_crawler/search/search_keyword_task.ex | 7 ++----- lib/google_crawler/task.ex | 7 +++++++ 5 files changed, 13 insertions(+), 9 deletions(-) create mode 100644 lib/google_crawler/task.ex diff --git a/lib/google_crawler/search.ex b/lib/google_crawler/search.ex index fb7d471..cff4f19 100644 --- a/lib/google_crawler/search.ex +++ b/lib/google_crawler/search.ex @@ -72,8 +72,7 @@ defmodule GoogleCrawler.Search do def create_and_search_keyword(attrs \\ %{}, user) do case create_keyword(attrs, user) do {:ok, %Keyword{} = keyword} -> - search_task = fn -> GoogleCrawler.Search.SearchKeywordTask.perform(keyword) end - Task.Supervisor.start_child(GoogleCrawler.TaskSupervisor, search_task) + GoogleCrawler.Task.perform(GoogleCrawler.Search.SearchKeywordTask, keyword) {:ok, %Keyword{}} {:error, %Ecto.Changeset{} = changeset} -> diff --git a/lib/google_crawler/search/keyword.ex b/lib/google_crawler/search/keyword.ex index 85f6c69..50d38f9 100644 --- a/lib/google_crawler/search/keyword.ex +++ b/lib/google_crawler/search/keyword.ex @@ -1,6 +1,6 @@ import EctoEnum -defenum(GoogleCrawler.Search.Keyword.Status, in_queue: 0, in_progress: 1, completed: 2) +defenum(GoogleCrawler.Search.Keyword.Status, in_queue: 0, in_progress: 1, completed: 2, failed: 3) defmodule GoogleCrawler.Search.Keyword do use Ecto.Schema diff --git a/lib/google_crawler/search/page_fetcher.ex b/lib/google_crawler/search/page_fetcher.ex index a41394a..3d5964f 100644 --- a/lib/google_crawler/search/page_fetcher.ex +++ b/lib/google_crawler/search/page_fetcher.ex @@ -2,7 +2,8 @@ defmodule GoogleCrawler.Search.PageFetcher do @url "https://www.google.com/search?q=" def fetch(keyword) do - case HTTPoison.get(@url <> keyword) do + IO.puts "Performing search ... #{@url <> URI.encode(keyword)}" + case HTTPoison.get(@url <> URI.encode(keyword)) do {:ok, %HTTPoison.Response{status_code: 200, body: body}} -> {:ok, body} diff --git a/lib/google_crawler/search/search_keyword_task.ex b/lib/google_crawler/search/search_keyword_task.ex index a6e21ba..0b2f293 100644 --- a/lib/google_crawler/search/search_keyword_task.ex +++ b/lib/google_crawler/search/search_keyword_task.ex @@ -5,22 +5,19 @@ defmodule GoogleCrawler.Search.SearchKeywordTask do alias GoogleCrawler.Search.PageScrapper def perform(%Keyword{} = keyword) do - # Update the status to in progress Search.update_keyword(keyword, %{status: :in_progress}) case PageFetcher.fetch(keyword.keyword) do {:ok, body} -> - # Store the result to the db result = PageScrapper.scrap(body) - IO.inspect result Search.update_keyword(keyword, %{ status: :completed, raw_html_result: result.raw_html_result }) {:error, reason} -> - # TODO: Retry the task - IO.inspect(reason) + Search.update_keyword(keyword, %{ status: :failed }) + raise "Keyword search failed: #{reason}" end end end diff --git a/lib/google_crawler/task.ex b/lib/google_crawler/task.ex new file mode 100644 index 0000000..ea33d73 --- /dev/null +++ b/lib/google_crawler/task.ex @@ -0,0 +1,7 @@ +defmodule GoogleCrawler.Task do + def perform(task, args) do + Task.Supervisor.start_child(GoogleCrawler.TaskSupervisor, fn -> + task.perform(args) + end, restart: :transient) + end +end From e3ebe3c2150d055c7a0cf71284795fd8f867c7f1 Mon Sep 17 00:00:00 2001 From: Rossukhon Leagmongkol Date: Wed, 1 Apr 2020 19:42:56 +0700 Subject: [PATCH 08/12] Add genserver to process the keyword --- lib/google_crawler/application.ex | 5 +- lib/google_crawler/search.ex | 2 +- lib/google_crawler/search/keyword.ex | 2 +- lib/google_crawler/search/page_fetcher.ex | 1 - lib/google_crawler/search/page_scrapper.ex | 2 +- .../search/search_keyword_task.ex | 10 +-- .../search/search_keyword_worker.ex | 84 +++++++++++++++++++ lib/google_crawler/task.ex | 7 -- .../controllers/keyword_controller.ex | 2 +- test/google_crawler/search_test.exs | 19 ++++- 10 files changed, 107 insertions(+), 27 deletions(-) create mode 100644 lib/google_crawler/search/search_keyword_worker.ex delete mode 100644 lib/google_crawler/task.ex diff --git a/lib/google_crawler/application.ex b/lib/google_crawler/application.ex index 332b486..5ee420a 100644 --- a/lib/google_crawler/application.ex +++ b/lib/google_crawler/application.ex @@ -12,8 +12,9 @@ defmodule GoogleCrawler.Application do GoogleCrawler.Repo, # Start the endpoint when the application starts GoogleCrawlerWeb.Endpoint, - # The supervisor for crawler background job - {Task.Supervisor, name: GoogleCrawler.TaskSupervisor} + # Start the Google Crawler worker + {Task.Supervisor, name: GoogleCrawler.TaskSupervisor}, + GoogleCrawler.SearchKeywordWorker ] # See https://hexdocs.pm/elixir/Supervisor.html diff --git a/lib/google_crawler/search.ex b/lib/google_crawler/search.ex index cff4f19..4e88b88 100644 --- a/lib/google_crawler/search.ex +++ b/lib/google_crawler/search.ex @@ -72,7 +72,7 @@ defmodule GoogleCrawler.Search do def create_and_search_keyword(attrs \\ %{}, user) do case create_keyword(attrs, user) do {:ok, %Keyword{} = keyword} -> - GoogleCrawler.Task.perform(GoogleCrawler.Search.SearchKeywordTask, keyword) + GoogleCrawler.SearchKeywordWorker.search(keyword.id) {:ok, %Keyword{}} {:error, %Ecto.Changeset{} = changeset} -> diff --git a/lib/google_crawler/search/keyword.ex b/lib/google_crawler/search/keyword.ex index 50d38f9..8872947 100644 --- a/lib/google_crawler/search/keyword.ex +++ b/lib/google_crawler/search/keyword.ex @@ -1,6 +1,6 @@ import EctoEnum -defenum(GoogleCrawler.Search.Keyword.Status, in_queue: 0, in_progress: 1, completed: 2, failed: 3) +defenum(GoogleCrawler.Search.Keyword.Status, in_queue: 0, in_progress: 1, failed: 2, completed: 3) defmodule GoogleCrawler.Search.Keyword do use Ecto.Schema diff --git a/lib/google_crawler/search/page_fetcher.ex b/lib/google_crawler/search/page_fetcher.ex index 3d5964f..b9f5aa5 100644 --- a/lib/google_crawler/search/page_fetcher.ex +++ b/lib/google_crawler/search/page_fetcher.ex @@ -2,7 +2,6 @@ defmodule GoogleCrawler.Search.PageFetcher do @url "https://www.google.com/search?q=" def fetch(keyword) do - IO.puts "Performing search ... #{@url <> URI.encode(keyword)}" case HTTPoison.get(@url <> URI.encode(keyword)) do {:ok, %HTTPoison.Response{status_code: 200, body: body}} -> {:ok, body} diff --git a/lib/google_crawler/search/page_scrapper.ex b/lib/google_crawler/search/page_scrapper.ex index 083d1dc..fd0f612 100644 --- a/lib/google_crawler/search/page_scrapper.ex +++ b/lib/google_crawler/search/page_scrapper.ex @@ -10,6 +10,6 @@ defmodule GoogleCrawler.Search.PageScrapper do html |> String.chunk(:printable) |> Enum.filter(&String.printable?/1) - |> Enum.join + |> Enum.join() end end diff --git a/lib/google_crawler/search/search_keyword_task.ex b/lib/google_crawler/search/search_keyword_task.ex index 0b2f293..3664463 100644 --- a/lib/google_crawler/search/search_keyword_task.ex +++ b/lib/google_crawler/search/search_keyword_task.ex @@ -1,22 +1,14 @@ defmodule GoogleCrawler.Search.SearchKeywordTask do - alias GoogleCrawler.Search alias GoogleCrawler.Search.Keyword alias GoogleCrawler.Search.PageFetcher alias GoogleCrawler.Search.PageScrapper def perform(%Keyword{} = keyword) do - Search.update_keyword(keyword, %{status: :in_progress}) - case PageFetcher.fetch(keyword.keyword) do {:ok, body} -> - result = PageScrapper.scrap(body) - Search.update_keyword(keyword, %{ - status: :completed, - raw_html_result: result.raw_html_result - }) + PageScrapper.scrap(body) {:error, reason} -> - Search.update_keyword(keyword, %{ status: :failed }) raise "Keyword search failed: #{reason}" end end diff --git a/lib/google_crawler/search/search_keyword_worker.ex b/lib/google_crawler/search/search_keyword_worker.ex new file mode 100644 index 0000000..c827f9e --- /dev/null +++ b/lib/google_crawler/search/search_keyword_worker.ex @@ -0,0 +1,84 @@ +defmodule GoogleCrawler.SearchKeywordWorker do + use GenServer + + alias GoogleCrawler.Search + alias GoogleCrawler.Search.Keyword + + @max_retry_count 3 + + # Client + + def start_link(_args) do + GenServer.start_link(__MODULE__, %{}, name: __MODULE__) + end + + def search(keyword_id) do + GenServer.call(__MODULE__, {:search, keyword_id}) + end + + # Server Callbacks + + def init(state) do + {:ok, state} + end + + def handle_call({:search, keyword_id}, _from, state) do + IO.puts("Handle call #{keyword_id}") + + keyword = Search.get_keyword(keyword_id) + Search.update_keyword(keyword, %{status: :in_progress}) + + # start task and store the state of the task with the retry count + # as a tuple of {keyword, retry_count} -> {%Keyword{}, 1} + task = start_task(keyword) + new_state = Map.put(state, task.ref, {keyword, 0}) + + {:reply, :ok, new_state} + end + + def handle_info({ref, result}, state) do + IO.puts("Handle info | success") + + {keyword, _retry_count} = Map.get(state, ref) + + Search.update_keyword(keyword, %{ + status: :completed, + raw_html_result: result.raw_html_result + }) + + # Demonitor the task and remove from the state + Process.demonitor(ref, [:flush]) + new_state = Map.delete(state, ref) + + {:noreply, new_state} + end + + def handle_info({:DOWN, ref, :process, _pid, _reason}, state) do + IO.puts("Handle info | failed") + + {keyword, retry_count} = Map.get(state, ref) + + new_state = + if retry_count < @max_retry_count do + IO.puts("Retry... #{retry_count}") + task = start_task(keyword) + + state + |> Map.delete(ref) + |> Map.put(task.ref, {keyword, retry_count + 1}) + else + IO.puts("Done with failed...") + Search.update_keyword(keyword, %{status: :failed}) + + Map.delete(state, ref) + end + + {:noreply, new_state} + end + + defp start_task(%Keyword{} = keyword) do + Task.Supervisor.async_nolink(GoogleCrawler.TaskSupervisor, fn -> + GoogleCrawler.Search.SearchKeywordTask.perform(keyword) + end) + end +end diff --git a/lib/google_crawler/task.ex b/lib/google_crawler/task.ex deleted file mode 100644 index ea33d73..0000000 --- a/lib/google_crawler/task.ex +++ /dev/null @@ -1,7 +0,0 @@ -defmodule GoogleCrawler.Task do - def perform(task, args) do - Task.Supervisor.start_child(GoogleCrawler.TaskSupervisor, fn -> - task.perform(args) - end, restart: :transient) - end -end diff --git a/lib/google_crawler_web/controllers/keyword_controller.ex b/lib/google_crawler_web/controllers/keyword_controller.ex index 7343d22..cf68b3a 100644 --- a/lib/google_crawler_web/controllers/keyword_controller.ex +++ b/lib/google_crawler_web/controllers/keyword_controller.ex @@ -6,6 +6,6 @@ defmodule GoogleCrawlerWeb.KeywordController do def show(conn, %{"id" => id}) do keyword = Search.get_keyword(id) - render conn, "show.html", keyword: keyword + render(conn, "show.html", keyword: keyword) end end diff --git a/test/google_crawler/search_test.exs b/test/google_crawler/search_test.exs index 8856ecb..0e7987a 100644 --- a/test/google_crawler/search_test.exs +++ b/test/google_crawler/search_test.exs @@ -30,7 +30,7 @@ defmodule GoogleCrawler.SearchTest do assert {:ok, %Keyword{} = keyword} = Search.create_keyword(keyword_attrs, user) assert keyword.keyword == "elixir" - assert keyword.status == :in_queue + assert keyword.status == :created assert keyword.user_id == user.id end @@ -45,9 +45,20 @@ defmodule GoogleCrawler.SearchTest do keyword = KeywordFactory.create() keyword_attrs = %{keyword: "new", status: :in_progress} - assert {:ok, %Keyword{} = keyword} = Search.update_keyword(keyword, keyword_attrs) - assert keyword.keyword == "new" - assert keyword.status == :in_progress + # assert {:ok, %Keyword{} = keyword} = Search.update_keyword(keyword, keyword_attrs) + # assert keyword.keyword == "new" + # assert keyword.status == :in_progress + + Search.update_keyword(keyword, %{status: :in_progress}) + Search.update_keyword(keyword, %{status: :failed}) + Search.update_keyword(keyword, %{status: :in_progress}) + Search.update_keyword(keyword, %{status: :failed}) + Search.update_keyword(keyword, %{status: :in_progress}) + Search.update_keyword(keyword, %{status: :failed}) + Search.update_keyword(keyword, %{status: :in_progress}) + Search.update_keyword(keyword, %{status: :completed}) + + IO.inspect(Search.get_keyword(keyword.id)) end test "update_keyword/2 with invalid data returns error changeset" do From 489791aacb7bca9870afbbeac741651cbbbdb776 Mon Sep 17 00:00:00 2001 From: Rossukhon Leagmongkol Date: Thu, 2 Apr 2020 16:49:48 +0700 Subject: [PATCH 09/12] Add tests for the keyword and the search task --- config/config.exs | 2 + config/test.exs | 3 + .../page_fetcher.ex => google/api_client.ex} | 10 +- .../page_scrapper.ex => google/scrapper.ex} | 2 +- .../search/search_keyword_task.ex | 11 +- .../search/search_keyword_worker.ex | 10 +- test/fixtures/search_result.html | 206 ++++++++++++++++++ .../search/search_keyword_task_test.exs | 16 ++ .../search/search_keyword_worker_test.exs | 46 ++++ test/support/mocks/google_api_client.ex | 16 ++ 10 files changed, 314 insertions(+), 8 deletions(-) rename lib/google_crawler/{search/page_fetcher.ex => google/api_client.ex} (56%) rename lib/google_crawler/{search/page_scrapper.ex => google/scrapper.ex} (84%) create mode 100644 test/fixtures/search_result.html create mode 100644 test/google_crawler/search/search_keyword_task_test.exs create mode 100644 test/google_crawler/search/search_keyword_worker_test.exs create mode 100644 test/support/mocks/google_api_client.ex diff --git a/config/config.exs b/config/config.exs index 1c78a16..fbdbed1 100644 --- a/config/config.exs +++ b/config/config.exs @@ -18,6 +18,8 @@ config :google_crawler, GoogleCrawlerWeb.Endpoint, pubsub: [name: GoogleCrawler.PubSub, adapter: Phoenix.PubSub.PG2], live_view: [signing_salt: "7PRdGFq8"] +config :google_crawler, :google_api_client, GoogleCrawler.Google.ApiClient + # Configures Elixir's Logger config :logger, :console, format: "$time $metadata[$level] $message\n", diff --git a/config/test.exs b/config/test.exs index 9fe1c50..644033d 100644 --- a/config/test.exs +++ b/config/test.exs @@ -14,5 +14,8 @@ config :google_crawler, GoogleCrawlerWeb.Endpoint, http: [port: 4002], server: false +# Use Mock client for testing +config :google_crawler, :google_api_client, GoogleCrawler.Google.MockApiClient + # Print only warnings and errors during test config :logger, level: :warn diff --git a/lib/google_crawler/search/page_fetcher.ex b/lib/google_crawler/google/api_client.ex similarity index 56% rename from lib/google_crawler/search/page_fetcher.ex rename to lib/google_crawler/google/api_client.ex index b9f5aa5..245013f 100644 --- a/lib/google_crawler/search/page_fetcher.ex +++ b/lib/google_crawler/google/api_client.ex @@ -1,7 +1,13 @@ -defmodule GoogleCrawler.Search.PageFetcher do +defmodule GoogleCrawler.Google.ApiClientBehaviour do + @callback search(keyword :: String.t()) :: {:ok, String.t()} | {:error, String.t()} +end + +defmodule GoogleCrawler.Google.ApiClient do + @behaviour GoogleCrawler.Google.ApiClientBehaviour + @url "https://www.google.com/search?q=" - def fetch(keyword) do + def search(keyword) do case HTTPoison.get(@url <> URI.encode(keyword)) do {:ok, %HTTPoison.Response{status_code: 200, body: body}} -> {:ok, body} diff --git a/lib/google_crawler/search/page_scrapper.ex b/lib/google_crawler/google/scrapper.ex similarity index 84% rename from lib/google_crawler/search/page_scrapper.ex rename to lib/google_crawler/google/scrapper.ex index fd0f612..2940793 100644 --- a/lib/google_crawler/search/page_scrapper.ex +++ b/lib/google_crawler/google/scrapper.ex @@ -1,4 +1,4 @@ -defmodule GoogleCrawler.Search.PageScrapper do +defmodule GoogleCrawler.Google.Scrapper do def scrap(html) do # TODO: Scrap the page content %{ diff --git a/lib/google_crawler/search/search_keyword_task.ex b/lib/google_crawler/search/search_keyword_task.ex index 3664463..bd14ffa 100644 --- a/lib/google_crawler/search/search_keyword_task.ex +++ b/lib/google_crawler/search/search_keyword_task.ex @@ -1,15 +1,18 @@ defmodule GoogleCrawler.Search.SearchKeywordTask do alias GoogleCrawler.Search.Keyword - alias GoogleCrawler.Search.PageFetcher - alias GoogleCrawler.Search.PageScrapper + alias GoogleCrawler.Google.Scrapper def perform(%Keyword{} = keyword) do - case PageFetcher.fetch(keyword.keyword) do + case google_api_client().search(keyword.keyword) do {:ok, body} -> - PageScrapper.scrap(body) + Scrapper.scrap(body) {:error, reason} -> raise "Keyword search failed: #{reason}" end end + + defp google_api_client do + Application.get_env(:google_crawler, :google_api_client) + end end diff --git a/lib/google_crawler/search/search_keyword_worker.ex b/lib/google_crawler/search/search_keyword_worker.ex index c827f9e..293f6a5 100644 --- a/lib/google_crawler/search/search_keyword_worker.ex +++ b/lib/google_crawler/search/search_keyword_worker.ex @@ -16,6 +16,10 @@ defmodule GoogleCrawler.SearchKeywordWorker do GenServer.call(__MODULE__, {:search, keyword_id}) end + def get_state do + GenServer.call(__MODULE__, {:get_state}) + end + # Server Callbacks def init(state) do @@ -33,7 +37,11 @@ defmodule GoogleCrawler.SearchKeywordWorker do task = start_task(keyword) new_state = Map.put(state, task.ref, {keyword, 0}) - {:reply, :ok, new_state} + {:reply, task, new_state} + end + + def handle_call({:get_state}, _from, state) do + {:reply, state, state} end def handle_info({ref, result}, state) do diff --git a/test/fixtures/search_result.html b/test/fixtures/search_result.html new file mode 100644 index 0000000..fa793a8 --- /dev/null +++ b/test/fixtures/search_result.html @@ -0,0 +1,206 @@ + +google - ค้นหาด้วย Google

      ลิงก์เกี่ยวกับการช่วยเหลือพิเศษ

      ข้ามไปยังเนื้อหาหลักความช่วยเหลือเกี่ยวกับการช่วยเหลือพิเศษ
      ความคิดเห็นเกี่ยวกับการเข้าถึงพิเศษ
      ลงชื่อเข้าสู่ระบบ
      Google
      • ลบ
        • แจ้งการคาดคะเนที่ไม่เหมาะสม

        โหมดการค้นหา

        ทั้งหมด
        วิดีโอ
        ค้นรูป
        ข่าวสาร
        แผนที่
        เพิ่มเติม
        ช็อปปิ้งหนังสือเที่ยวบินการเงิน
        การตั้งค่า
        การตั้งค่าการค้นหาภาษา (Languages)
        เปิดการค้นหาปลอดภัย
        การค้นหาขั้นสูงกิจกรรมการค้นหาข้อมูลของคุณใน Searchความช่วยเหลือในการค้นหา
        เครื่องมือ
          ผลการค้นหาประมาณ 21,930,000,000 รายการ (0.42 วินาที) 
          Looking for results in English?
          Change to English
          ใช้ภาษาภาษาไทยต่อไป
          การตั้งค่าภาษา

          ผลการค้นหา

          ผลการค้นหาเว็บพร้อมไซต์ลิงก์


          Google

          www.google.co.th › ...
          www.google.co.th › ...
          1. แคช
          2. ใกล้เคียง
          บัญชี Google · Assistant · ค้นหา · Maps · YouTube ... ศิลปะวัฒนธรรม · เพิ่มเติมจาก Google · ลงชื่อเข้าสู่ระบบ. Google. ×. รายงานเรื่องนี้. ยกเลิก. ตกลง. ลบ. ไม่พบตำแหน่ง - -.

          ค้น รูป

          ส่งความคิดเห็น. ทั้งหมดภาพ. บัญชี Google · Assistant · ค้นหา · Maps ...

          Google Map

          Google Maps. การจราจร ขนส่ง จักรยาน ดาวเทียม ภูมิประเทศ ...
          ผลการค้นหาเพิ่มเติมจาก google.co.th »

          ผลการค้นเว็บ


          Google

          www.google.co.th
          www.google.co.th
          1. แคช
          2. ใกล้เคียง
          Search the world's information, including webpages, images, videos and more. Google has many special features to help you find exactly what you're looking ...

          เกี่ยวกับ - Google

          about.google › intl
          about.google › intl
          1. แคช
          พันธกิจของ Google คือการจัดระเบียบข้อมูลของโลก และทำให้ข้อมูลดังกล่าวสามารถเข้าถึงและเป็นประโยชน์ได้อย่างทั่วถึง เรียนรู้เกี่ยวกับประวัติบริษัท ผลิตภัณฑ์ของเรา และอื่นๆ.

          ผลิตภัณฑ์ของเรา - Google

          about.google › intl › ALL_th › products
          about.google › intl › ALL_th › products
          1. แคช
          ใช้งานได้ทุกที่. ระบบปฏิบัติการ Android ไอคอน; Wear OS by Google ไอคอน; Chromebook ไอคอน; Android ...

          Google Groups

          groups.google.com › homeredir
          groups.google.com › homeredir
          1. แคช
          2. ใกล้เคียง
          Google Groups ช่วยให้คุณสามารถสร้างและมีส่วนร่วมในฟอรัมออนไลน์และกลุ่มที่มีประสบการณ์อันมีค่าซึ่งติดต่อกันผ่านอีเมล เพื่อการสนทนากันในชุมชน.

          Google Account

          account.google.com
          account.google.com
          1. แคช
          แปลหน้านี้
          When you sign in to your Google Account, you can see and manage your info, activity, security options, and privacy preferences to make Google work better for ...

          The Keyword | Google

          www.blog.google
          www.blog.google
          1. แคช
          แปลหน้านี้
          Discover all the latest about our products, technology, and Google culture on our official blog.

          Welcome to My Activity - Google

          myactivity.google.com
          myactivity.google.com
          1. แคช
          2. ใกล้เคียง
          แปลหน้านี้
          Data helps make Google services more useful for you. Sign in to review and manage your activity, including things you've searched for, websites you've visited, ...

          Security - Google Account

          myaccount.google.com › intro › security
          myaccount.google.com › intro › security
          1. แคช
          2. ใกล้เคียง
          แปลหน้านี้
          To review and adjust your security settings and get recommendations to help you keep your account secure, sign in to your account. Sign in ...

          การค้นหาที่เกี่ยวข้องกับ google

          www.google.com th

          google.co.th หน้าแรก

          ยืนยันบัญชี google

          google แปลภาษา

          www.google.co.th เป็นหน้าแรกwww

          บริการของ google

          เกี่ยวกับ Google ทั้งหมด

          www.google.co.th เป็นหน้าแรก map

          การนำทางหน้าเว็บ

          12345678910ถัดไป

          ผลการค้นหาเสริม

          ผลการค้นหาความรู้

          กูเกิล
          บริษัท

          คำอธิบาย

          กูเกิล เป็นบริษัทมหาชนอเมริกัน มีรายได้หลักจากการโฆษณาออนไลน์ที่ปรากฏในเสิร์ชเอนจินของกูเกิล อีเมล แผนที่ออนไลน์ ซอฟต์แวร์จัดการด้านสำนักงาน เครือข่ายออนไลน์ และวิดีโอออนไลน์ รวมถึงการขายอุปกรณ์ช่วยในการค้นหา กูเกิลสำนักงานใหญ่ที่รู้จักในชื่อกูเกิลเพล็กซ์ตั้งอยู่ที่เมืองเมาน์เทนวิว รัฐแคลิฟอร์เนีย โดยมีพนักงาน 16,805 ... วิกิพีเดีย
          ประธานเจ้าหน้าที่บริหาร: ศุนทัร ปิจไช (2 ต.ค. 2558–) กำลังมาแรง
          ก่อตั้ง: 4 กันยายน 2541, เมนโลพาร์ก, แคลิฟอร์เนีย, อเมริกา
          สำนักงานใหญ่: เมาน์เทนวิว, แคลิฟอร์เนีย, อเมริกา
          องค์กรหลัก: อัลฟาเบท (2015–)
          บริษัทในเครือ: YouTube, Google.org, เนสท์ แล็บส์, YouTube TV, เพิ่มเติม
          ผู้ก่อตั้ง: แลร์รี เพจ, เซอร์เกย์ บริน
          และผู้คนยังค้นหา
          ดูอีกกว่า 15 รายการดูอีกกว่า 15 รายการ
          จีเมล
          จีเมล
          อินสตาแกรม
          อินสตาแกรม
          ทวิตเตอร์
          ทวิตเตอร์
          เน็ตฟลิกซ์
          เน็ตฟลิกซ์
          ยาฮู!
          ยาฮู!
          คลิกที่ข้อผิดพลาด
          หากไม่มีข้อผิดพลาด โปรดแสดงความคิดเห็นทั่วไป
          ความคิดเห็น
          ข้อจำกัดความรับผิดชอบ

          ลิงก์ส่วนท้าย

          ไทย
          - -  - ดูข้อมูลเพิ่มเติม
          ความช่วยเหลือส่งความคิดเห็นความเป็นส่วนตัวข้อกำหนด
          แอป Google
          diff --git a/test/google_crawler/search/search_keyword_task_test.exs b/test/google_crawler/search/search_keyword_task_test.exs new file mode 100644 index 0000000..f1aa04b --- /dev/null +++ b/test/google_crawler/search/search_keyword_task_test.exs @@ -0,0 +1,16 @@ +defmodule GoogleCrawler.Search.SearchKeywordTaskTest do + use ExUnit.Case + + alias GoogleCrawler.Search.Keyword + alias GoogleCrawler.Search.SearchKeywordTask + + test "perform/1 returns the scapping result when it is success" do + assert %{raw_html_result: result} = SearchKeywordTask.perform(%Keyword{keyword: "elixir"}) + end + + test "perform/1 raises an error reason when it is failed" do + assert_raise RuntimeError, fn -> + SearchKeywordTask.perform(%Keyword{keyword: "error"}) + end + end +end diff --git a/test/google_crawler/search/search_keyword_worker_test.exs b/test/google_crawler/search/search_keyword_worker_test.exs new file mode 100644 index 0000000..0260b6d --- /dev/null +++ b/test/google_crawler/search/search_keyword_worker_test.exs @@ -0,0 +1,46 @@ +defmodule GoogleCrawler.Search.SearchKeywordWorkerTest do + use ExUnit.Case + use GoogleCrawler.DataCase + + alias GoogleCrawler.SearchKeywordWorker + alias GoogleCrawler.Search + alias GoogleCrawler.KeywordFactory + + test "search/1 performs search task with the given keyword id" do + keyword = KeywordFactory.create(%{}) + + task = SearchKeywordWorker.search(keyword.id) + task_ref = task.ref + + assert %{^task_ref => {keyword, 0}} = SearchKeywordWorker.get_state() + assert Search.get_keyword(keyword.id).status == :in_progress + end + + test "search/1 updates the keyword result when the task is completed" do + keyword = KeywordFactory.create(%{}) + + SearchKeywordWorker.search(keyword.id) + + # Find a way to test without sleep 😔 + :timer.sleep(1000) + + keyword = Search.get_keyword(keyword.id) + assert keyword.status == :completed + assert keyword.raw_html_result != nil + + assert SearchKeywordWorker.get_state() == %{} + end + + test "search/1 updates the keyword status to failed when the task is failed" do + keyword = KeywordFactory.create(%{keyword: "error"}) + + task = SearchKeywordWorker.search(keyword.id) + task_ref = task.ref + + # Find a way to test without sleep 😔 + :timer.sleep(1000) + + assert Search.get_keyword(keyword.id).status == :failed + assert SearchKeywordWorker.get_state() == %{} + end +end diff --git a/test/support/mocks/google_api_client.ex b/test/support/mocks/google_api_client.ex new file mode 100644 index 0000000..bf50d94 --- /dev/null +++ b/test/support/mocks/google_api_client.ex @@ -0,0 +1,16 @@ +defmodule GoogleCrawler.Google.MockApiClient do + @behaviour GoogleCrawler.Google.ApiClientBehaviour + + def search("error") do + {:error, "Bad request"} + end + + def search(_keyword) do + {:ok, response_fixtures('search_result.html')} + end + + defp response_fixtures(path) do + Path.join(["test/fixtures", path]) + |> File.read!() + end +end From 9b3fd44bf2bb56d069df55f1f32494d865ab8828 Mon Sep 17 00:00:00 2001 From: Rossukhon Leagmongkol Date: Thu, 2 Apr 2020 17:15:24 +0700 Subject: [PATCH 10/12] Remove logging message and update tests --- .../search/search_keyword_worker.ex | 9 --------- .../search/search_keyword_worker_test.exs | 3 +++ test/google_crawler/search_test.exs | 19 ++++--------------- 3 files changed, 7 insertions(+), 24 deletions(-) diff --git a/lib/google_crawler/search/search_keyword_worker.ex b/lib/google_crawler/search/search_keyword_worker.ex index 293f6a5..f2ed80d 100644 --- a/lib/google_crawler/search/search_keyword_worker.ex +++ b/lib/google_crawler/search/search_keyword_worker.ex @@ -27,8 +27,6 @@ defmodule GoogleCrawler.SearchKeywordWorker do end def handle_call({:search, keyword_id}, _from, state) do - IO.puts("Handle call #{keyword_id}") - keyword = Search.get_keyword(keyword_id) Search.update_keyword(keyword, %{status: :in_progress}) @@ -45,8 +43,6 @@ defmodule GoogleCrawler.SearchKeywordWorker do end def handle_info({ref, result}, state) do - IO.puts("Handle info | success") - {keyword, _retry_count} = Map.get(state, ref) Search.update_keyword(keyword, %{ @@ -62,22 +58,17 @@ defmodule GoogleCrawler.SearchKeywordWorker do end def handle_info({:DOWN, ref, :process, _pid, _reason}, state) do - IO.puts("Handle info | failed") - {keyword, retry_count} = Map.get(state, ref) new_state = if retry_count < @max_retry_count do - IO.puts("Retry... #{retry_count}") task = start_task(keyword) state |> Map.delete(ref) |> Map.put(task.ref, {keyword, retry_count + 1}) else - IO.puts("Done with failed...") Search.update_keyword(keyword, %{status: :failed}) - Map.delete(state, ref) end diff --git a/test/google_crawler/search/search_keyword_worker_test.exs b/test/google_crawler/search/search_keyword_worker_test.exs index 0260b6d..db0fcd8 100644 --- a/test/google_crawler/search/search_keyword_worker_test.exs +++ b/test/google_crawler/search/search_keyword_worker_test.exs @@ -14,6 +14,9 @@ defmodule GoogleCrawler.Search.SearchKeywordWorkerTest do assert %{^task_ref => {keyword, 0}} = SearchKeywordWorker.get_state() assert Search.get_keyword(keyword.id).status == :in_progress + + :timer.sleep(1000) + assert SearchKeywordWorker.get_state() == %{} end test "search/1 updates the keyword result when the task is completed" do diff --git a/test/google_crawler/search_test.exs b/test/google_crawler/search_test.exs index 0e7987a..8856ecb 100644 --- a/test/google_crawler/search_test.exs +++ b/test/google_crawler/search_test.exs @@ -30,7 +30,7 @@ defmodule GoogleCrawler.SearchTest do assert {:ok, %Keyword{} = keyword} = Search.create_keyword(keyword_attrs, user) assert keyword.keyword == "elixir" - assert keyword.status == :created + assert keyword.status == :in_queue assert keyword.user_id == user.id end @@ -45,20 +45,9 @@ defmodule GoogleCrawler.SearchTest do keyword = KeywordFactory.create() keyword_attrs = %{keyword: "new", status: :in_progress} - # assert {:ok, %Keyword{} = keyword} = Search.update_keyword(keyword, keyword_attrs) - # assert keyword.keyword == "new" - # assert keyword.status == :in_progress - - Search.update_keyword(keyword, %{status: :in_progress}) - Search.update_keyword(keyword, %{status: :failed}) - Search.update_keyword(keyword, %{status: :in_progress}) - Search.update_keyword(keyword, %{status: :failed}) - Search.update_keyword(keyword, %{status: :in_progress}) - Search.update_keyword(keyword, %{status: :failed}) - Search.update_keyword(keyword, %{status: :in_progress}) - Search.update_keyword(keyword, %{status: :completed}) - - IO.inspect(Search.get_keyword(keyword.id)) + assert {:ok, %Keyword{} = keyword} = Search.update_keyword(keyword, keyword_attrs) + assert keyword.keyword == "new" + assert keyword.status == :in_progress end test "update_keyword/2 with invalid data returns error changeset" do From b8b20f079cc6984c7fc62bd7fe11a1fbf6166ae4 Mon Sep 17 00:00:00 2001 From: Rossukhon Leagmongkol Date: Thu, 2 Apr 2020 17:36:32 +0700 Subject: [PATCH 11/12] Add test for create and search keyword function --- test/google_crawler/search_test.exs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/test/google_crawler/search_test.exs b/test/google_crawler/search_test.exs index 8856ecb..52c9b49 100644 --- a/test/google_crawler/search_test.exs +++ b/test/google_crawler/search_test.exs @@ -41,6 +41,21 @@ defmodule GoogleCrawler.SearchTest do assert {:error, %Ecto.Changeset{}} = Search.create_keyword(keyword_attrs, user) end + test "create_and_search_keyword/2 with valid data creates the keyword and triggers the task to search the keyword" do + user = UserFactory.create() + keyword_attrs = KeywordFactory.build_attrs() + + assert {:ok, %Keyword{}} = Search.create_and_search_keyword(keyword_attrs, user) + assert GoogleCrawler.SearchKeywordWorker.get_state() != %{} + end + + test "create_and_search_keyword/2 with invalid data creates the keyword and triggers the task to search the keyword" do + user = UserFactory.create() + keyword_attrs = KeywordFactory.build_attrs(%{keyword: ""}) + + assert {:error, %Ecto.Changeset{}} = Search.create_and_search_keyword(keyword_attrs, user) + end + test "update_keyword/2 with valid data updates the keyword" do keyword = KeywordFactory.create() keyword_attrs = %{keyword: "new", status: :in_progress} From 98c3e6c771aa6f86ac5aad9a14dc802b0504dc14 Mon Sep 17 00:00:00 2001 From: Rossukhon Leagmongkol Date: Fri, 3 Apr 2020 10:36:39 +0700 Subject: [PATCH 12/12] Fix typo --- test/google_crawler/search/search_keyword_task_test.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/google_crawler/search/search_keyword_task_test.exs b/test/google_crawler/search/search_keyword_task_test.exs index f1aa04b..d8f7d39 100644 --- a/test/google_crawler/search/search_keyword_task_test.exs +++ b/test/google_crawler/search/search_keyword_task_test.exs @@ -4,7 +4,7 @@ defmodule GoogleCrawler.Search.SearchKeywordTaskTest do alias GoogleCrawler.Search.Keyword alias GoogleCrawler.Search.SearchKeywordTask - test "perform/1 returns the scapping result when it is success" do + test "perform/1 returns the scrapping result when it is success" do assert %{raw_html_result: result} = SearchKeywordTask.perform(%Keyword{keyword: "elixir"}) end