From 8d6e759a96d7a83eb2525dfb61c400d6ec50f892 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafael=20Eppl=C3=A9e?= Date: Mon, 24 Nov 2025 17:40:30 +0100 Subject: [PATCH 1/6] Add planning docs for search --- doc/search.md | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 doc/search.md diff --git a/doc/search.md b/doc/search.md new file mode 100644 index 0000000..b63967a --- /dev/null +++ b/doc/search.md @@ -0,0 +1,47 @@ +# Search + +Design document for full-text search through + +- bookmark titles +- content of bookmarked html websites +- list titles +- list descriptions (called "content" in the code) + +## Requirements + +- No extra service to deploy +- Up to a certain size, search should take <500ms and be a lot faster for small datasets +- Index size should be reasonable, e.g. not more than 2x of original content +- target: 50 users per instance with 50k bookmarks each should be easy for hosters +- need to return matched positions for highlighting them in search results +- should support language-aware stemming +- Should have a "good" way to rank results + +## "Good" Ranking + +- Should some level of fuzziness be involved? +- Weight matches in the bookmark title higher than website content +- BM25 is a good baseline + +## PostgreSQL full-text-search + +- Seems like the ranking functions `ts_rank` and `ts_rank_cd` have pretty heavy performance impact, although for the size of linkblocks this might not be a problem +- No BM25 ranking, but it can at least normalize word frequency by document length +- tsvectors can only take a limited number of lexeme positions (but an unlimited number of lexemes?) +- Can rank different parts of the tsvector differently (title, body) +- quicker to implement than Tantivy +- Let's evaluate how good the ranking actually is + +## Tantivy + +- Uses BM25 for ranking +- Ranking is faster than with Postgres +- Extra implementation complexity if indexes are cached on disk, or extra memory & compute required if they are stored in-memory + - Check the indexing performance & space requirements +- For on-disk storage, needs extra work to robustly handle file corruption, recovery from bugs, etc. + - Alternatively: store indexes in postgres + +## pg_search + +- Implements BM25 and more in postgres using tantivy +- [Requires custom postgres installation](https://github.com/paradedb/paradedb/tree/main/pg_search#installation) From f88cb1ab2335cc395c4121cf7bdf139599f6f878 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafael=20Eppl=C3=A9e?= Date: Wed, 26 Nov 2025 12:55:24 +0100 Subject: [PATCH 2/6] Add RequestBuilder::visit_link --- src/tests/util/request_builder.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/tests/util/request_builder.rs b/src/tests/util/request_builder.rs index b62d2d5..081d6b3 100644 --- a/src/tests/util/request_builder.rs +++ b/src/tests/util/request_builder.rs @@ -10,6 +10,7 @@ use tower::{Service, ServiceExt}; use visdom::Vis; use super::dom::assert_form_matches; +use crate::tests::util::{html_decode::html_decode, test_app::TestApp}; pub struct RequestBuilder { router: axum::Router, @@ -171,4 +172,17 @@ impl TestPage { self.request_builder = self.request_builder.expect_status(expected); self } + + pub async fn visit_link(&self, app: &mut TestApp, text_contains: &str) -> TestPage { + let url = self + .dom + .find("a") + .filter_by(|_, a| a.html().contains(text_contains)) + .attr("href") + .unwrap() + .to_string(); + let url = html_decode(&url); + + app.req().get(&url).await.test_page().await + } } From 35a5817d3c7e42f2a8b3fe3c9dbd09639e5bbb12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafael=20Eppl=C3=A9e?= Date: Wed, 26 Nov 2025 12:55:24 +0100 Subject: [PATCH 3/6] TestApp: Log out before logging in --- src/tests/util/test_app.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/tests/util/test_app.rs b/src/tests/util/test_app.rs index 5f02b9e..24ab4e1 100644 --- a/src/tests/util/test_app.rs +++ b/src/tests/util/test_app.rs @@ -107,6 +107,7 @@ impl TestApp { } pub async fn login_user(&mut self, username: &str, password: &str) { + self.logged_in_cookie = None; let login_page = self.req().get("/login").await.test_page().await; let input = crate::forms::users::Login { From 8e0a6aee74ce8fbbd79b1123b5e0603f3cd0e1b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafael=20Eppl=C3=A9e?= Date: Fri, 28 Nov 2025 15:58:01 +0100 Subject: [PATCH 4/6] Support GET forms in fill_form test helper --- src/tests/util/request_builder.rs | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/src/tests/util/request_builder.rs b/src/tests/util/request_builder.rs index 081d6b3..4ce8abf 100644 --- a/src/tests/util/request_builder.rs +++ b/src/tests/util/request_builder.rs @@ -156,6 +156,7 @@ impl TestResponse { pub struct TestPage { pub dom: visdom::types::Elements<'static>, + #[expect(dead_code)] pub url: String, pub request_builder: RequestBuilder, } @@ -163,9 +164,29 @@ pub struct TestPage { impl TestPage { pub async fn fill_form(self, form_selector: &str, input: &I) -> TestResponse { let form = self.dom.find(form_selector); + let method = form + .attr("method") + .map_or("post".to_string(), |val| val.to_string()) + .to_lowercase(); + + let action = form + .attr("action") + .expect("Missing action attribute for form {form:?}") + .to_string(); assert_form_matches(&form, &input); - self.request_builder.post(&self.url, input).await + match method.as_str() { + "post" => self.request_builder.post(&action, input).await, + "get" => { + let queries = serde_qs::to_string(input).expect("Failed to serialize input"); + let url = format!("{action}?{queries}"); + self.request_builder.get(&url).await + } + _ => panic!( + "Unsupported method {method} for form with action {:?}", + form.attr("action") + ), + } } pub fn expect_status(mut self, expected: StatusCode) -> Self { From 3fc1ff1a2ce14afcf4c9589a9e1fa0f9269adddf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafael=20Eppl=C3=A9e?= Date: Fri, 28 Nov 2025 15:58:01 +0100 Subject: [PATCH 5/6] Persist logged in cookie for requests dispatched by TestPage --- src/tests/util/request_builder.rs | 33 ++++++++++++++++++------------- src/tests/util/test_app.rs | 6 +----- 2 files changed, 20 insertions(+), 19 deletions(-) diff --git a/src/tests/util/request_builder.rs b/src/tests/util/request_builder.rs index 4ce8abf..654df0f 100644 --- a/src/tests/util/request_builder.rs +++ b/src/tests/util/request_builder.rs @@ -10,7 +10,7 @@ use tower::{Service, ServiceExt}; use visdom::Vis; use super::dom::assert_form_matches; -use crate::tests::util::{html_decode::html_decode, test_app::TestApp}; +use crate::tests::util::html_decode::html_decode; pub struct RequestBuilder { router: axum::Router, @@ -18,14 +18,16 @@ pub struct RequestBuilder { /// If it returns a different status, we'll panic. expected_status: StatusCode, request: request::Builder, + logged_in_cookie: Option, } impl RequestBuilder { - pub fn new(router: &Router) -> Self { + pub fn new(router: &Router, logged_in_cookie: Option) -> Self { RequestBuilder { router: router.clone(), expected_status: StatusCode::OK, request: Request::builder(), + logged_in_cookie, } } @@ -47,6 +49,10 @@ impl RequestBuilder { where Input: Serialize, { + if let Some(cookie) = &self.logged_in_cookie { + self.request = self.request.header(axum::http::header::COOKIE, cookie); + } + let request = self .request .method(http::Method::POST) @@ -71,12 +77,15 @@ impl RequestBuilder { TestResponse { response, - router: self.router, - original_url: url.to_string(), + new_request_builder: RequestBuilder::new(&self.router, self.logged_in_cookie), } } pub async fn get(mut self, url: &str) -> TestResponse { + if let Some(cookie) = &self.logged_in_cookie { + self.request = self.request.header(axum::http::header::COOKIE, cookie); + } + let request = self.request.uri(url).body(Body::empty()).unwrap(); let response = ServiceExt::>::ready(&mut self.router) @@ -92,8 +101,7 @@ impl RequestBuilder { TestResponse { response, - router: self.router, - original_url: url.to_string(), + new_request_builder: RequestBuilder::new(&self.router, self.logged_in_cookie), } } @@ -113,8 +121,7 @@ impl RequestBuilder { pub struct TestResponse { response: Response, - original_url: String, - router: axum::Router, + new_request_builder: RequestBuilder, } impl TestResponse { @@ -148,16 +155,14 @@ impl TestResponse { TestPage { dom, - url: self.original_url, - request_builder: RequestBuilder::new(&self.router), + // TODO this doesn't persist the previous login cookie + request_builder: self.new_request_builder, } } } pub struct TestPage { pub dom: visdom::types::Elements<'static>, - #[expect(dead_code)] - pub url: String, pub request_builder: RequestBuilder, } @@ -194,7 +199,7 @@ impl TestPage { self } - pub async fn visit_link(&self, app: &mut TestApp, text_contains: &str) -> TestPage { + pub async fn visit_link(self, text_contains: &str) -> TestPage { let url = self .dom .find("a") @@ -204,6 +209,6 @@ impl TestPage { .to_string(); let url = html_decode(&url); - app.req().get(&url).await.test_page().await + self.request_builder.get(&url).await.test_page().await } } diff --git a/src/tests/util/test_app.rs b/src/tests/util/test_app.rs index 24ab4e1..86a5f18 100644 --- a/src/tests/util/test_app.rs +++ b/src/tests/util/test_app.rs @@ -66,11 +66,7 @@ impl TestApp { } pub fn req(&mut self) -> RequestBuilder { - let mut req = RequestBuilder::new(&self.router); - if let Some(cookie) = &self.logged_in_cookie { - req = req.header(axum::http::header::COOKIE, cookie); - } - req + RequestBuilder::new(&self.router, self.logged_in_cookie.clone()) } /// Since there's no route for creating users yet, we're doing this via the From bf22e99f99bdaafb68e2be0a388b7c7f8889024f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafael=20Eppl=C3=A9e?= Date: Mon, 24 Nov 2025 14:55:35 +0100 Subject: [PATCH 6/6] Add simple bookmark title search --- ...c37c2286cbd49d488f20fa2952214958f004b.json | 24 ++ ...410170aedfdeddd8e9198e0832a7a70745422.json | 36 +++ ...bdc68ff24588473338219d8144a5e5ab15dd2.json | 24 ++ .vscode/settings.json | 6 +- Cargo.lock | 23 ++ Cargo.toml | 1 + Justfile | 2 +- src/db/mod.rs | 1 + src/db/search.rs | 106 ++++++++ src/routes/mod.rs | 1 + src/routes/search.rs | 45 ++++ src/server.rs | 1 + src/tests/mod.rs | 1 + src/tests/search.rs | 252 ++++++++++++++++++ ...ts__bookmarks__get_unsorted_bookmarks.snap | 8 + .../linkblocks__tests__index__index.snap | 8 + ...blocks__tests__lists__get_create_list.snap | 8 + ..._finds_bookmarks_with_various_queries.snap | 65 +++++ ...ds_bookmarks_with_various_queries.snap.new | 66 +++++ ...arch_returns_no_results_when_no_match.snap | 54 ++++ src/tests/util/html_decode.rs | 10 + src/tests/util/mod.rs | 1 + src/views/layout.rs | 49 +++- src/views/mod.rs | 1 + src/views/search_results.rs | 107 ++++++++ 25 files changed, 891 insertions(+), 9 deletions(-) create mode 100644 .sqlx/query-28d3abb588c2aed52b5b368ce43c37c2286cbd49d488f20fa2952214958f004b.json create mode 100644 .sqlx/query-34286eadd367d1d8822fc0f6c1e410170aedfdeddd8e9198e0832a7a70745422.json create mode 100644 .sqlx/query-f00a77561fa98d54c0408bfc327bdc68ff24588473338219d8144a5e5ab15dd2.json create mode 100644 src/db/search.rs create mode 100644 src/routes/search.rs create mode 100644 src/tests/search.rs create mode 100644 src/tests/snapshots/linkblocks__tests__search__search_finds_bookmarks_with_various_queries.snap create mode 100644 src/tests/snapshots/linkblocks__tests__search__search_finds_bookmarks_with_various_queries.snap.new create mode 100644 src/tests/snapshots/linkblocks__tests__search__search_returns_no_results_when_no_match.snap create mode 100644 src/tests/util/html_decode.rs create mode 100644 src/views/search_results.rs diff --git a/.sqlx/query-28d3abb588c2aed52b5b368ce43c37c2286cbd49d488f20fa2952214958f004b.json b/.sqlx/query-28d3abb588c2aed52b5b368ce43c37c2286cbd49d488f20fa2952214958f004b.json new file mode 100644 index 0000000..d3e50e4 --- /dev/null +++ b/.sqlx/query-28d3abb588c2aed52b5b368ce43c37c2286cbd49d488f20fa2952214958f004b.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "\n select bookmarks.id\n from bookmarks\n where (bookmarks.title ilike '%' || $1 || '%')\n and bookmarks.ap_user_id = $2\n and ($3::uuid is null or bookmarks.id < $3)\n order by bookmarks.id desc\n limit 5\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "Text", + "Uuid", + "Uuid" + ] + }, + "nullable": [ + false + ] + }, + "hash": "28d3abb588c2aed52b5b368ce43c37c2286cbd49d488f20fa2952214958f004b" +} diff --git a/.sqlx/query-34286eadd367d1d8822fc0f6c1e410170aedfdeddd8e9198e0832a7a70745422.json b/.sqlx/query-34286eadd367d1d8822fc0f6c1e410170aedfdeddd8e9198e0832a7a70745422.json new file mode 100644 index 0000000..a1a96c0 --- /dev/null +++ b/.sqlx/query-34286eadd367d1d8822fc0f6c1e410170aedfdeddd8e9198e0832a7a70745422.json @@ -0,0 +1,36 @@ +{ + "db_name": "PostgreSQL", + "query": "\n select title, url as bookmark_url, id as bookmark_id\n from bookmarks\n where (bookmarks.title ilike '%' || $1 || '%')\n and bookmarks.ap_user_id = $2\n and ($3::uuid is null or bookmarks.id > $3)\n order by bookmarks.id asc\n limit 4\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "title", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "bookmark_url", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "bookmark_id", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "Text", + "Uuid", + "Uuid" + ] + }, + "nullable": [ + false, + false, + false + ] + }, + "hash": "34286eadd367d1d8822fc0f6c1e410170aedfdeddd8e9198e0832a7a70745422" +} diff --git a/.sqlx/query-f00a77561fa98d54c0408bfc327bdc68ff24588473338219d8144a5e5ab15dd2.json b/.sqlx/query-f00a77561fa98d54c0408bfc327bdc68ff24588473338219d8144a5e5ab15dd2.json new file mode 100644 index 0000000..614e998 --- /dev/null +++ b/.sqlx/query-f00a77561fa98d54c0408bfc327bdc68ff24588473338219d8144a5e5ab15dd2.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "\n select bookmarks.id\n from bookmarks\n where (bookmarks.title ilike '%' || $1 || '%')\n and bookmarks.ap_user_id = $2\n and ($3::uuid is null or bookmarks.id > $3)\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "Text", + "Uuid", + "Uuid" + ] + }, + "nullable": [ + false + ] + }, + "hash": "f00a77561fa98d54c0408bfc327bdc68ff24588473338219d8144a5e5ab15dd2" +} diff --git a/.vscode/settings.json b/.vscode/settings.json index 0967ef4..1836221 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1 +1,5 @@ -{} +{ + "rust-analyzer.cargo.extraEnv": { + "SQLX_OFFLINE": "false" + } +} diff --git a/Cargo.lock b/Cargo.lock index 2d6177d..7c4f54f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -701,6 +701,12 @@ version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc55fe0d1f6c107595572ec8b107c0999bb1a2e0b75e37429a4fb0d6474a0e7d" +[[package]] +name = "diff" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" + [[package]] name = "digest" version = "0.10.7" @@ -1707,6 +1713,7 @@ dependencies = [ "mime_guess", "openidconnect", "percent-encoding", + "pretty_assertions", "railwind", "rand 0.9.2", "redact", @@ -2146,6 +2153,16 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "pretty_assertions" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ae130e2f271fbc2ac3a40fb1d07180839cdbbe443c7a27e1e3c13c5cac0116d" +dependencies = [ + "diff", + "yansi", +] + [[package]] name = "primeorder" version = "0.13.6" @@ -4268,6 +4285,12 @@ version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" +[[package]] +name = "yansi" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" + [[package]] name = "yoke" version = "0.7.5" diff --git a/Cargo.toml b/Cargo.toml index fe85744..642b486 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -86,6 +86,7 @@ test-log = { version = "0.2.18", features = [ ], default-features = false } itertools = "0.14.0" insta = "1.43.2" +pretty_assertions = "1.4.1" [package.metadata.bin] just = { version = "1.38.0", locked = true } diff --git a/Justfile b/Justfile index a6c3b25..e224297 100644 --- a/Justfile +++ b/Justfile @@ -82,7 +82,7 @@ exec-database-cli: start-database podman exec -ti -u postgres linkblocks_postgres psql ${DATABASE_NAME} generate-database-info: start-database migrate-database - cargo bin sqlx-cli prepare -- --all-targets + SQLX_OFFLINE=false cargo bin sqlx-cli prepare -- --all-targets start-test-database: #!/usr/bin/env bash diff --git a/src/db/mod.rs b/src/db/mod.rs index 7cb1ce1..de7ca55 100644 --- a/src/db/mod.rs +++ b/src/db/mod.rs @@ -18,6 +18,7 @@ pub use users::User; pub mod bookmarks; pub mod migration_hooks; pub use bookmarks::Bookmark; +pub mod search; pub async fn migrate(pool: &PgPool, base_url: &Url, up_to_version: Option) -> Result<()> { tracing::info!("Migrating the database..."); diff --git a/src/db/search.rs b/src/db/search.rs new file mode 100644 index 0000000..e964ed5 --- /dev/null +++ b/src/db/search.rs @@ -0,0 +1,106 @@ +use sqlx::{query, query_as}; +use uuid::Uuid; + +use super::AppTx; +use crate::response_error::ResponseResult; + +pub enum PreviousPage { + DoesNotExist, + IsFirstPage, + AfterBookmarkId(Uuid), +} + +pub struct Results { + pub bookmarks: Vec, + pub previous_page: PreviousPage, + pub next_page_after_bookmark_id: Option, +} + +pub struct Result { + pub title: String, + pub bookmark_id: Uuid, + pub bookmark_url: String, +} + +pub async fn search( + tx: &mut AppTx, + term: &str, + ap_user_id: Uuid, + after_bookmark_id: Option, +) -> ResponseResult { + let bookmarks = query_as!( + Result, + r#" + select title, url as bookmark_url, id as bookmark_id + from bookmarks + where (bookmarks.title ilike '%' || $1 || '%') + and bookmarks.ap_user_id = $2 + and ($3::uuid is null or bookmarks.id > $3) + order by bookmarks.id asc + limit 4 + "#, + term, + ap_user_id, + after_bookmark_id + ) + .fetch_all(&mut **tx) + .await?; + + let last_id = bookmarks.last().map(|b| b.bookmark_id); + let next_page_after_bookmark_id = query!( + r#" + select bookmarks.id + from bookmarks + where (bookmarks.title ilike '%' || $1 || '%') + and bookmarks.ap_user_id = $2 + and ($3::uuid is null or bookmarks.id > $3) + "#, + term, + ap_user_id, + last_id + ) + .fetch_optional(&mut **tx) + .await? + .and(last_id); + + let first_id = bookmarks.first().map(|b| b.bookmark_id); + // Check if there are *any* bookmarks before the first of the current page. + // If so, fetch the ids for the previous page and take the first one. + // We need to fetch multiple bookmarks because we don't know how small the + // previous page is. + let previous_bookmarks = query!( + r#" + select bookmarks.id + from bookmarks + where (bookmarks.title ilike '%' || $1 || '%') + and bookmarks.ap_user_id = $2 + and ($3::uuid is null or bookmarks.id < $3) + order by bookmarks.id desc + limit 5 + "#, + term, + ap_user_id, + first_id + ) + .fetch_all(&mut **tx) + .await?; + let previous_page = if let Some(last) = previous_bookmarks.last() { + if previous_bookmarks.len() == 5 { + // There's another page before the previous page, so we can reference the last + // bookmark of that page. + PreviousPage::AfterBookmarkId(last.id) + } else { + // This is the first page, so we have no bookmark id to query "after" + PreviousPage::IsFirstPage + } + } else { + // the query returned 0 results, so there is no previous page. + PreviousPage::DoesNotExist + }; + + Ok(Results { + bookmarks, + previous_page, + next_page_after_bookmark_id, + }) +} diff --git a/src/routes/mod.rs b/src/routes/mod.rs index 01faebc..1d9afb2 100644 --- a/src/routes/mod.rs +++ b/src/routes/mod.rs @@ -5,3 +5,4 @@ pub mod index; pub mod links; pub mod lists; pub mod users; +pub mod search; diff --git a/src/routes/search.rs b/src/routes/search.rs new file mode 100644 index 0000000..65c5186 --- /dev/null +++ b/src/routes/search.rs @@ -0,0 +1,45 @@ +use axum::{Router, extract::Query, routing::get}; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +use crate::{ + authentication::AuthUser, + db::{self}, + extract, + htmf_response::HtmfResponse, + response_error::ResponseResult, + server::AppState, + views, + views::layout, +}; +pub fn router() -> Router { + let router: Router = Router::new(); + router.route("/search", get(get_search)) +} + +#[derive(Deserialize, Serialize)] +pub struct SearchQuery { + /// The words to search for + pub q: String, + pub after_bookmark_id: Option, +} + +async fn get_search( + auth_user: AuthUser, + extract::Tx(mut tx): extract::Tx, + + Query(query): Query, +) -> ResponseResult { + let results = db::search::search( + &mut tx, + &query.q, + auth_user.ap_user_id, + query.after_bookmark_id, + ) + .await?; + let mut layout = layout::Template::from_db(&mut tx, Some(&auth_user)).await?; + layout.previous_search_input = Some(query.q); + Ok(HtmfResponse(views::search_results::view( + &views::search_results::Data { layout, results }, + ))) +} diff --git a/src/server.rs b/src/server.rs index 8370487..55e21da 100644 --- a/src/server.rs +++ b/src/server.rs @@ -59,6 +59,7 @@ pub async fn app(state: AppState) -> anyhow::Result { .merge(routes::bookmarks::router()) .merge(routes::links::router()) .merge(routes::federation::router()) + .merge(routes::search::router()) .merge(routes::assets::router().with_state(())) // TODO add layer to use the same URL for AP and HTML // this should simplify things and be more error tolerant for other services diff --git a/src/tests/mod.rs b/src/tests/mod.rs index b7deaa1..809b0dc 100644 --- a/src/tests/mod.rs +++ b/src/tests/mod.rs @@ -7,5 +7,6 @@ mod federation; mod index; mod lists; mod migrations; +mod search; mod users; mod util; diff --git a/src/tests/search.rs b/src/tests/search.rs new file mode 100644 index 0000000..5b94451 --- /dev/null +++ b/src/tests/search.rs @@ -0,0 +1,252 @@ +use pretty_assertions::assert_eq; + +use crate::{ + db::{self, bookmarks::InsertBookmark}, + routes::search::SearchQuery, + tests::util::test_app::TestApp, +}; + +#[test_log::test(tokio::test)] +async fn search_finds_bookmarks_with_various_queries() -> anyhow::Result<()> { + let mut app = TestApp::new().await; + let user = app.create_test_user().await; + app.login_test_user().await; + + // Create bookmarks with different titles (6 Rust bookmarks to trigger + // pagination since page size is 4, plus other bookmarks) + let mut tx = app.tx().await; + let titles = vec![ + "Learning Rust Programming", + "Advanced Rust Patterns", + "Rust Async Programming", + "Rust Performance Optimization", + "Rust Web Development", + "Rust Macros Guide", + // Non-Rust bookmarks to ensure search filtering works + "Python Tutorial", + "C++ Programming Guide", + ]; + + for title in &titles { + db::bookmarks::insert_local( + &mut tx, + user.ap_user_id, + InsertBookmark { + url: format!( + "https://example.com/{}", + title.to_lowercase().replace(' ', "-") + ), + title: (*title).to_string(), + }, + &app.base_url, + ) + .await?; + } + + tx.commit().await?; + + let home = app.req().get("/").await.test_page().await; + let search_results = home + .fill_form( + "form[action='/search']", + &SearchQuery { + q: "Rust".to_string(), + after_bookmark_id: None, + }, + ) + .await + .test_page() + .await; + + // Test exact word match - searching for "Rust" should find 4 out of 6 Rust + // bookmarks on first page (ordered by UUID) + let html = search_results.dom.htmls(); + + // Count how many Rust bookmarks appear in the first page + let rust_count = titles.iter().filter(|&title| html.contains(title)).count(); + assert_eq!( + rust_count, 4, + "First page should show exactly 4 Rust bookmarks" + ); + + // Verify non-Rust bookmarks are not included + assert!(!html.contains("Python Tutorial")); + assert!(!html.contains("C++ Programming Guide")); + + // Test case insensitivity + let search_results = app.req().get("/search?q=python").await.test_page().await; + let html = search_results.dom.htmls(); + assert!(html.contains("Python Tutorial")); + + // Test partial word match - "gram" appears in "Programming" and "Guide" + let search_results = app.req().get("/search?q=gram").await.test_page().await; + let html = search_results.dom.htmls(); + // At least one bookmark with "Programming" or "Guide" should appear + let has_programming = html.contains("Programming"); + let has_guide = html.contains("Guide"); + assert!( + has_programming || has_guide, + "Should find bookmarks containing 'gram'" + ); + + // Test special characters + let search_results = app.req().get("/search?q=C%2B%2B").await.test_page().await; + let html = search_results.dom.htmls(); + assert!(html.contains("C++ Programming Guide")); + assert!(!html.contains("Python Tutorial")); + + Ok(()) +} + +#[test_log::test(tokio::test)] +async fn search_only_returns_users_own_bookmarks() -> anyhow::Result<()> { + let mut app = TestApp::new().await; + let user1 = app.create_test_user().await; + let user2 = app.create_user("otheruser", "otherpassword").await; + + // Create bookmarks for both users with similar titles + let mut tx = app.tx().await; + let bookmark_1 = db::bookmarks::insert_local( + &mut tx, + user1.ap_user_id, + InsertBookmark { + url: "https://example.com/user1".to_string(), + title: "My Rust Tutorial".to_string(), + }, + &app.base_url, + ) + .await?; + let bookmark_2 = db::bookmarks::insert_local( + &mut tx, + user2.ap_user_id, + InsertBookmark { + url: "https://example.com/user2".to_string(), + title: "Other User's Rust Guide".to_string(), + }, + &app.base_url, + ) + .await?; + tx.commit().await?; + + let query = "/search?q=Rust"; + // Login as user1 and search + app.login_test_user().await; + let search_results = app.req().get(query).await.test_page().await; + + let html = search_results.dom.htmls(); + assert!(html.contains(&bookmark_1.id.to_string())); + assert!(!html.contains(&bookmark_2.id.to_string())); + + // verify that the same query matches the other bookmark as well + app.login_user(&user2.username, "otherpassword").await; + let search_results = app.req().get(query).await.test_page().await; + let html = search_results.dom.htmls(); + tracing::debug!("{}", search_results.dom.find("main").html()); + assert!(!html.contains(&bookmark_1.id.to_string())); + assert!(html.contains(&bookmark_2.id.to_string())); + + Ok(()) +} + +#[test_log::test(tokio::test)] +async fn search_pagination_navigation() -> anyhow::Result<()> { + // TODO update this test to use the pagination links provided in the html, + // instead of generating the URLs inline here + let mut app = TestApp::new().await; + let user = app.create_test_user().await; + app.login_test_user().await; + + // Create enough bookmarks to span multiple pages (page size is 4) + let mut tx = app.tx().await; + let mut bookmarks = Vec::new(); + for i in 1..=15 { + let bookmark = db::bookmarks::insert_local( + &mut tx, + user.ap_user_id, + InsertBookmark { + url: format!("https://example.com/test{i}"), + title: format!("Test Bookmark {i:02}"), + }, + &app.base_url, + ) + .await?; + bookmarks.push((bookmark.id, bookmark.title.clone())); + } + tx.commit().await?; + + // Sort bookmarks by ID to match the database sort order + bookmarks.sort_by_key(|(id, _)| *id); + tracing::debug!("{bookmarks:#?}"); + + // Test first page - should show first 4 bookmarks sorted by ID + let first_page = app.req().get("/search?q=Test").await.test_page().await; + for link in first_page.dom.find("a") { + println!("- {}", link.outer_html()); + } + let html = first_page.dom.find("main").htmls(); + assert!(html.contains(&bookmarks[0].1)); // First bookmark + assert!(html.contains(&bookmarks[3].1)); // Fourth bookmark + assert!(!html.contains(&bookmarks[4].1)); // Fifth bookmark shouldn't be visible + assert!(html.contains("Next page")); + assert!(!html.contains("Previous page")); + + // Test second page (forward pagination) + let second_page = first_page.visit_link("Next page").await; + for link in second_page.dom.find("a") { + println!("- {}", link.outer_html()); + } + let second_page_html = second_page.dom.find("main").htmls(); + assert!(second_page_html.contains(&bookmarks[4].1)); // Fifth bookmark + assert!(second_page_html.contains(&bookmarks[7].1)); // Eighth bookmark + assert!(!second_page_html.contains(&bookmarks[3].1)); // Fourth bookmark from page 1 + assert!(!second_page_html.contains(&bookmarks[8].1)); // Ninth bookmark from page 3 + assert!(second_page_html.contains("Previous page")); + assert!(second_page_html.contains("Next page")); + + let third_page = second_page.visit_link("Next page").await; + for link in third_page.dom.find("a") { + println!("- {}", link.outer_html()); + } + + // Test backward pagination - go back to second page + let back_to_second = third_page.visit_link("Previous page").await; + for link in back_to_second.dom.find("a") { + println!("- {}", link.outer_html()); + } + + let html = back_to_second.dom.find("main").htmls(); + assert!(html.contains(&bookmarks[4].1)); // Fifth bookmark + assert!(html.contains(&bookmarks[7].1)); // Eighth bookmark + assert!(!html.contains(&bookmarks[3].1)); // Fourth bookmark from page 1 + assert!(!html.contains(&bookmarks[8].1)); // Ninth bookmark from page 3 + assert_eq!(back_to_second.dom.find("main").htmls(), second_page_html); + + Ok(()) +} + +#[test_log::test(tokio::test)] +async fn search_preserves_query_in_pagination() -> anyhow::Result<()> { + let mut app = TestApp::new().await; + app.create_test_user().await; + app.login_test_user().await; + + let search_results = app.req().get("/search?q=Rust").await.test_page().await; + + // Check that the search query is preserved in the pagination form + assert!(search_results.dom.html().contains(r#"value="Rust""#)); + + Ok(()) +} + +#[test_log::test(tokio::test)] +async fn search_requires_authentication() -> anyhow::Result<()> { + let mut app = TestApp::new().await; + + // Try to search without logging in - should redirect to login page + app.req() + .expect_status(axum::http::StatusCode::SEE_OTHER) + .get("/search?q=test") + .await; + + Ok(()) +} diff --git a/src/tests/snapshots/linkblocks__tests__bookmarks__get_unsorted_bookmarks.snap b/src/tests/snapshots/linkblocks__tests__bookmarks__get_unsorted_bookmarks.snap index e8c5668..6ccc16c 100644 --- a/src/tests/snapshots/linkblocks__tests__bookmarks__get_unsorted_bookmarks.snap +++ b/src/tests/snapshots/linkblocks__tests__bookmarks__get_unsorted_bookmarks.snap @@ -13,6 +13,14 @@ expression: unsorted_bookmarks.dom.htmls()
+
+
+ + +
+

Unsorted Bookmarks

diff --git a/src/tests/snapshots/linkblocks__tests__index__index.snap b/src/tests/snapshots/linkblocks__tests__index__index.snap index 7c07450..8d68ef0 100644 --- a/src/tests/snapshots/linkblocks__tests__index__index.snap +++ b/src/tests/snapshots/linkblocks__tests__index__index.snap @@ -13,6 +13,14 @@ expression: index.dom.htmls()
+
+
+ + +
+

Welcome to linkblocks!

diff --git a/src/tests/snapshots/linkblocks__tests__lists__get_create_list.snap b/src/tests/snapshots/linkblocks__tests__lists__get_create_list.snap index fe51236..4eb98a9 100644 --- a/src/tests/snapshots/linkblocks__tests__lists__get_create_list.snap +++ b/src/tests/snapshots/linkblocks__tests__lists__get_create_list.snap @@ -13,6 +13,14 @@ expression: create_list.dom.htmls()
+
+
+ + +
+

Create a list

diff --git a/src/tests/snapshots/linkblocks__tests__search__search_finds_bookmarks_with_various_queries.snap b/src/tests/snapshots/linkblocks__tests__search__search_finds_bookmarks_with_various_queries.snap new file mode 100644 index 0000000..ced8bef --- /dev/null +++ b/src/tests/snapshots/linkblocks__tests__search__search_finds_bookmarks_with_various_queries.snap @@ -0,0 +1,65 @@ +--- +source: src/tests/search.rs +expression: html +--- + + + + + + + linkblocks + + +
+
+
+ + + + +
+
+
+ C++ Programming Guide +

+ https://example.com/cpp +

+
+
+ Connect +
+
+
+
+
+ +
+ diff --git a/src/tests/snapshots/linkblocks__tests__search__search_finds_bookmarks_with_various_queries.snap.new b/src/tests/snapshots/linkblocks__tests__search__search_finds_bookmarks_with_various_queries.snap.new new file mode 100644 index 0000000..3c13b5d --- /dev/null +++ b/src/tests/snapshots/linkblocks__tests__search__search_finds_bookmarks_with_various_queries.snap.new @@ -0,0 +1,66 @@ +--- +source: src/tests/search.rs +assertion_line: 74 +expression: html +--- + + + + + + + linkblocks + + +
+
+
+
+ + +
+
+
+
+ C++ Programming Guide +

+ https://example.com/cpp +

+
+
+ Connect +
+
+
+
+
+ +
+ diff --git a/src/tests/snapshots/linkblocks__tests__search__search_returns_no_results_when_no_match.snap b/src/tests/snapshots/linkblocks__tests__search__search_returns_no_results_when_no_match.snap new file mode 100644 index 0000000..720a3bb --- /dev/null +++ b/src/tests/snapshots/linkblocks__tests__search__search_returns_no_results_when_no_match.snap @@ -0,0 +1,54 @@ +--- +source: src/tests/search.rs +expression: html +--- + + + + + + + linkblocks + + +
+
+
+
+ + +
+
+
+
+
+ +
+ diff --git a/src/tests/util/html_decode.rs b/src/tests/util/html_decode.rs new file mode 100644 index 0000000..c9e1b3e --- /dev/null +++ b/src/tests/util/html_decode.rs @@ -0,0 +1,10 @@ +/// Used to unescape html returned from the server, e.g. when extracting URLs or +/// other content. +pub fn html_decode(input: &str) -> String { + input + .replace("<", "<") + .replace(">", ">") + .replace("&", "&") + .replace(""", r#"""#) + .replace("'", "'") +} diff --git a/src/tests/util/mod.rs b/src/tests/util/mod.rs index c637ba2..1dc49d4 100644 --- a/src/tests/util/mod.rs +++ b/src/tests/util/mod.rs @@ -1,4 +1,5 @@ pub mod db; pub mod dom; +pub mod html_decode; pub mod request_builder; pub mod test_app; diff --git a/src/views/layout.rs b/src/views/layout.rs index 2b325e0..af4c3c4 100644 --- a/src/views/layout.rs +++ b/src/views/layout.rs @@ -9,6 +9,7 @@ use crate::{ pub struct Template { pub authed_info: Option, + pub previous_search_input: Option, } impl Template { @@ -20,18 +21,52 @@ impl Template { }; Ok(Template { authed_info: auth_info, + previous_search_input: None, }) } } pub fn layout(children: Children, layout: &Template) -> Element { - base_document(div(class("flex-row-reverse h-full sm:flex")).with([ - main_(class("sm:overflow-y-auto sm:grow")).with(children), - match &layout.authed_info { - Some(info) => sidebar(info), - None => fragment(), - }, - ])) + base_document( + div(class("flex-row-reverse h-full sm:flex")).with([ + main_(class("sm:overflow-y-auto sm:grow")) + .with(match &layout.authed_info { + Some(_info) => search(layout.previous_search_input.as_deref()), + None => fragment(), + }) + .with(children), + match &layout.authed_info { + Some(info) => sidebar(info), + None => fragment(), + }, + ]), + ) +} + +fn search(previous_input: Option<&str>) -> Element { + div(class("p-2")).with( + form([ + action("/search"), + method("get"), + attr("hx-boost", "true"), + class("inline-flex w-full max-w-lg"), + ]) + .with([ + // TODO: When using tridactyl's "go to input", switching to the button by pressing the + // tab key doesn't work + input([ + type_("text"), + name("q"), + placeholder("Search all bookmarks"), + value(previous_input.unwrap_or("")), + class("py-1.5 px-3 bg-neutral-900 grow border rounded-l border-neutral-700"), + ]), + button(class( + "px-2 text-neutral-400 shrink border-y border-r rounded-r border-neutral-700", + )) + .with("Search"), + ]), + ) } fn sidebar(authed_info: &AuthedInfo) -> Element { diff --git a/src/views/mod.rs b/src/views/mod.rs index 64cc696..2250461 100644 --- a/src/views/mod.rs +++ b/src/views/mod.rs @@ -15,5 +15,6 @@ pub mod login; pub mod login_demo; pub mod oidc_select_username; pub mod profile; +pub mod search_results; pub mod unsorted_bookmarks; pub mod users; diff --git a/src/views/search_results.rs b/src/views/search_results.rs new file mode 100644 index 0000000..7801e33 --- /dev/null +++ b/src/views/search_results.rs @@ -0,0 +1,107 @@ +use htmf::{element::Element, prelude_inline::*}; + +use crate::{ + db, + views::{content, layout}, +}; + +pub struct Data { + pub layout: layout::Template, + pub results: db::search::Results, +} + +pub fn view(data: &Data) -> Element { + layout::layout(results(data), &data.layout) +} + +// TODO use percent encoding for previous search input in urls + +fn results(data: &Data) -> Element { + fragment( + data.results + .bookmarks + .iter() + .map(|r| list_item(r, data)) + .collect::>(), + ) + .with(pagination(data)) +} + +fn pagination(data: &Data) -> Element { + section( + class("flex flex-row gap-2 justify-center w-full p-4 border-t border-neutral-700"), + [ + match data.results.previous_page { + db::search::PreviousPage::AfterBookmarkId(id) => { + let url = format!( + "/search?q={}&after_bookmark_id={}", + data.layout.previous_search_input.as_deref().unwrap_or(""), + id + ); + a([href(url)], "Previous page") + } + db::search::PreviousPage::IsFirstPage => { + let url = format!( + "/search?q={}", + data.layout.previous_search_input.as_deref().unwrap_or(""), + ); + a([href(url)], "Previous page") + } + db::search::PreviousPage::DoesNotExist => nothing(), + }, + match data.results.next_page_after_bookmark_id { + Some(next_page_after_bookmark_id) => { + let url = format!( + "/search?q={}&after_bookmark_id={}", + data.layout.previous_search_input.as_deref().unwrap_or(""), + next_page_after_bookmark_id + ); + a([href(url)], "Next page") + } + None => nothing(), + }, + ], + ) +} + +fn list_item(result: &db::search::Result, Data { layout, .. }: &Data) -> Element { + section( + class("flex flex-wrap items-end gap-2 px-4 pt-4 pb-4 border-t border-neutral-700"), + [ + div(class("overflow-hidden"), list_item_bookmark(result)), + if let Some(_authed_info) = &layout.authed_info { + div( + class( + "flex flex-wrap justify-end flex-1 pt-2 text-sm basis-32 gap-x-2 \ + text-neutral-400", + ), + [a( + [ + class("hover:text-neutral-100"), + href(format!("/links/create?dest_id={}", result.bookmark_id)), + ], + "Connect", + )], + ) + } else { + nothing() + }, + ], + ) +} + +fn list_item_bookmark(result: &db::search::Result) -> Element { + fragment([ + a( + [ + class( + "block overflow-hidden leading-8 text-orange-100 hover:text-orange-300 \ + text-ellipsis whitespace-nowrap", + ), + href(&result.bookmark_url), + ], + &result.title, + ), + content::link_url(&result.bookmark_url), + ]) +}