diff --git a/..gitignore.swp b/..gitignore.swp new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/Cargo.lock b/Cargo.lock index ec55e4af77f78a..247e92d48ad123 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -153,6 +153,7 @@ dependencies = [ "db", "derive_more 0.99.20", "editor", + "encodings", "env_logger 0.11.8", "fs", "futures 0.3.31", @@ -3354,6 +3355,7 @@ dependencies = [ "dashmap 6.1.0", "debugger_ui", "editor", + "encodings", "envy", "extension", "file_finder", @@ -3720,6 +3722,7 @@ dependencies = [ "dirs 4.0.0", "edit_prediction", "editor", + "encodings", "fs", "futures 0.3.31", "gpui", @@ -5511,6 +5514,70 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" +[[package]] +name = "encoding" +version = "0.2.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b0d943856b990d12d3b55b359144ff341533e516d94098b1d3fc1ac666d36ec" +dependencies = [ + "encoding-index-japanese", + "encoding-index-korean", + "encoding-index-simpchinese", + "encoding-index-singlebyte", + "encoding-index-tradchinese", +] + +[[package]] +name = "encoding-index-japanese" +version = "1.20141219.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04e8b2ff42e9a05335dbf8b5c6f7567e5591d0d916ccef4e0b1710d32a0d0c91" +dependencies = [ + "encoding_index_tests", +] + +[[package]] +name = "encoding-index-korean" +version = "1.20141219.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dc33fb8e6bcba213fe2f14275f0963fd16f0a02c878e3095ecfdf5bee529d81" +dependencies = [ + "encoding_index_tests", +] + +[[package]] +name = "encoding-index-simpchinese" +version = "1.20141219.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d87a7194909b9118fc707194baa434a4e3b0fb6a5a757c73c3adb07aa25031f7" +dependencies = [ + "encoding_index_tests", +] + +[[package]] +name = "encoding-index-singlebyte" +version = "1.20141219.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3351d5acffb224af9ca265f435b859c7c01537c0849754d3db3fdf2bfe2ae84a" +dependencies = [ + "encoding_index_tests", +] + +[[package]] +name = "encoding-index-tradchinese" +version = "1.20141219.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd0e20d5688ce3cab59eb3ef3a2083a5c77bf496cb798dc6fcdb75f323890c18" +dependencies = [ + "encoding_index_tests", +] + +[[package]] +name = "encoding_index_tests" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a246d82be1c9d791c5dfde9a2bd045fc3cbba3fa2b11ad558f27d01712f00569" + [[package]] name = "encoding_rs" version = "0.8.35" @@ -5520,6 +5587,34 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "encodings" +version = "0.1.0" +dependencies = [ + "anyhow", + "encoding_rs", +] + +[[package]] +name = "encodings_ui" +version = "0.1.0" +dependencies = [ + "anyhow", + "editor", + "encoding_rs", + "fs", + "futures 0.3.31", + "fuzzy", + "gpui", + "language", + "picker", + "settings", + "ui", + "util", + "workspace", + "zed_actions", +] + [[package]] name = "endi" version = "1.1.0" @@ -5897,6 +5992,7 @@ dependencies = [ "criterion", "ctor", "dap", + "encodings", "extension", "fs", "futures 0.3.31", @@ -6396,6 +6492,8 @@ dependencies = [ "async-trait", "cocoa 0.26.0", "collections", + "encoding", + "encodings", "fsevent", "futures 0.3.31", "git", @@ -7097,6 +7195,7 @@ dependencies = [ "ctor", "db", "editor", + "encodings", "futures 0.3.31", "fuzzy", "git", @@ -8771,6 +8870,8 @@ dependencies = [ "ctor", "diffy", "ec4rs", + "encoding", + "encodings", "fs", "futures 0.3.31", "fuzzy", @@ -12986,6 +13087,7 @@ dependencies = [ "context_server", "dap", "dap_adapters", + "encodings", "extension", "fancy-regex 0.14.0", "fs", @@ -13961,6 +14063,7 @@ dependencies = [ "dap_adapters", "debug_adapter_extension", "editor", + "encodings", "env_logger 0.11.8", "extension", "extension_host", @@ -20725,6 +20828,7 @@ dependencies = [ "component", "dap", "db", + "encodings", "fs", "futures 0.3.31", "gpui", @@ -20767,6 +20871,8 @@ dependencies = [ "async-lock 2.8.0", "clock", "collections", + "encoding", + "encodings", "fs", "futures 0.3.31", "fuzzy", @@ -21176,6 +21282,8 @@ dependencies = [ "diagnostics", "edit_prediction_button", "editor", + "encodings", + "encodings_ui", "env_logger 0.11.8", "extension", "extension_host", diff --git a/Cargo.toml b/Cargo.toml index 369082ff16736f..8e5bc5a79cfc17 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -59,6 +59,8 @@ members = [ "crates/zeta2_tools", "crates/editor", "crates/eval", + "crates/encodings", + "crates/encodings_ui", "crates/explorer_command_injector", "crates/extension", "crates/extension_api", @@ -221,6 +223,8 @@ members = [ "tooling/perf", "tooling/xtask", + "crates/encodings", + "crates/encodings_ui", ] default-members = ["crates/zed"] @@ -242,7 +246,6 @@ activity_indicator = { path = "crates/activity_indicator" } agent_ui = { path = "crates/agent_ui" } agent_settings = { path = "crates/agent_settings" } agent_servers = { path = "crates/agent_servers" } -ai = { path = "crates/ai" } ai_onboarding = { path = "crates/ai_onboarding" } anthropic = { path = "crates/anthropic" } askpass = { path = "crates/askpass" } @@ -252,7 +255,6 @@ assistant_slash_command = { path = "crates/assistant_slash_command" } assistant_slash_commands = { path = "crates/assistant_slash_commands" } audio = { path = "crates/audio" } auto_update = { path = "crates/auto_update" } -auto_update_helper = { path = "crates/auto_update_helper" } auto_update_ui = { path = "crates/auto_update_ui" } aws_http_client = { path = "crates/aws_http_client" } bedrock = { path = "crates/bedrock" } @@ -315,6 +317,8 @@ edit_prediction = { path = "crates/edit_prediction" } edit_prediction_button = { path = "crates/edit_prediction_button" } edit_prediction_context = { path = "crates/edit_prediction_context" } zeta2_tools = { path = "crates/zeta2_tools" } +encodings = {path = "crates/encodings"} +encodings_ui = {path = "crates/encodings_ui"} inspector_ui = { path = "crates/inspector_ui" } install_cli = { path = "crates/install_cli" } journal = { path = "crates/journal" } @@ -355,8 +359,6 @@ panel = { path = "crates/panel" } paths = { path = "crates/paths" } perf = { path = "tooling/perf" } picker = { path = "crates/picker" } -plugin = { path = "crates/plugin" } -plugin_macros = { path = "crates/plugin_macros" } prettier = { path = "crates/prettier" } settings_profile_selector = { path = "crates/settings_profile_selector" } project = { path = "crates/project" } @@ -390,7 +392,6 @@ snippets_ui = { path = "crates/snippets_ui" } sqlez = { path = "crates/sqlez" } sqlez_macros = { path = "crates/sqlez_macros" } story = { path = "crates/story" } -storybook = { path = "crates/storybook" } streaming_diff = { path = "crates/streaming_diff" } sum_tree = { path = "crates/sum_tree" } supermaven = { path = "crates/supermaven" } @@ -407,7 +408,6 @@ terminal_view = { path = "crates/terminal_view" } text = { path = "crates/text" } theme = { path = "crates/theme" } theme_extension = { path = "crates/theme_extension" } -theme_importer = { path = "crates/theme_importer" } theme_selector = { path = "crates/theme_selector" } time_format = { path = "crates/time_format" } title_bar = { path = "crates/title_bar" } @@ -501,6 +501,7 @@ documented = "0.9.1" dotenvy = "0.15.0" ec4rs = "1.1" emojis = "0.6.1" +encoding_rs = "0.8" env_logger = "0.11" exec = "0.3.1" fancy-regex = "0.14.0" @@ -790,11 +791,7 @@ codegen-units = 16 [profile.dev.package] taffy = { opt-level = 3 } cranelift-codegen = { opt-level = 3 } -cranelift-codegen-meta = { opt-level = 3 } -cranelift-codegen-shared = { opt-level = 3 } resvg = { opt-level = 3 } -rustybuzz = { opt-level = 3 } -ttf-parser = { opt-level = 3 } wasmtime-cranelift = { opt-level = 3 } wasmtime = { opt-level = 3 } # Build single-source-file crates with cg=1 as it helps make `cargo build` of a whole workspace a bit faster @@ -804,7 +801,6 @@ breadcrumbs = { codegen-units = 1 } collections = { codegen-units = 1 } command_palette = { codegen-units = 1 } command_palette_hooks = { codegen-units = 1 } -extension_cli = { codegen-units = 1 } feature_flags = { codegen-units = 1 } file_icons = { codegen-units = 1 } fsevent = { codegen-units = 1 } diff --git a/assets/settings/default.json b/assets/settings/default.json index f62cc1844732db..973dfd46a0d00c 100644 --- a/assets/settings/default.json +++ b/assets/settings/default.json @@ -1352,7 +1352,9 @@ // Whether to show the cursor position button in the status bar. "cursor_position_button": true, // Whether to show active line endings button in the status bar. - "line_endings_button": false + "line_endings_button": false, + // Whether to show the encoding indicator in the status bar. + "encoding_indicator": true }, // Settings specific to the terminal "terminal": { diff --git a/crates/agent/Cargo.toml b/crates/agent/Cargo.toml index e962c876a38f78..f1c277c3ada94b 100644 --- a/crates/agent/Cargo.toml +++ b/crates/agent/Cargo.toml @@ -32,6 +32,7 @@ collections.workspace = true context_server.workspace = true db.workspace = true derive_more.workspace = true +encodings.workspace = true fs.workspace = true futures.workspace = true git.workspace = true diff --git a/crates/agent/src/tools/edit_file_tool.rs b/crates/agent/src/tools/edit_file_tool.rs index 078273dbb8a439..2fe299392ea6ae 100644 --- a/crates/agent/src/tools/edit_file_tool.rs +++ b/crates/agent/src/tools/edit_file_tool.rs @@ -563,6 +563,7 @@ mod tests { use super::*; use crate::{ContextServerRegistry, Templates}; use client::TelemetrySettings; + use encodings::Encoding; use fs::Fs; use gpui::{TestAppContext, UpdateGlobal}; use language_model::fake_provider::FakeLanguageModel; @@ -744,6 +745,7 @@ mod tests { path!("/root/src/main.rs").as_ref(), &Rope::from_str_small("initial content"), language::LineEnding::Unix, + Encoding::default(), ) .await .unwrap(); @@ -911,6 +913,7 @@ mod tests { path!("/root/src/main.rs").as_ref(), &Rope::from_str_small("initial content"), language::LineEnding::Unix, + Encoding::default(), ) .await .unwrap(); diff --git a/crates/agent_ui/src/acp/message_editor.rs b/crates/agent_ui/src/acp/message_editor.rs index 856cc4d0d47d1e..0198fca47fe063 100644 --- a/crates/agent_ui/src/acp/message_editor.rs +++ b/crates/agent_ui/src/acp/message_editor.rs @@ -1248,7 +1248,7 @@ fn full_mention_for_directory( worktree_id, path: worktree_path, }; - buffer_store.open_buffer(project_path, cx) + buffer_store.open_buffer(project_path, &Default::default(), cx) }) }); diff --git a/crates/agent_ui/src/context.rs b/crates/agent_ui/src/context.rs index 2a1ff4a1d9d3e0..fbe76961967aca 100644 --- a/crates/agent_ui/src/context.rs +++ b/crates/agent_ui/src/context.rs @@ -287,7 +287,7 @@ impl DirectoryContextHandle { let open_task = project.update(cx, |project, cx| { project.buffer_store().update(cx, |buffer_store, cx| { let project_path = ProjectPath { worktree_id, path }; - buffer_store.open_buffer(project_path, cx) + buffer_store.open_buffer(project_path, &Default::default(), cx) }) }); diff --git a/crates/collab/Cargo.toml b/crates/collab/Cargo.toml index c8467da7954b19..01ef52291fa089 100644 --- a/crates/collab/Cargo.toml +++ b/crates/collab/Cargo.toml @@ -31,6 +31,7 @@ chrono.workspace = true clock.workspace = true collections.workspace = true dashmap.workspace = true +encodings.workspace = true envy = "0.4.2" futures.workspace = true gpui.workspace = true diff --git a/crates/collab/src/tests/integration_tests.rs b/crates/collab/src/tests/integration_tests.rs index 37e6622b0343bc..1adb2ae5fb65dc 100644 --- a/crates/collab/src/tests/integration_tests.rs +++ b/crates/collab/src/tests/integration_tests.rs @@ -12,6 +12,7 @@ use buffer_diff::{DiffHunkSecondaryStatus, DiffHunkStatus, assert_hunks}; use call::{ActiveCall, ParticipantLocation, Room, room}; use client::{RECEIVE_TIMEOUT, User}; use collections::{HashMap, HashSet}; +use encodings::Encoding; use fs::{FakeFs, Fs as _, RemoveOptions}; use futures::{StreamExt as _, channel::mpsc}; use git::{ @@ -3701,6 +3702,7 @@ async fn test_buffer_reloading( path!("/dir/a.txt").as_ref(), &new_contents, LineEnding::Windows, + Encoding::default(), ) .await .unwrap(); @@ -4481,6 +4483,7 @@ async fn test_reloading_buffer_manually( path!("/a/a.rs").as_ref(), &Rope::from_str_small("let seven = 7;"), LineEnding::Unix, + Encoding::default(), ) .await .unwrap(); diff --git a/crates/collab/src/tests/random_project_collaboration_tests.rs b/crates/collab/src/tests/random_project_collaboration_tests.rs index 399f1a663fe727..9f0c47bcd722ef 100644 --- a/crates/collab/src/tests/random_project_collaboration_tests.rs +++ b/crates/collab/src/tests/random_project_collaboration_tests.rs @@ -5,6 +5,7 @@ use async_trait::async_trait; use call::ActiveCall; use collections::{BTreeMap, HashMap}; use editor::Bias; +use encodings::Encoding; use fs::{FakeFs, Fs as _}; use git::status::{FileStatus, StatusCode, TrackedStatus, UnmergedStatus, UnmergedStatusCode}; use gpui::{BackgroundExecutor, Entity, TestAppContext}; @@ -943,6 +944,7 @@ impl RandomizedTest for ProjectCollaborationTest { &path, &Rope::from_str_small(content.as_str()), text::LineEnding::Unix, + Encoding::default(), ) .await .unwrap(); diff --git a/crates/copilot/Cargo.toml b/crates/copilot/Cargo.toml index d9ea4709eadcfa..ba7209368a4f5a 100644 --- a/crates/copilot/Cargo.toml +++ b/crates/copilot/Cargo.toml @@ -30,6 +30,7 @@ client.workspace = true collections.workspace = true command_palette_hooks.workspace = true dirs.workspace = true +encodings.workspace = true fs.workspace = true futures.workspace = true gpui.workspace = true @@ -54,6 +55,7 @@ util.workspace = true workspace.workspace = true itertools.workspace = true + [target.'cfg(windows)'.dependencies] async-std = { version = "1.12.0", features = ["unstable"] } diff --git a/crates/copilot/src/copilot.rs b/crates/copilot/src/copilot.rs index 41c8a17c2d251e..c77b49dea38317 100644 --- a/crates/copilot/src/copilot.rs +++ b/crates/copilot/src/copilot.rs @@ -1241,6 +1241,7 @@ async fn get_copilot_lsp(fs: Arc, node_runtime: NodeRuntime) -> anyhow:: #[cfg(test)] mod tests { use super::*; + use encodings::{Encoding, EncodingOptions}; use gpui::TestAppContext; use util::{path, paths::PathStyle, rel_path::rel_path}; @@ -1451,7 +1452,12 @@ mod tests { self.abs_path.clone() } - fn load(&self, _: &App) -> Task> { + fn load( + &self, + _: &App, + _: &EncodingOptions, + _: Option>, + ) -> Task> { unimplemented!() } diff --git a/crates/edit_prediction_context/src/syntax_index.rs b/crates/edit_prediction_context/src/syntax_index.rs index 76aa10c076d95a..dc2a465eee13fa 100644 --- a/crates/edit_prediction_context/src/syntax_index.rs +++ b/crates/edit_prediction_context/src/syntax_index.rs @@ -523,8 +523,9 @@ impl SyntaxIndex { }; let snapshot_task = worktree.update(cx, |worktree, cx| { - let load_task = worktree.load_file(&project_path.path, cx); + let load_task = worktree.load_file(&project_path.path, &Default::default(), None, cx); let worktree_abs_path = worktree.abs_path(); + cx.spawn(async move |_this, cx| { let loaded_file = load_task.await?; let language = language.await?; diff --git a/crates/encodings/Cargo.toml b/crates/encodings/Cargo.toml new file mode 100644 index 00000000000000..0e900d576012e5 --- /dev/null +++ b/crates/encodings/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "encodings" +version = "0.1.0" +publish.workspace = true +edition.workspace = true + +[dependencies] +anyhow.workspace = true +encoding_rs.workspace = true + +[lints] +workspace = true diff --git a/crates/encodings/LICENSE-GPL b/crates/encodings/LICENSE-GPL new file mode 120000 index 00000000000000..89e542f750cd38 --- /dev/null +++ b/crates/encodings/LICENSE-GPL @@ -0,0 +1 @@ +../../LICENSE-GPL \ No newline at end of file diff --git a/crates/encodings/src/lib.rs b/crates/encodings/src/lib.rs new file mode 100644 index 00000000000000..6158084e87464e --- /dev/null +++ b/crates/encodings/src/lib.rs @@ -0,0 +1,188 @@ +use encoding_rs; +use std::{ + fmt::Debug, + sync::{Arc, Mutex, atomic::AtomicBool}, +}; + +pub use encoding_rs::{ + BIG5, EUC_JP, EUC_KR, GB18030, GBK, IBM866, ISO_2022_JP, ISO_8859_2, ISO_8859_3, ISO_8859_4, + ISO_8859_5, ISO_8859_6, ISO_8859_7, ISO_8859_8, ISO_8859_8_I, ISO_8859_10, ISO_8859_13, + ISO_8859_14, ISO_8859_15, ISO_8859_16, KOI8_R, KOI8_U, MACINTOSH, SHIFT_JIS, UTF_8, UTF_16BE, + UTF_16LE, WINDOWS_874, WINDOWS_1250, WINDOWS_1251, WINDOWS_1252, WINDOWS_1253, WINDOWS_1254, + WINDOWS_1255, WINDOWS_1256, WINDOWS_1257, WINDOWS_1258, X_MAC_CYRILLIC, +}; + +pub struct Encoding(Mutex<&'static encoding_rs::Encoding>); + +impl Debug for Encoding { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_tuple(&format!("Encoding{:?}", self.0)) + .field(&self.get().name()) + .finish() + } +} + +impl Clone for Encoding { + fn clone(&self) -> Self { + Encoding(Mutex::new(self.get())) + } +} + +impl Default for Encoding { + fn default() -> Self { + Encoding(Mutex::new(UTF_8)) + } +} + +impl From<&'static encoding_rs::Encoding> for Encoding { + fn from(encoding: &'static encoding_rs::Encoding) -> Self { + Encoding::new(encoding) + } +} + +unsafe impl Send for Encoding {} +unsafe impl Sync for Encoding {} + +impl Encoding { + pub fn new(encoding: &'static encoding_rs::Encoding) -> Self { + Self(Mutex::new(encoding)) + } + + pub fn set(&self, encoding: &'static encoding_rs::Encoding) { + *self.0.lock().unwrap() = encoding; + } + + pub fn get(&self) -> &'static encoding_rs::Encoding { + *self.0.lock().unwrap() + } + + pub async fn decode( + &self, + input: Vec, + force: bool, + detect_utf16: bool, + buffer_encoding: Option>, + ) -> anyhow::Result { + // Check if the input starts with a BOM for UTF-16 encodings only if detect_utf16 is true. + if detect_utf16 { + if let Some(encoding) = match input.get(..2) { + Some([0xFF, 0xFE]) => Some(UTF_16LE), + Some([0xFE, 0xFF]) => Some(UTF_16BE), + _ => None, + } { + self.set(encoding); + + if let Some(v) = buffer_encoding { + v.set(encoding) + } + } + } + + let (cow, had_errors) = self.get().decode_with_bom_removal(&input); + + if force { + return Ok(cow.to_string()); + } + + if !had_errors { + Ok(cow.to_string()) + } else { + Err(anyhow::anyhow!( + "The file contains invalid bytes for the specified encoding: {}.\nThis usually means that the file is not a regular text file, or is encoded in a different encoding.\nContinuing to open it may result in data loss if saved.", + self.get().name() + )) + } + } + + pub async fn encode(&self, input: String) -> anyhow::Result> { + if self.get() == UTF_16BE { + let mut data = Vec::::with_capacity(input.len() * 2); + + // Convert the input string to UTF-16BE bytes + let utf16be_bytes = input.encode_utf16().flat_map(|u| u.to_be_bytes()); + + data.extend(utf16be_bytes); + return Ok(data); + } else if self.get() == UTF_16LE { + let mut data = Vec::::with_capacity(input.len() * 2); + + // Convert the input string to UTF-16LE bytes + let utf16le_bytes = input.encode_utf16().flat_map(|u| u.to_le_bytes()); + + data.extend(utf16le_bytes); + return Ok(data); + } else { + let (cow, _encoding_used, _had_errors) = self.get().encode(&input); + + Ok(cow.into_owned()) + } + } + + pub fn reset(&self) { + self.set(UTF_8); + } +} + +/// Convert a byte vector from a specified encoding to a UTF-8 string. +pub async fn to_utf8( + input: Vec, + options: &EncodingOptions, + buffer_encoding: Option>, +) -> anyhow::Result { + options + .encoding + .decode( + input, + options.force.load(std::sync::atomic::Ordering::Acquire), + options + .detect_utf16 + .load(std::sync::atomic::Ordering::Acquire), + buffer_encoding, + ) + .await +} + +/// Convert a UTF-8 string to a byte vector in a specified encoding. +pub async fn from_utf8(input: String, target: Encoding) -> anyhow::Result> { + target.encode(input).await +} + +pub struct EncodingOptions { + pub encoding: Arc, + pub force: AtomicBool, + pub detect_utf16: AtomicBool, +} + +impl EncodingOptions { + pub fn reset(&self) { + self.encoding.reset(); + + self.force + .store(false, std::sync::atomic::Ordering::Release); + + self.detect_utf16 + .store(true, std::sync::atomic::Ordering::Release); + } +} + +impl Default for EncodingOptions { + fn default() -> Self { + EncodingOptions { + encoding: Arc::new(Encoding::default()), + force: AtomicBool::new(false), + detect_utf16: AtomicBool::new(true), + } + } +} + +impl Clone for EncodingOptions { + fn clone(&self) -> Self { + EncodingOptions { + encoding: Arc::new(self.encoding.get().into()), + force: AtomicBool::new(self.force.load(std::sync::atomic::Ordering::Acquire)), + detect_utf16: AtomicBool::new( + self.detect_utf16.load(std::sync::atomic::Ordering::Acquire), + ), + } + } +} diff --git a/crates/encodings_ui/Cargo.toml b/crates/encodings_ui/Cargo.toml new file mode 100644 index 00000000000000..64c5d14161963e --- /dev/null +++ b/crates/encodings_ui/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "encodings_ui" +version = "0.1.0" +publish.workspace = true +edition.workspace = true + +[dependencies] +anyhow.workspace = true +editor.workspace = true +encoding_rs.workspace = true +fs.workspace = true +futures.workspace = true +fuzzy.workspace = true +gpui.workspace = true +language.workspace = true +picker.workspace = true +settings.workspace = true +ui.workspace = true +util.workspace = true +workspace.workspace = true +zed_actions.workspace = true + + +[lints] +workspace = true diff --git a/crates/encodings_ui/LICENSE-GPL b/crates/encodings_ui/LICENSE-GPL new file mode 120000 index 00000000000000..89e542f750cd38 --- /dev/null +++ b/crates/encodings_ui/LICENSE-GPL @@ -0,0 +1 @@ +../../LICENSE-GPL \ No newline at end of file diff --git a/crates/encodings_ui/src/lib.rs b/crates/encodings_ui/src/lib.rs new file mode 100644 index 00000000000000..1cef23b7010e3f --- /dev/null +++ b/crates/encodings_ui/src/lib.rs @@ -0,0 +1,345 @@ +//! A crate for handling file encodings in the text editor. + +use crate::selectors::encoding::Action; +use editor::Editor; +use encoding_rs::Encoding; +use gpui::{ClickEvent, Entity, Subscription, WeakEntity}; +use language::Buffer; +use ui::{App, Button, ButtonCommon, Context, LabelSize, Render, Tooltip, Window, div}; +use ui::{Clickable, ParentElement}; +use util::ResultExt; +use workspace::{ + CloseActiveItem, ItemHandle, OpenOptions, StatusItemView, Workspace, + with_active_or_new_workspace, +}; +use zed_actions::encodings_ui::{ForceOpen, Toggle}; + +use crate::selectors::encoding::EncodingSelector; +use crate::selectors::save_or_reopen::EncodingSaveOrReopenSelector; + +/// A status bar item that shows the current file encoding and allows changing it. +pub struct EncodingIndicator { + pub encoding: Option<&'static Encoding>, + pub workspace: WeakEntity, + + /// Subscription to observe changes in the active editor + observe_editor: Option, + + /// Subscription to observe changes in the `encoding` field of the `Buffer` struct + observe_buffer_encoding: Option, + + /// Whether to show the indicator or not, based on whether an editor is active + show: bool, + + /// Whether to show `EncodingSaveOrReopenSelector`. It will be shown only when + /// the current buffer is associated with a file. + show_save_or_reopen_selector: bool, +} + +pub mod selectors; + +impl Render for EncodingIndicator { + fn render(&mut self, _window: &mut Window, cx: &mut Context) -> impl ui::IntoElement { + let status_element = div(); + let show_save_or_reopen_selector = self.show_save_or_reopen_selector; + + if !self.show { + return status_element; + } + + status_element.child( + Button::new( + "encoding", + encoding_name(self.encoding.unwrap_or(encoding_rs::UTF_8)), + ) + .label_size(LabelSize::Small) + .tooltip(Tooltip::text("Select Encoding")) + .on_click(cx.listener(move |indicator, _: &ClickEvent, window, cx| { + if let Some(workspace) = indicator.workspace.upgrade() { + workspace.update(cx, move |workspace, cx| { + // Open the `EncodingSaveOrReopenSelector` if the buffer is associated with a file, + if show_save_or_reopen_selector { + EncodingSaveOrReopenSelector::toggle(workspace, window, cx) + } + // otherwise, open the `EncodingSelector` directly. + else { + let (_, buffer, _) = workspace + .active_item(cx) + .unwrap() + .act_as::(cx) + .unwrap() + .read(cx) + .active_excerpt(cx) + .unwrap(); + + let weak_workspace = workspace.weak_handle(); + + if let Some(path) = buffer.read(cx).file() { + let path = path.clone().path().to_rel_path_buf(); + workspace.toggle_modal(window, cx, |window, cx| { + let selector = EncodingSelector::new( + window, + cx, + Action::Save, + Some(buffer.downgrade()), + weak_workspace, + Some(path.as_std_path().to_path_buf()), + ); + selector + }); + } + } + }) + } + })), + ) + } +} + +impl EncodingIndicator { + pub fn new( + encoding: Option<&'static Encoding>, + workspace: WeakEntity, + observe_editor: Option, + observe_buffer_encoding: Option, + ) -> EncodingIndicator { + EncodingIndicator { + encoding, + workspace, + observe_editor, + show: false, + observe_buffer_encoding, + show_save_or_reopen_selector: false, + } + } + + /// Update the encoding when the active editor is switched. + pub fn update_when_editor_is_switched( + &mut self, + editor: Entity, + _: &mut Window, + cx: &mut Context, + ) { + let editor = editor.read(cx); + if let Some((_, buffer, _)) = editor.active_excerpt(cx) { + let encoding = buffer.read(cx).encoding.clone(); + self.encoding = Some(encoding.get()); + + if let Some(_) = buffer.read(cx).file() { + self.show_save_or_reopen_selector = true; + } else { + self.show_save_or_reopen_selector = false; + } + } + + cx.notify(); + } + + /// Update the encoding when the `encoding` field of the `Buffer` struct changes. + pub fn update_when_buffer_encoding_changes( + &mut self, + buffer: Entity, + _: &mut Window, + cx: &mut Context, + ) { + let encoding = buffer.read(cx).encoding.clone(); + self.encoding = Some(encoding.get()); + cx.notify(); + } +} + +impl StatusItemView for EncodingIndicator { + fn set_active_pane_item( + &mut self, + active_pane_item: Option<&dyn ItemHandle>, + window: &mut Window, + cx: &mut Context, + ) { + match active_pane_item.and_then(|item| item.downcast::()) { + Some(editor) => { + self.observe_editor = + Some(cx.observe_in(&editor, window, Self::update_when_editor_is_switched)); + if let Some((_, buffer, _)) = &editor.read(cx).active_excerpt(cx) { + self.observe_buffer_encoding = Some(cx.observe_in( + buffer, + window, + Self::update_when_buffer_encoding_changes, + )); + } + self.update_when_editor_is_switched(editor, window, cx); + self.show = true; + } + None => { + self.encoding = None; + self.observe_editor = None; + self.show = false; + } + } + } +} + +/// Get a human-readable name for the given encoding. +pub fn encoding_name(encoding: &'static Encoding) -> String { + let name = encoding.name(); + + match name { + "UTF-8" => "UTF-8", + "UTF-16LE" => "UTF-16 LE", + "UTF-16BE" => "UTF-16 BE", + "windows-1252" => "Windows-1252", + "windows-1251" => "Windows-1251", + "windows-1250" => "Windows-1250", + "ISO-8859-2" => "ISO 8859-2", + "ISO-8859-3" => "ISO 8859-3", + "ISO-8859-4" => "ISO 8859-4", + "ISO-8859-5" => "ISO 8859-5", + "ISO-8859-6" => "ISO 8859-6", + "ISO-8859-7" => "ISO 8859-7", + "ISO-8859-8" => "ISO 8859-8", + "ISO-8859-13" => "ISO 8859-13", + "ISO-8859-15" => "ISO 8859-15", + "KOI8-R" => "KOI8-R", + "KOI8-U" => "KOI8-U", + "macintosh" => "MacRoman", + "x-mac-cyrillic" => "Mac Cyrillic", + "windows-874" => "Windows-874", + "windows-1253" => "Windows-1253", + "windows-1254" => "Windows-1254", + "windows-1255" => "Windows-1255", + "windows-1256" => "Windows-1256", + "windows-1257" => "Windows-1257", + "windows-1258" => "Windows-1258", + "EUC-KR" => "Windows-949", + "EUC-JP" => "EUC-JP", + "ISO-2022-JP" => "ISO 2022-JP", + "GBK" => "GBK", + "gb18030" => "GB18030", + "Big5" => "Big5", + _ => name, + } + .to_string() +} + +/// Get an encoding from its index in the predefined list. +/// If the index is out of range, UTF-8 is returned as a default. +pub fn encoding_from_index(index: usize) -> &'static Encoding { + match index { + 0 => encoding_rs::UTF_8, + 1 => encoding_rs::UTF_16LE, + 2 => encoding_rs::UTF_16BE, + 3 => encoding_rs::WINDOWS_1252, + 4 => encoding_rs::WINDOWS_1251, + 5 => encoding_rs::WINDOWS_1250, + 6 => encoding_rs::ISO_8859_2, + 7 => encoding_rs::ISO_8859_3, + 8 => encoding_rs::ISO_8859_4, + 9 => encoding_rs::ISO_8859_5, + 10 => encoding_rs::ISO_8859_6, + 11 => encoding_rs::ISO_8859_7, + 12 => encoding_rs::ISO_8859_8, + 13 => encoding_rs::ISO_8859_13, + 14 => encoding_rs::ISO_8859_15, + 15 => encoding_rs::KOI8_R, + 16 => encoding_rs::KOI8_U, + 17 => encoding_rs::MACINTOSH, + 18 => encoding_rs::X_MAC_CYRILLIC, + 19 => encoding_rs::WINDOWS_874, + 20 => encoding_rs::WINDOWS_1253, + 21 => encoding_rs::WINDOWS_1254, + 22 => encoding_rs::WINDOWS_1255, + 23 => encoding_rs::WINDOWS_1256, + 24 => encoding_rs::WINDOWS_1257, + 25 => encoding_rs::WINDOWS_1258, + 26 => encoding_rs::EUC_KR, + 27 => encoding_rs::EUC_JP, + 28 => encoding_rs::ISO_2022_JP, + 29 => encoding_rs::GBK, + 30 => encoding_rs::GB18030, + 31 => encoding_rs::BIG5, + _ => encoding_rs::UTF_8, + } +} + +/// Get an encoding from its name. +pub fn encoding_from_name(name: &str) -> &'static Encoding { + match name { + "UTF-8" => encoding_rs::UTF_8, + "UTF-16 LE" => encoding_rs::UTF_16LE, + "UTF-16 BE" => encoding_rs::UTF_16BE, + "Windows-1252" => encoding_rs::WINDOWS_1252, + "Windows-1251" => encoding_rs::WINDOWS_1251, + "Windows-1250" => encoding_rs::WINDOWS_1250, + "ISO 8859-2" => encoding_rs::ISO_8859_2, + "ISO 8859-3" => encoding_rs::ISO_8859_3, + "ISO 8859-4" => encoding_rs::ISO_8859_4, + "ISO 8859-5" => encoding_rs::ISO_8859_5, + "ISO 8859-6" => encoding_rs::ISO_8859_6, + "ISO 8859-7" => encoding_rs::ISO_8859_7, + "ISO 8859-8" => encoding_rs::ISO_8859_8, + "ISO 8859-13" => encoding_rs::ISO_8859_13, + "ISO 8859-15" => encoding_rs::ISO_8859_15, + "KOI8-R" => encoding_rs::KOI8_R, + "KOI8-U" => encoding_rs::KOI8_U, + "MacRoman" => encoding_rs::MACINTOSH, + "Mac Cyrillic" => encoding_rs::X_MAC_CYRILLIC, + "Windows-874" => encoding_rs::WINDOWS_874, + "Windows-1253" => encoding_rs::WINDOWS_1253, + "Windows-1254" => encoding_rs::WINDOWS_1254, + "Windows-1255" => encoding_rs::WINDOWS_1255, + "Windows-1256" => encoding_rs::WINDOWS_1256, + "Windows-1257" => encoding_rs::WINDOWS_1257, + "Windows-1258" => encoding_rs::WINDOWS_1258, + "Windows-949" => encoding_rs::EUC_KR, + "EUC-JP" => encoding_rs::EUC_JP, + "ISO 2022-JP" => encoding_rs::ISO_2022_JP, + "GBK" => encoding_rs::GBK, + "GB18030" => encoding_rs::GB18030, + "Big5" => encoding_rs::BIG5, + _ => encoding_rs::UTF_8, // Default to UTF-8 for unknown names + } +} + +pub fn init(cx: &mut App) { + cx.on_action(|action: &Toggle, cx: &mut App| { + let Toggle(path) = action.clone(); + let path = path.to_path_buf(); + + with_active_or_new_workspace(cx, |workspace, window, cx| { + let weak_workspace = workspace.weak_handle(); + workspace.toggle_modal(window, cx, |window, cx| { + EncodingSelector::new(window, cx, Action::Reopen, None, weak_workspace, Some(path)) + }); + }); + }); + + cx.on_action(|action: &ForceOpen, cx: &mut App| { + let ForceOpen(path) = action.clone(); + let path = path.to_path_buf(); + + with_active_or_new_workspace(cx, |workspace, window, cx| { + workspace.active_pane().update(cx, |pane, cx| { + pane.close_active_item(&CloseActiveItem::default(), window, cx) + .detach(); + }); + + workspace + .encoding_options + .force + .store(true, std::sync::atomic::Ordering::Release); + + let open_task = workspace.open_abs_path(path, OpenOptions::default(), window, cx); + let weak_workspace = workspace.weak_handle(); + + cx.spawn(async move |_, cx| { + let workspace = weak_workspace.upgrade().unwrap(); + open_task.await.log_err(); + workspace + .update(cx, |workspace: &mut Workspace, _| { + *workspace.encoding_options.force.get_mut() = false; + }) + .log_err(); + }) + .detach(); + }); + }); +} diff --git a/crates/encodings_ui/src/selectors.rs b/crates/encodings_ui/src/selectors.rs new file mode 100644 index 00000000000000..fc5a6793a96f2a --- /dev/null +++ b/crates/encodings_ui/src/selectors.rs @@ -0,0 +1,632 @@ +/// This module contains the encoding selectors for saving or reopening files with a different encoding. +/// It provides a modal view that allows the user to choose between saving with a different encoding +/// or reopening with a different encoding. +pub mod save_or_reopen { + use editor::Editor; + use gpui::Styled; + use gpui::{AppContext, ParentElement}; + use picker::Picker; + use picker::PickerDelegate; + use std::sync::atomic::AtomicBool; + use util::ResultExt; + + use fuzzy::{StringMatch, StringMatchCandidate}; + use gpui::{DismissEvent, Entity, EventEmitter, Focusable, WeakEntity}; + + use ui::{Context, HighlightedLabel, ListItem, Render, Window, rems, v_flex}; + use workspace::{ModalView, Workspace}; + + use crate::selectors::encoding::{Action, EncodingSelector}; + + /// A modal view that allows the user to select between saving with a different encoding or + /// reopening with a different encoding. + pub struct EncodingSaveOrReopenSelector { + picker: Entity>, + pub current_selection: usize, + } + + impl EncodingSaveOrReopenSelector { + pub fn new( + window: &mut Window, + cx: &mut Context, + workspace: WeakEntity, + ) -> Self { + let delegate = EncodingSaveOrReopenDelegate::new(cx.entity().downgrade(), workspace); + + let picker = cx.new(|cx| Picker::uniform_list(delegate, window, cx)); + + Self { + picker, + current_selection: 0, + } + } + + /// Toggle the modal view for selecting between saving with a different encoding or + /// reopening with a different encoding. + pub fn toggle(workspace: &mut Workspace, window: &mut Window, cx: &mut Context) { + let weak_workspace = workspace.weak_handle(); + workspace.toggle_modal(window, cx, |window, cx| { + EncodingSaveOrReopenSelector::new(window, cx, weak_workspace) + }); + } + } + + impl Focusable for EncodingSaveOrReopenSelector { + fn focus_handle(&self, cx: &ui::App) -> gpui::FocusHandle { + self.picker.focus_handle(cx) + } + } + + impl Render for EncodingSaveOrReopenSelector { + fn render( + &mut self, + _window: &mut Window, + _cx: &mut Context, + ) -> impl ui::IntoElement { + v_flex().w(rems(34.0)).child(self.picker.clone()) + } + } + + impl ModalView for EncodingSaveOrReopenSelector {} + + impl EventEmitter for EncodingSaveOrReopenSelector {} + + pub struct EncodingSaveOrReopenDelegate { + selector: WeakEntity, + current_selection: usize, + matches: Vec, + pub actions: Vec, + workspace: WeakEntity, + } + + impl EncodingSaveOrReopenDelegate { + pub fn new( + selector: WeakEntity, + workspace: WeakEntity, + ) -> Self { + Self { + selector, + current_selection: 0, + matches: Vec::new(), + actions: vec![ + StringMatchCandidate::new(0, "Save with encoding"), + StringMatchCandidate::new(1, "Reopen with encoding"), + ], + workspace, + } + } + + pub fn get_actions(&self) -> (&str, &str) { + (&self.actions[0].string, &self.actions[1].string) + } + + /// Handle the action selected by the user. + pub fn post_selection( + &self, + cx: &mut Context>, + window: &mut Window, + ) -> Option<()> { + if self.current_selection == 0 { + if let Some(workspace) = self.workspace.upgrade() { + let (_, buffer, _) = workspace + .read(cx) + .active_item(cx)? + .act_as::(cx)? + .read(cx) + .active_excerpt(cx)?; + + let weak_workspace = workspace.read(cx).weak_handle(); + + if let Some(file) = buffer.read(cx).file() { + let path = file.as_local()?.abs_path(cx); + + workspace.update(cx, |workspace, cx| { + workspace.toggle_modal(window, cx, |window, cx| { + let selector = EncodingSelector::new( + window, + cx, + Action::Save, + Some(buffer.downgrade()), + weak_workspace, + Some(path), + ); + selector + }) + }); + } + } + } else if self.current_selection == 1 { + if let Some(workspace) = self.workspace.upgrade() { + let (_, buffer, _) = workspace + .read(cx) + .active_item(cx)? + .act_as::(cx)? + .read(cx) + .active_excerpt(cx)?; + + let weak_workspace = workspace.read(cx).weak_handle(); + + if let Some(file) = buffer.read(cx).file() { + let path = file.as_local()?.abs_path(cx); + + workspace.update(cx, |workspace, cx| { + workspace.toggle_modal(window, cx, |window, cx| { + let selector = EncodingSelector::new( + window, + cx, + Action::Reopen, + Some(buffer.downgrade()), + weak_workspace, + Some(path), + ); + selector + }); + }); + } + } + } + + Some(()) + } + } + + impl PickerDelegate for EncodingSaveOrReopenDelegate { + type ListItem = ListItem; + + fn match_count(&self) -> usize { + self.matches.len() + } + + fn selected_index(&self) -> usize { + self.current_selection + } + + fn set_selected_index( + &mut self, + ix: usize, + _window: &mut Window, + cx: &mut Context>, + ) { + self.current_selection = ix; + self.selector + .update(cx, |selector, _cx| { + selector.current_selection = ix; + }) + .log_err(); + } + + fn placeholder_text(&self, _window: &mut Window, _cx: &mut ui::App) -> std::sync::Arc { + "Select an action...".into() + } + + fn update_matches( + &mut self, + query: String, + window: &mut Window, + cx: &mut Context>, + ) -> gpui::Task<()> { + let executor = cx.background_executor().clone(); + let actions = self.actions.clone(); + + cx.spawn_in(window, async move |this, cx| { + let matches = if query.is_empty() { + actions + .into_iter() + .enumerate() + .map(|(index, value)| StringMatch { + candidate_id: index, + score: 0.0, + positions: vec![], + string: value.string, + }) + .collect::>() + } else { + fuzzy::match_strings( + &actions, + &query, + false, + false, + 2, + &AtomicBool::new(false), + executor, + ) + .await + }; + + this.update(cx, |picker, cx| { + let delegate = &mut picker.delegate; + delegate.matches = matches; + delegate.current_selection = delegate + .current_selection + .min(delegate.matches.len().saturating_sub(1)); + delegate + .selector + .update(cx, |selector, _cx| { + selector.current_selection = delegate.current_selection + }) + .log_err(); + cx.notify(); + }) + .log_err(); + }) + } + + fn confirm(&mut self, _: bool, window: &mut Window, cx: &mut Context>) { + self.dismissed(window, cx); + if self.selector.is_upgradable() { + self.post_selection(cx, window); + } + } + + fn dismissed(&mut self, _window: &mut Window, cx: &mut Context>) { + self.selector + .update(cx, |_, cx| cx.emit(DismissEvent)) + .log_err(); + } + + fn render_match( + &self, + ix: usize, + _: bool, + _: &mut Window, + _: &mut Context>, + ) -> Option { + Some( + ListItem::new(ix) + .child(HighlightedLabel::new( + &self.matches[ix].string, + self.matches[ix].positions.clone(), + )) + .spacing(ui::ListItemSpacing::Sparse), + ) + } + } +} + +/// This module contains the encoding selector for choosing an encoding to save or reopen a file with. +pub mod encoding { + use editor::Editor; + use std::{path::PathBuf, sync::atomic::AtomicBool}; + + use fuzzy::{StringMatch, StringMatchCandidate}; + use gpui::{ + AppContext, DismissEvent, Entity, EventEmitter, Focusable, WeakEntity, http_client::anyhow, + }; + use language::Buffer; + use picker::{Picker, PickerDelegate}; + use ui::{ + Context, HighlightedLabel, ListItem, ListItemSpacing, ParentElement, Render, Styled, + Window, rems, v_flex, + }; + use util::ResultExt; + use workspace::{CloseActiveItem, ModalView, OpenOptions, Workspace}; + + use crate::encoding_from_name; + + /// A modal view that allows the user to select an encoding from a list of encodings. + pub struct EncodingSelector { + picker: Entity>, + workspace: WeakEntity, + path: Option, + } + + pub struct EncodingSelectorDelegate { + current_selection: usize, + encodings: Vec, + matches: Vec, + selector: WeakEntity, + buffer: Option>, + action: Action, + } + + impl EncodingSelectorDelegate { + pub fn new( + selector: WeakEntity, + buffer: Option>, + action: Action, + ) -> EncodingSelectorDelegate { + EncodingSelectorDelegate { + current_selection: 0, + encodings: vec![ + StringMatchCandidate::new(0, "UTF-8"), + StringMatchCandidate::new(1, "UTF-16 LE"), + StringMatchCandidate::new(2, "UTF-16 BE"), + StringMatchCandidate::new(3, "Windows-1252"), + StringMatchCandidate::new(4, "Windows-1251"), + StringMatchCandidate::new(5, "Windows-1250"), + StringMatchCandidate::new(6, "ISO 8859-2"), + StringMatchCandidate::new(7, "ISO 8859-3"), + StringMatchCandidate::new(8, "ISO 8859-4"), + StringMatchCandidate::new(9, "ISO 8859-5"), + StringMatchCandidate::new(10, "ISO 8859-6"), + StringMatchCandidate::new(11, "ISO 8859-7"), + StringMatchCandidate::new(12, "ISO 8859-8"), + StringMatchCandidate::new(13, "ISO 8859-13"), + StringMatchCandidate::new(14, "ISO 8859-15"), + StringMatchCandidate::new(15, "KOI8-R"), + StringMatchCandidate::new(16, "KOI8-U"), + StringMatchCandidate::new(17, "MacRoman"), + StringMatchCandidate::new(18, "Mac Cyrillic"), + StringMatchCandidate::new(19, "Windows-874"), + StringMatchCandidate::new(20, "Windows-1253"), + StringMatchCandidate::new(21, "Windows-1254"), + StringMatchCandidate::new(22, "Windows-1255"), + StringMatchCandidate::new(23, "Windows-1256"), + StringMatchCandidate::new(24, "Windows-1257"), + StringMatchCandidate::new(25, "Windows-1258"), + StringMatchCandidate::new(26, "Windows-949"), + StringMatchCandidate::new(27, "EUC-JP"), + StringMatchCandidate::new(28, "ISO 2022-JP"), + StringMatchCandidate::new(29, "GBK"), + StringMatchCandidate::new(30, "GB18030"), + StringMatchCandidate::new(31, "Big5"), + ], + matches: Vec::new(), + selector, + buffer: buffer, + action, + } + } + } + + impl PickerDelegate for EncodingSelectorDelegate { + type ListItem = ListItem; + + fn match_count(&self) -> usize { + self.matches.len() + } + + fn selected_index(&self) -> usize { + self.current_selection + } + + fn set_selected_index(&mut self, ix: usize, _: &mut Window, _: &mut Context>) { + self.current_selection = ix; + } + + fn placeholder_text(&self, _window: &mut Window, _cx: &mut ui::App) -> std::sync::Arc { + "Select an encoding...".into() + } + + fn update_matches( + &mut self, + query: String, + window: &mut Window, + cx: &mut Context>, + ) -> gpui::Task<()> { + let executor = cx.background_executor().clone(); + let encodings = self.encodings.clone(); + + cx.spawn_in(window, async move |picker, cx| { + let matches: Vec; + + if query.is_empty() { + matches = encodings + .into_iter() + .enumerate() + .map(|(index, value)| StringMatch { + candidate_id: index, + score: 0.0, + positions: Vec::new(), + string: value.string, + }) + .collect(); + } else { + matches = fuzzy::match_strings( + &encodings, + &query, + true, + false, + 30, + &AtomicBool::new(false), + executor, + ) + .await + } + picker + .update(cx, |picker, cx| { + let delegate = &mut picker.delegate; + delegate.matches = matches; + delegate.current_selection = delegate + .current_selection + .min(delegate.matches.len().saturating_sub(1)); + cx.notify(); + }) + .log_err(); + }) + } + + fn confirm(&mut self, _: bool, window: &mut Window, cx: &mut Context>) { + let workspace = self + .selector + .upgrade() + .unwrap() + .read(cx) + .workspace + .upgrade() + .unwrap(); + + let weak_workspace = workspace.read(cx).weak_handle(); + + let current_selection = self.matches[self.current_selection].string.clone(); + + if let Some(buffer) = &self.buffer + && let Some(buffer) = buffer.upgrade() + { + let path = self + .selector + .upgrade() + .unwrap() + .read(cx) + .path + .clone() + .unwrap(); + + let reload = buffer.update(cx, |buffer, cx| buffer.reload(cx)); + + buffer.update(cx, |buffer, _| { + buffer.update_encoding(encoding_from_name(¤t_selection).into()) + }); + + self.dismissed(window, cx); + + if self.action == Action::Reopen { + buffer.update(cx, |_, cx| { + cx.spawn_in(window, async move |_, cx| { + if let Err(_) | Ok(None) = reload.await { + let workspace = weak_workspace.upgrade().unwrap(); + + workspace + .update_in(cx, |workspace, window, cx| { + workspace + .encoding_options + .encoding + .set(encoding_from_name(¤t_selection)); + + *workspace.encoding_options.force.get_mut() = false; + + *workspace.encoding_options.detect_utf16.get_mut() = true; + + workspace + .active_pane() + .update(cx, |pane, cx| { + pane.close_active_item( + &CloseActiveItem::default(), + window, + cx, + ) + }) + .detach(); + + workspace + .open_abs_path(path, OpenOptions::default(), window, cx) + .detach() + }) + .log_err(); + } + }) + .detach() + }); + } else if self.action == Action::Save { + workspace.update(cx, |workspace, cx| { + workspace + .save_active_item(workspace::SaveIntent::Save, window, cx) + .detach(); + }); + } + } else { + if let Some(path) = self.selector.upgrade().unwrap().read(cx).path.clone() { + workspace.update(cx, |workspace, cx| { + workspace.active_pane().update(cx, |pane, cx| { + pane.close_active_item(&CloseActiveItem::default(), window, cx) + .detach(); + }); + }); + + let encoding = + encoding_from_name(self.matches[self.current_selection].string.as_str()); + + let open_task = workspace.update(cx, |workspace, cx| { + workspace.encoding_options.encoding.set(encoding); + + workspace.open_abs_path(path, OpenOptions::default(), window, cx) + }); + + cx.spawn(async move |_, cx| { + if let Ok(_) = { + let result = open_task.await; + workspace + .update(cx, |workspace, _| { + *workspace.encoding_options.force.get_mut() = false; + }) + .unwrap(); + + result + } && let Ok(Ok((_, buffer, _))) = + workspace.read_with(cx, |workspace, cx| { + if let Some(active_item) = workspace.active_item(cx) + && let Some(editor) = active_item.act_as::(cx) + { + Ok(editor.read(cx).active_excerpt(cx).unwrap()) + } else { + Err(anyhow!("error")) + } + }) + { + buffer + .update(cx, |buffer, _| buffer.update_encoding(encoding.into())) + .log_err(); + } + }) + .detach(); + } + } + } + + fn dismissed(&mut self, _: &mut Window, cx: &mut Context>) { + self.selector + .update(cx, |_, cx| cx.emit(DismissEvent)) + .log_err(); + } + + fn render_match( + &self, + ix: usize, + _: bool, + _: &mut Window, + _: &mut Context>, + ) -> Option { + Some( + ListItem::new(ix) + .child(HighlightedLabel::new( + &self.matches[ix].string, + self.matches[ix].positions.clone(), + )) + .spacing(ListItemSpacing::Sparse), + ) + } + } + + /// The action to perform after selecting an encoding. + #[derive(PartialEq, Clone)] + pub enum Action { + Save, + Reopen, + } + + impl EncodingSelector { + pub fn new( + window: &mut Window, + cx: &mut Context, + action: Action, + buffer: Option>, + workspace: WeakEntity, + path: Option, + ) -> EncodingSelector { + let delegate = EncodingSelectorDelegate::new(cx.entity().downgrade(), buffer, action); + let picker = cx.new(|cx| Picker::uniform_list(delegate, window, cx)); + + EncodingSelector { + picker, + workspace, + path, + } + } + } + + impl EventEmitter for EncodingSelector {} + + impl Focusable for EncodingSelector { + fn focus_handle(&self, cx: &ui::App) -> gpui::FocusHandle { + self.picker.focus_handle(cx) + } + } + + impl ModalView for EncodingSelector {} + + impl Render for EncodingSelector { + fn render(&mut self, _: &mut Window, _: &mut Context) -> impl ui::IntoElement { + v_flex().w(rems(34.0)).child(self.picker.clone()) + } + } +} diff --git a/crates/extension_host/Cargo.toml b/crates/extension_host/Cargo.toml index 16cbd9ac0c0ef9..229f2e2d185c4b 100644 --- a/crates/extension_host/Cargo.toml +++ b/crates/extension_host/Cargo.toml @@ -23,6 +23,7 @@ async-trait.workspace = true client.workspace = true collections.workspace = true dap.workspace = true +encodings.workspace = true extension.workspace = true fs.workspace = true futures.workspace = true diff --git a/crates/extension_host/src/extension_host.rs b/crates/extension_host/src/extension_host.rs index 50b5169f7ad119..c58bce9d51a877 100644 --- a/crates/extension_host/src/extension_host.rs +++ b/crates/extension_host/src/extension_host.rs @@ -12,6 +12,7 @@ use async_tar::Archive; use client::ExtensionProvides; use client::{Client, ExtensionMetadata, GetExtensionsResponse, proto, telemetry::Telemetry}; use collections::{BTreeMap, BTreeSet, HashMap, HashSet, btree_map}; +use encodings::Encoding; pub use extension::ExtensionManifest; use extension::extension_builder::{CompileExtensionOptions, ExtensionBuilder}; use extension::{ @@ -1506,6 +1507,7 @@ impl ExtensionStore { &index_path, &Rope::from_str(&index_json, &executor), Default::default(), + Encoding::default(), ) .await .context("failed to save extension index") @@ -1678,6 +1680,7 @@ impl ExtensionStore { &tmp_dir.join(EXTENSION_TOML), &Rope::from_str_small(&manifest_toml), language::LineEnding::Unix, + Encoding::default(), ) .await?; } else { diff --git a/crates/fs/Cargo.toml b/crates/fs/Cargo.toml index d6413cb7a07b5a..25867218d57176 100644 --- a/crates/fs/Cargo.toml +++ b/crates/fs/Cargo.toml @@ -16,6 +16,7 @@ anyhow.workspace = true async-tar.workspace = true async-trait.workspace = true collections.workspace = true +encodings.workspace = true futures.workspace = true git.workspace = true gpui.workspace = true @@ -33,6 +34,8 @@ tempfile.workspace = true text.workspace = true time.workspace = true util.workspace = true +encoding = "0.2.33" + [target.'cfg(target_os = "macos")'.dependencies] fsevent.workspace = true diff --git a/crates/fs/src/fs.rs b/crates/fs/src/fs.rs index c794303ef71232..59719e47af4a44 100644 --- a/crates/fs/src/fs.rs +++ b/crates/fs/src/fs.rs @@ -58,6 +58,7 @@ use smol::io::AsyncReadExt; #[cfg(any(test, feature = "test-support"))] use std::ffi::OsStr; +use encodings::{Encoding, EncodingOptions, from_utf8, to_utf8}; #[cfg(any(test, feature = "test-support"))] pub use fake_git_repo::{LOAD_HEAD_TEXT_TASK, LOAD_INDEX_TEXT_TASK}; @@ -115,9 +116,25 @@ pub trait Fs: Send + Sync { async fn load(&self, path: &Path) -> Result { Ok(String::from_utf8(self.load_bytes(path).await?)?) } + + async fn load_with_encoding( + &self, + path: &Path, + options: &EncodingOptions, + buffer_encoding: Option>, + ) -> Result { + Ok(to_utf8(self.load_bytes(path).await?, options, buffer_encoding).await?) + } + async fn load_bytes(&self, path: &Path) -> Result>; async fn atomic_write(&self, path: PathBuf, text: String) -> Result<()>; - async fn save(&self, path: &Path, text: &Rope, line_ending: LineEnding) -> Result<()>; + async fn save( + &self, + path: &Path, + text: &Rope, + line_ending: LineEnding, + encoding: Encoding, + ) -> Result<()>; async fn write(&self, path: &Path, content: &[u8]) -> Result<()>; async fn canonicalize(&self, path: &Path) -> Result; async fn is_file(&self, path: &Path) -> bool; @@ -599,9 +616,8 @@ impl Fs for RealFs { async fn load(&self, path: &Path) -> Result { let path = path.to_path_buf(); - self.executor - .spawn(async move { Ok(std::fs::read_to_string(path)?) }) - .await + let text = smol::unblock(|| std::fs::read_to_string(path)).await?; + Ok(text) } async fn load_bytes(&self, path: &Path) -> Result> { @@ -659,16 +675,37 @@ impl Fs for RealFs { Ok(()) } - async fn save(&self, path: &Path, text: &Rope, line_ending: LineEnding) -> Result<()> { + async fn save( + &self, + path: &Path, + text: &Rope, + line_ending: LineEnding, + encoding: Encoding, + ) -> Result<()> { let buffer_size = text.summary().len.min(10 * 1024); if let Some(path) = path.parent() { self.create_dir(path).await?; } let file = smol::fs::File::create(path).await?; let mut writer = smol::io::BufWriter::with_capacity(buffer_size, file); + + // BOM for UTF-16 is written at the start of the file here because + // if BOM is written in the `encode` function of `fs::encodings`, it would be written + // twice. Hence, it is written only here. + if encoding.get() == encodings::UTF_16BE { + // Write BOM for UTF-16BE + writer.write_all(&[0xFE, 0xFF]).await?; + } else if encoding.get() == encodings::UTF_16LE { + // Write BOM for UTF-16LE + writer.write_all(&[0xFF, 0xFE]).await?; + } + for chunk in chunks(text, line_ending) { - writer.write_all(chunk.as_bytes()).await?; + writer + .write_all(&from_utf8(chunk.to_string(), Encoding::new(encoding.get())).await?) + .await? } + writer.flush().await?; Ok(()) } @@ -2380,14 +2417,22 @@ impl Fs for FakeFs { Ok(()) } - async fn save(&self, path: &Path, text: &Rope, line_ending: LineEnding) -> Result<()> { + async fn save( + &self, + path: &Path, + text: &Rope, + line_ending: LineEnding, + encoding: Encoding, + ) -> Result<()> { + use encodings::from_utf8; + self.simulate_random_delay().await; let path = normalize_path(path); let content = chunks(text, line_ending).collect::(); if let Some(path) = path.parent() { self.create_dir(path).await?; } - self.write_file_internal(path, content.into_bytes(), false)?; + self.write_file_internal(path, from_utf8(content, encoding).await?, false)?; Ok(()) } diff --git a/crates/git_ui/Cargo.toml b/crates/git_ui/Cargo.toml index 486e43fea94f53..6a8d9d3daf10b6 100644 --- a/crates/git_ui/Cargo.toml +++ b/crates/git_ui/Cargo.toml @@ -29,6 +29,7 @@ command_palette_hooks.workspace = true component.workspace = true db.workspace = true editor.workspace = true +encodings.workspace = true futures.workspace = true fuzzy.workspace = true git.workspace = true diff --git a/crates/git_ui/src/file_diff_view.rs b/crates/git_ui/src/file_diff_view.rs index a99b7f8e2428ca..27811859b1ab95 100644 --- a/crates/git_ui/src/file_diff_view.rs +++ b/crates/git_ui/src/file_diff_view.rs @@ -358,6 +358,7 @@ impl Render for FileDiffView { mod tests { use super::*; use editor::test::editor_test_context::assert_state_with_diff; + use encodings::Encoding; use gpui::TestAppContext; use language::Rope; use project::{FakeFs, Fs, Project}; @@ -440,6 +441,7 @@ mod tests { ", )), Default::default(), + Encoding::default(), ) .await .unwrap(); @@ -474,6 +476,7 @@ mod tests { ", )), Default::default(), + Encoding::default(), ) .await .unwrap(); diff --git a/crates/language/Cargo.toml b/crates/language/Cargo.toml index ffc5ad85d14c29..d24934ee24d400 100644 --- a/crates/language/Cargo.toml +++ b/crates/language/Cargo.toml @@ -30,7 +30,9 @@ anyhow.workspace = true async-trait.workspace = true clock.workspace = true collections.workspace = true +diffy = "0.4.2" ec4rs.workspace = true +encodings.workspace = true fs.workspace = true futures.workspace = true fuzzy.workspace = true @@ -67,7 +69,7 @@ unicase = "2.6" util.workspace = true watch.workspace = true zlog.workspace = true -diffy = "0.4.2" +encoding = "0.2.33" [dev-dependencies] collections = { workspace = true, features = ["test-support"] } diff --git a/crates/language/src/buffer.rs b/crates/language/src/buffer.rs index d67434741032ae..a82c49ae546980 100644 --- a/crates/language/src/buffer.rs +++ b/crates/language/src/buffer.rs @@ -21,6 +21,7 @@ use anyhow::{Context as _, Result}; use clock::Lamport; pub use clock::ReplicaId; use collections::HashMap; +use encodings::{Encoding, EncodingOptions}; use fs::MTime; use futures::channel::oneshot; use gpui::{ @@ -126,6 +127,7 @@ pub struct Buffer { has_unsaved_edits: Cell<(clock::Global, bool)>, change_bits: Vec>>, _subscriptions: Vec, + pub encoding: Arc, } #[derive(Copy, Clone, Debug, PartialEq, Eq)] @@ -371,6 +373,10 @@ pub trait File: Send + Sync + Any { /// Return whether Zed considers this to be a private file. fn is_private(&self) -> bool; + + fn encoding(&self) -> Option> { + unimplemented!() + } } /// The file's storage status - whether it's stored (`Present`), and if so when it was last @@ -412,7 +418,12 @@ pub trait LocalFile: File { fn abs_path(&self, cx: &App) -> PathBuf; /// Loads the file contents from disk and returns them as a UTF-8 encoded string. - fn load(&self, cx: &App) -> Task>; + fn load( + &self, + cx: &App, + options: &EncodingOptions, + buffer_encoding: Option>, + ) -> Task>; /// Loads the file's contents from disk. fn load_bytes(&self, cx: &App) -> Task>>; @@ -839,6 +850,18 @@ impl Buffer { ) } + /// Replace the text buffer. This function is in contrast to `set_text` in that it does not + /// change the buffer's editing state + pub fn replace_text_buffer(&mut self, new: TextBuffer, cx: &mut Context) { + self.text = new; + self.saved_version = self.version.clone(); + self.has_unsaved_edits.set((self.version.clone(), false)); + + self.was_changed(); + cx.emit(BufferEvent::DirtyChanged); + cx.notify(); + } + /// Create a new buffer with the given base text that has proper line endings and other normalization applied. pub fn local_normalized( base_text_normalized: Rope, @@ -1006,6 +1029,7 @@ impl Buffer { has_conflict: false, change_bits: Default::default(), _subscriptions: Vec::new(), + encoding: Arc::new(Encoding::new(encodings::UTF_8)), } } @@ -1341,12 +1365,20 @@ impl Buffer { /// Reloads the contents of the buffer from disk. pub fn reload(&mut self, cx: &Context) -> oneshot::Receiver> { let (tx, rx) = futures::channel::oneshot::channel(); + let encoding = (*self.encoding).clone(); + + let buffer_encoding = self.encoding.clone(); + let options = EncodingOptions::default(); + options.encoding.set(encoding.get()); + let prev_version = self.text.version(); self.reload_task = Some(cx.spawn(async move |this, cx| { let Some((new_mtime, new_text)) = this.update(cx, |this, cx| { let file = this.file.as_ref()?.as_local()?; - Some((file.disk_state().mtime(), file.load(cx))) + Some((file.disk_state().mtime(), { + file.load(cx, &options, Some(buffer_encoding)) + })) })? else { return Ok(()); @@ -1399,6 +1431,9 @@ impl Buffer { cx.notify(); } + pub fn replace_file(&mut self, new_file: Arc) { + self.file = Some(new_file); + } /// Updates the [`File`] backing this buffer. This should be called when /// the file has changed or has been deleted. pub fn file_updated(&mut self, new_file: Arc, cx: &mut Context) { @@ -2899,6 +2934,11 @@ impl Buffer { pub fn preserve_preview(&self) -> bool { !self.has_edits_since(&self.preview_version) } + + /// Update the buffer + pub fn update_encoding(&mut self, encoding: Encoding) { + self.encoding.set(encoding.get()); + } } #[doc(hidden)] @@ -5220,7 +5260,12 @@ impl LocalFile for TestFile { .join(self.path.as_std_path()) } - fn load(&self, _cx: &App) -> Task> { + fn load( + &self, + _cx: &App, + _options: &EncodingOptions, + _buffer_encoding: Option>, + ) -> Task> { unimplemented!() } diff --git a/crates/languages/src/json.rs b/crates/languages/src/json.rs index 45fa2dd75cce05..30d3c34f1502c8 100644 --- a/crates/languages/src/json.rs +++ b/crates/languages/src/json.rs @@ -56,7 +56,9 @@ impl ContextProvider for JsonTaskProvider { cx.spawn(async move |cx| { let contents = file .worktree - .update(cx, |this, cx| this.load_file(&file.path, cx)) + .update(cx, |this, cx| { + this.load_file(&file.path, &Default::default(), None, cx) + }) .ok()? .await .ok()?; diff --git a/crates/project/Cargo.toml b/crates/project/Cargo.toml index d9285a8c24ec51..d05cdef0b3d1c7 100644 --- a/crates/project/Cargo.toml +++ b/crates/project/Cargo.toml @@ -39,6 +39,7 @@ clock.workspace = true collections.workspace = true context_server.workspace = true dap.workspace = true +encodings.workspace = true extension.workspace = true fancy-regex.workspace = true fs.workspace = true @@ -90,6 +91,7 @@ worktree.workspace = true zeroize.workspace = true zlog.workspace = true + [dev-dependencies] client = { workspace = true, features = ["test-support"] } collections = { workspace = true, features = ["test-support"] } diff --git a/crates/project/src/buffer_store.rs b/crates/project/src/buffer_store.rs index 3fb70251869058..a27f7a0759f512 100644 --- a/crates/project/src/buffer_store.rs +++ b/crates/project/src/buffer_store.rs @@ -7,8 +7,10 @@ use crate::{ use anyhow::{Context as _, Result, anyhow}; use client::Client; use collections::{HashMap, HashSet, hash_map}; +use encodings::EncodingOptions; use fs::Fs; -use futures::{Future, FutureExt as _, StreamExt, channel::oneshot, future::Shared}; +use futures::StreamExt; +use futures::{Future, FutureExt as _, channel::oneshot, future::Shared}; use gpui::{ App, AppContext as _, AsyncApp, Context, Entity, EventEmitter, Subscription, Task, WeakEntity, }; @@ -26,7 +28,7 @@ use rpc::{ use smol::channel::Receiver; use std::{io, pin::pin, sync::Arc, time::Instant}; use text::{BufferId, ReplicaId}; -use util::{ResultExt as _, TryFutureExt, debug_panic, maybe, paths::PathStyle, rel_path::RelPath}; +use util::{ResultExt as _, TryFutureExt, debug_panic, maybe, rel_path::RelPath}; use worktree::{File, PathChange, ProjectEntryId, Worktree, WorktreeId}; /// A set of open buffers. @@ -387,6 +389,8 @@ impl LocalBufferStore { let version = buffer.version(); let buffer_id = buffer.remote_id(); let file = buffer.file().cloned(); + let encoding = buffer.encoding.clone(); + if file .as_ref() .is_some_and(|file| file.disk_state() == DiskState::New) @@ -395,7 +399,7 @@ impl LocalBufferStore { } let save = worktree.update(cx, |worktree, cx| { - worktree.write_file(path, text, line_ending, cx) + worktree.write_file(path.clone(), text, line_ending, (*encoding).clone(), cx) }); cx.spawn(async move |this, cx| { @@ -524,6 +528,7 @@ impl LocalBufferStore { path: entry.path.clone(), worktree: worktree.clone(), is_private: entry.is_private, + encoding: None, } } else { File { @@ -533,6 +538,7 @@ impl LocalBufferStore { path: old_file.path.clone(), worktree: worktree.clone(), is_private: old_file.is_private, + encoding: None, } }; @@ -623,27 +629,42 @@ impl LocalBufferStore { &self, path: Arc, worktree: Entity, + options: &EncodingOptions, cx: &mut Context, ) -> Task>> { - let load_file = worktree.update(cx, |worktree, cx| worktree.load_file(path.as_ref(), cx)); + let options = options.clone(); + let encoding = options.encoding.clone(); + + let load_buffer = worktree.update(cx, |worktree, cx| { + let reservation = cx.reserve_entity(); + let buffer_id = BufferId::from(reservation.entity_id().as_non_zero_u64()); + + let load_file_task = worktree.load_file(path.as_ref(), &options, None, cx); + + cx.spawn(async move |_, cx| { + let loaded_file = load_file_task.await?; + let background_executor = cx.background_executor().clone(); + + let buffer = cx.insert_entity(reservation, |_| { + Buffer::build( + text::Buffer::new( + ReplicaId::LOCAL, + buffer_id, + loaded_file.text, + &background_executor, + ), + Some(loaded_file.file), + Capability::ReadWrite, + ) + })?; + + Ok(buffer) + }) + }); + cx.spawn(async move |this, cx| { - let path = path.clone(); - let buffer = match load_file.await.with_context(|| { - format!("Could not open path: {}", path.display(PathStyle::local())) - }) { - Ok(loaded) => { - let reservation = cx.reserve_entity::()?; - let buffer_id = BufferId::from(reservation.entity_id().as_non_zero_u64()); - let executor = cx.background_executor().clone(); - let text_buffer = cx - .background_spawn(async move { - text::Buffer::new(ReplicaId::LOCAL, buffer_id, loaded.text, &executor) - }) - .await; - cx.insert_entity(reservation, |_| { - Buffer::build(text_buffer, Some(loaded.file), Capability::ReadWrite) - })? - } + let buffer = match load_buffer.await { + Ok(buffer) => buffer, Err(error) if is_not_found_error(&error) => cx.new(|cx| { let buffer_id = BufferId::from(cx.entity_id().as_non_zero_u64()); let text_buffer = text::Buffer::new( @@ -661,6 +682,7 @@ impl LocalBufferStore { entry_id: None, is_local: true, is_private: false, + encoding: Some(encoding.clone()), })), Capability::ReadWrite, ) @@ -688,6 +710,10 @@ impl LocalBufferStore { anyhow::Ok(()) })??; + buffer.update(cx, |buffer, _| { + buffer.update_encoding(encoding.get().into()) + })?; + Ok(buffer) }) } @@ -818,6 +844,7 @@ impl BufferStore { pub fn open_buffer( &mut self, project_path: ProjectPath, + options: &EncodingOptions, cx: &mut Context, ) -> Task>> { if let Some(buffer) = self.get_by_path(&project_path) { @@ -841,7 +868,7 @@ impl BufferStore { return Task::ready(Err(anyhow!("no such worktree"))); }; let load_buffer = match &self.state { - BufferStoreState::Local(this) => this.open_buffer(path, worktree, cx), + BufferStoreState::Local(this) => this.open_buffer(path, worktree, options, cx), BufferStoreState::Remote(this) => this.open_buffer(path, worktree, cx), }; @@ -1154,7 +1181,7 @@ impl BufferStore { let buffers = this.update(cx, |this, cx| { project_paths .into_iter() - .map(|project_path| this.open_buffer(project_path, cx)) + .map(|project_path| this.open_buffer(project_path, &Default::default(), cx)) .collect::>() })?; for buffer_task in buffers { diff --git a/crates/project/src/debugger/breakpoint_store.rs b/crates/project/src/debugger/breakpoint_store.rs index 42663ab9852a5d..fe29bebceae51c 100644 --- a/crates/project/src/debugger/breakpoint_store.rs +++ b/crates/project/src/debugger/breakpoint_store.rs @@ -796,7 +796,7 @@ impl BreakpointStore { worktree_id: worktree.read(cx).id(), path: relative_path, }; - this.open_buffer(path, cx) + this.open_buffer(path, &Default::default(), cx) })? .await; let Ok(buffer) = buffer else { diff --git a/crates/project/src/image_store.rs b/crates/project/src/image_store.rs index 8fcf9c8a6172f8..2f24927e043064 100644 --- a/crates/project/src/image_store.rs +++ b/crates/project/src/image_store.rs @@ -605,6 +605,7 @@ impl LocalImageStore { path: entry.path.clone(), worktree: worktree.clone(), is_private: entry.is_private, + encoding: None, } } else { worktree::File { @@ -614,6 +615,7 @@ impl LocalImageStore { path: old_file.path.clone(), worktree: worktree.clone(), is_private: old_file.is_private, + encoding: None, } }; diff --git a/crates/project/src/invalid_item_view.rs b/crates/project/src/invalid_item_view.rs index fdcdd16a69ce73..c5dfd6436f8a0a 100644 --- a/crates/project/src/invalid_item_view.rs +++ b/crates/project/src/invalid_item_view.rs @@ -4,7 +4,7 @@ use gpui::{EventEmitter, FocusHandle, Focusable}; use ui::{ App, Button, ButtonCommon, ButtonStyle, Clickable, Context, FluentBuilder, InteractiveElement, KeyBinding, Label, LabelCommon, LabelSize, ParentElement, Render, SharedString, Styled as _, - Window, h_flex, v_flex, + TintColor, Window, div, h_flex, v_flex, }; use zed_actions::workspace::OpenWithSystem; @@ -78,6 +78,7 @@ impl Focusable for InvalidItemView { impl Render for InvalidItemView { fn render(&mut self, window: &mut Window, cx: &mut Context) -> impl gpui::IntoElement { let abs_path = self.abs_path.clone(); + v_flex() .size_full() .track_focus(&self.focus_handle(cx)) @@ -86,15 +87,18 @@ impl Render for InvalidItemView { .overflow_hidden() .key_context("InvalidBuffer") .child( - h_flex().size_full().justify_center().child( + h_flex().size_full().justify_center().items_center().child( v_flex() - .justify_center() .gap_2() + .max_w_96() .child(h_flex().justify_center().child("Could not open file")) .child( - h_flex() - .justify_center() - .child(Label::new(self.error.clone()).size(LabelSize::Small)), + h_flex().justify_center().child( + div() + .whitespace_normal() + .text_center() + .child(Label::new(self.error.clone()).size(LabelSize::Small)), + ), ) .when(self.is_local, |contents| { contents.child( diff --git a/crates/project/src/lsp_store.rs b/crates/project/src/lsp_store.rs index 762070796f068f..f803d0bc82532f 100644 --- a/crates/project/src/lsp_store.rs +++ b/crates/project/src/lsp_store.rs @@ -8336,7 +8336,7 @@ impl LspStore { lsp_store .update(cx, |lsp_store, cx| { lsp_store.buffer_store().update(cx, |buffer_store, cx| { - buffer_store.open_buffer(project_path, cx) + buffer_store.open_buffer(project_path, &Default::default(), cx) }) })? .await diff --git a/crates/project/src/lsp_store/rust_analyzer_ext.rs b/crates/project/src/lsp_store/rust_analyzer_ext.rs index 4d5f134e5f1682..ef74b9e26f072a 100644 --- a/crates/project/src/lsp_store/rust_analyzer_ext.rs +++ b/crates/project/src/lsp_store/rust_analyzer_ext.rs @@ -91,7 +91,7 @@ pub fn cancel_flycheck( let buffer = buffer_path.map(|buffer_path| { project.update(cx, |project, cx| { project.buffer_store().update(cx, |buffer_store, cx| { - buffer_store.open_buffer(buffer_path, cx) + buffer_store.open_buffer(buffer_path, &Default::default(), cx) }) }) }); @@ -140,7 +140,7 @@ pub fn run_flycheck( let buffer = buffer_path.map(|buffer_path| { project.update(cx, |project, cx| { project.buffer_store().update(cx, |buffer_store, cx| { - buffer_store.open_buffer(buffer_path, cx) + buffer_store.open_buffer(buffer_path, &Default::default(), cx) }) }) }); @@ -198,7 +198,7 @@ pub fn clear_flycheck( let buffer = buffer_path.map(|buffer_path| { project.update(cx, |project, cx| { project.buffer_store().update(cx, |buffer_store, cx| { - buffer_store.open_buffer(buffer_path, cx) + buffer_store.open_buffer(buffer_path, &Default::default(), cx) }) }) }); diff --git a/crates/project/src/prettier_store.rs b/crates/project/src/prettier_store.rs index 3743f9769eaaff..9539146ebb4029 100644 --- a/crates/project/src/prettier_store.rs +++ b/crates/project/src/prettier_store.rs @@ -7,6 +7,7 @@ use std::{ use anyhow::{Context as _, Result, anyhow}; use collections::{HashMap, HashSet}; +use encodings::Encoding; use fs::Fs; use futures::{ FutureExt, @@ -981,10 +982,12 @@ async fn save_prettier_server_file( executor: &BackgroundExecutor, ) -> anyhow::Result<()> { let prettier_wrapper_path = default_prettier_dir().join(prettier::PRETTIER_SERVER_FILE); + let encoding = Encoding::default(); fs.save( &prettier_wrapper_path, &text::Rope::from_str(prettier::PRETTIER_SERVER_JS, executor), text::LineEnding::Unix, + encoding, ) .await .with_context(|| { diff --git a/crates/project/src/project.rs b/crates/project/src/project.rs index 7c7fe9a4309161..69d201be295855 100644 --- a/crates/project/src/project.rs +++ b/crates/project/src/project.rs @@ -26,9 +26,12 @@ mod project_tests; mod environment; use buffer_diff::BufferDiff; use context_server_store::ContextServerStore; + +use encodings::{Encoding, EncodingOptions}; pub use environment::ProjectEnvironmentEvent; use git::repository::get_git_committer; use git_store::{Repository, RepositoryId}; + pub mod search_history; mod yarn; @@ -215,6 +218,7 @@ pub struct Project { settings_observer: Entity, toolchain_store: Option>, agent_location: Option, + pub encoding_options: EncodingOptions, } #[derive(Clone, Debug, PartialEq, Eq)] @@ -1225,6 +1229,7 @@ impl Project { toolchain_store: Some(toolchain_store), agent_location: None, + encoding_options: EncodingOptions::default(), } }) } @@ -1410,6 +1415,7 @@ impl Project { toolchain_store: Some(toolchain_store), agent_location: None, + encoding_options: EncodingOptions::default(), }; // remote server -> local machine handlers @@ -1663,7 +1669,9 @@ impl Project { remotely_created_models: Arc::new(Mutex::new(RemotelyCreatedModels::default())), toolchain_store: None, agent_location: None, + encoding_options: EncodingOptions::default(), }; + project.set_role(role, cx); for worktree in worktrees { project.add_worktree(&worktree, cx); @@ -2712,7 +2720,7 @@ impl Project { } self.buffer_store.update(cx, |buffer_store, cx| { - buffer_store.open_buffer(path.into(), cx) + buffer_store.open_buffer(path.into(), &self.encoding_options, cx) }) } @@ -5394,7 +5402,9 @@ impl Project { }; cx.spawn(async move |cx| { let file = worktree - .update(cx, |worktree, cx| worktree.load_file(&rel_path, cx))? + .update(cx, |worktree, cx| { + worktree.load_file(&rel_path, &Default::default(), None, cx) + })? .await .context("Failed to load settings file")?; @@ -5408,6 +5418,7 @@ impl Project { rel_path.clone(), Rope::from_str(&new_text, cx.background_executor()), line_ending, + Encoding::default(), cx, ) })? diff --git a/crates/project/src/project_search.rs b/crates/project/src/project_search.rs new file mode 100644 index 00000000000000..b68632d9ca1e2f --- /dev/null +++ b/crates/project/src/project_search.rs @@ -0,0 +1,754 @@ +use std::{ + io::{BufRead, BufReader}, + path::Path, + pin::pin, + sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }, +}; + +use anyhow::Context; +use collections::HashSet; +use fs::Fs; +use futures::{SinkExt, StreamExt, select_biased, stream::FuturesOrdered}; +use gpui::{App, AppContext, AsyncApp, Entity, Task}; +use language::{Buffer, BufferSnapshot}; +use parking_lot::Mutex; +use postage::oneshot; +use rpc::{AnyProtoClient, proto}; +use smol::{ + channel::{Receiver, Sender, bounded, unbounded}, + future::FutureExt, +}; + +use text::BufferId; +use util::{ResultExt, maybe, paths::compare_rel_paths}; +use worktree::{Entry, ProjectEntryId, Snapshot, Worktree}; + +use crate::{ + Project, ProjectItem, ProjectPath, RemotelyCreatedModels, + buffer_store::BufferStore, + search::{SearchQuery, SearchResult}, + worktree_store::WorktreeStore, +}; + +pub struct Search { + buffer_store: Entity, + worktree_store: Entity, + limit: usize, + kind: SearchKind, +} + +/// Represents search setup, before it is actually kicked off with Search::into_results +enum SearchKind { + /// Search for candidates by inspecting file contents on file system, avoiding loading the buffer unless we know that a given file contains a match. + Local { + fs: Arc, + worktrees: Vec>, + }, + /// Query remote host for candidates. As of writing, the host runs a local search in "buffers with matches only" mode. + Remote { + client: AnyProtoClient, + remote_id: u64, + models: Arc>, + }, + /// Run search against a known set of candidates. Even when working with a remote host, this won't round-trip to host. + OpenBuffersOnly, +} + +/// Represents results of project search and allows one to either obtain match positions OR +/// just the handles to buffers that may match the search. Grabbing the handles is cheaper than obtaining full match positions, because in that case we'll look for +/// at most one match in each file. +#[must_use] +pub struct SearchResultsHandle { + results: Receiver, + matching_buffers: Receiver>, + trigger_search: Box Task<()> + Send + Sync>, +} + +impl SearchResultsHandle { + pub fn results(self, cx: &mut App) -> Receiver { + (self.trigger_search)(cx).detach(); + self.results + } + pub fn matching_buffers(self, cx: &mut App) -> Receiver> { + (self.trigger_search)(cx).detach(); + self.matching_buffers + } +} + +#[derive(Clone)] +enum FindSearchCandidates { + Local { + fs: Arc, + /// Start off with all paths in project and filter them based on: + /// - Include filters + /// - Exclude filters + /// - Only open buffers + /// - Scan ignored files + /// Put another way: filter out files that can't match (without looking at file contents) + input_paths_rx: Receiver, + /// After that, if the buffer is not yet loaded, we'll figure out if it contains at least one match + /// based on disk contents of a buffer. This step is not performed for buffers we already have in memory. + confirm_contents_will_match_tx: Sender, + confirm_contents_will_match_rx: Receiver, + /// Of those that contain at least one match (or are already in memory), look for rest of matches (and figure out their ranges). + /// But wait - first, we need to go back to the main thread to open a buffer (& create an entity for it). + get_buffer_for_full_scan_tx: Sender, + }, + Remote, + OpenBuffersOnly, +} + +impl Search { + pub fn local( + fs: Arc, + buffer_store: Entity, + worktree_store: Entity, + limit: usize, + cx: &mut App, + ) -> Self { + let worktrees = worktree_store.read(cx).visible_worktrees(cx).collect(); + Self { + kind: SearchKind::Local { fs, worktrees }, + buffer_store, + worktree_store, + limit, + } + } + + pub(crate) fn remote( + buffer_store: Entity, + worktree_store: Entity, + limit: usize, + client_state: (AnyProtoClient, u64, Arc>), + ) -> Self { + Self { + kind: SearchKind::Remote { + client: client_state.0, + remote_id: client_state.1, + models: client_state.2, + }, + buffer_store, + worktree_store, + limit, + } + } + pub(crate) fn open_buffers_only( + buffer_store: Entity, + worktree_store: Entity, + limit: usize, + ) -> Self { + Self { + kind: SearchKind::OpenBuffersOnly, + buffer_store, + worktree_store, + limit, + } + } + + pub(crate) const MAX_SEARCH_RESULT_FILES: usize = 5_000; + pub(crate) const MAX_SEARCH_RESULT_RANGES: usize = 10_000; + /// Prepares a project search run. The resulting [`SearchResultsHandle`] has to be used to specify whether you're interested in matching buffers + /// or full search results. + pub fn into_handle(mut self, query: SearchQuery, cx: &mut App) -> SearchResultsHandle { + let mut open_buffers = HashSet::default(); + let mut unnamed_buffers = Vec::new(); + const MAX_CONCURRENT_BUFFER_OPENS: usize = 64; + let buffers = self.buffer_store.read(cx); + for handle in buffers.buffers() { + let buffer = handle.read(cx); + if !buffers.is_searchable(&buffer.remote_id()) { + continue; + } else if let Some(entry_id) = buffer.entry_id(cx) { + open_buffers.insert(entry_id); + } else { + self.limit -= self.limit.saturating_sub(1); + unnamed_buffers.push(handle) + }; + } + let executor = cx.background_executor().clone(); + let (tx, rx) = unbounded(); + let (grab_buffer_snapshot_tx, grab_buffer_snapshot_rx) = unbounded(); + let matching_buffers = grab_buffer_snapshot_rx.clone(); + let trigger_search = Box::new(move |cx: &mut App| { + cx.spawn(async move |cx| { + for buffer in unnamed_buffers { + _ = grab_buffer_snapshot_tx.send(buffer).await; + } + + let (find_all_matches_tx, find_all_matches_rx) = + bounded(MAX_CONCURRENT_BUFFER_OPENS); + + let (candidate_searcher, tasks) = match self.kind { + SearchKind::OpenBuffersOnly => { + let Ok(open_buffers) = cx.update(|cx| self.all_loaded_buffers(&query, cx)) + else { + return; + }; + let fill_requests = cx + .background_spawn(async move { + for buffer in open_buffers { + if let Err(_) = grab_buffer_snapshot_tx.send(buffer).await { + return; + } + } + }) + .boxed_local(); + (FindSearchCandidates::OpenBuffersOnly, vec![fill_requests]) + } + SearchKind::Local { + fs, + ref mut worktrees, + } => { + let (get_buffer_for_full_scan_tx, get_buffer_for_full_scan_rx) = + unbounded(); + let (confirm_contents_will_match_tx, confirm_contents_will_match_rx) = + bounded(64); + let (sorted_search_results_tx, sorted_search_results_rx) = unbounded(); + + let (input_paths_tx, input_paths_rx) = unbounded(); + + let tasks = vec![ + cx.spawn(Self::provide_search_paths( + std::mem::take(worktrees), + query.include_ignored(), + input_paths_tx, + sorted_search_results_tx, + )) + .boxed_local(), + Self::open_buffers( + &self.buffer_store, + get_buffer_for_full_scan_rx, + grab_buffer_snapshot_tx, + cx.clone(), + ) + .boxed_local(), + cx.background_spawn(Self::maintain_sorted_search_results( + sorted_search_results_rx, + get_buffer_for_full_scan_tx.clone(), + self.limit, + )) + .boxed_local(), + ]; + ( + FindSearchCandidates::Local { + fs, + get_buffer_for_full_scan_tx, + confirm_contents_will_match_tx, + confirm_contents_will_match_rx, + input_paths_rx, + }, + tasks, + ) + } + SearchKind::Remote { + client, + remote_id, + models, + } => { + let request = client.request(proto::FindSearchCandidates { + project_id: remote_id, + query: Some(query.to_proto()), + limit: self.limit as _, + }); + let Ok(guard) = cx.update(|cx| { + Project::retain_remotely_created_models_impl( + &models, + &self.buffer_store, + &self.worktree_store, + cx, + ) + }) else { + return; + }; + let buffer_store = self.buffer_store.downgrade(); + let issue_remote_buffers_request = cx + .spawn(async move |cx| { + let _ = maybe!(async move { + let response = request.await?; + + for buffer_id in response.buffer_ids { + let buffer_id = BufferId::new(buffer_id)?; + let buffer = buffer_store + .update(cx, |buffer_store, cx| { + buffer_store.wait_for_remote_buffer(buffer_id, cx) + })? + .await?; + let _ = grab_buffer_snapshot_tx.send(buffer).await; + } + + drop(guard); + anyhow::Ok(()) + }) + .await + .log_err(); + }) + .boxed_local(); + ( + FindSearchCandidates::Remote, + vec![issue_remote_buffers_request], + ) + } + }; + + let matches_count = AtomicUsize::new(0); + let matched_buffer_count = AtomicUsize::new(0); + + let worker_pool = executor.scoped(|scope| { + let num_cpus = executor.num_cpus(); + + assert!(num_cpus > 0); + for _ in 0..executor.num_cpus() - 1 { + let worker = Worker { + query: &query, + open_buffers: &open_buffers, + matched_buffer_count: &matched_buffer_count, + matches_count: &matches_count, + candidates: candidate_searcher.clone(), + find_all_matches_rx: find_all_matches_rx.clone(), + publish_matches: tx.clone(), + }; + scope.spawn(worker.run()); + } + drop(tx); + drop(find_all_matches_rx); + drop(candidate_searcher); + }); + + let buffer_snapshots = Self::grab_buffer_snapshots( + grab_buffer_snapshot_rx, + find_all_matches_tx, + cx.clone(), + ); + futures::future::join_all( + [worker_pool.boxed_local(), buffer_snapshots.boxed_local()] + .into_iter() + .chain(tasks), + ) + .await; + }) + }); + + SearchResultsHandle { + results: rx, + matching_buffers, + trigger_search, + } + } + + fn provide_search_paths( + worktrees: Vec>, + include_ignored: bool, + tx: Sender, + results: Sender>, + ) -> impl AsyncFnOnce(&mut AsyncApp) { + async move |cx| { + _ = maybe!(async move { + for worktree in worktrees { + let (mut snapshot, worktree_settings) = worktree + .read_with(cx, |this, _| { + Some((this.snapshot(), this.as_local()?.settings())) + })? + .context("The worktree is not local")?; + if include_ignored { + // Pre-fetch all of the ignored directories as they're going to be searched. + let mut entries_to_refresh = vec![]; + for entry in snapshot.entries(include_ignored, 0) { + if entry.is_ignored && entry.kind.is_unloaded() { + if !worktree_settings.is_path_excluded(&entry.path) { + entries_to_refresh.push(entry.path.clone()); + } + } + } + let barrier = worktree.update(cx, |this, _| { + let local = this.as_local_mut()?; + let barrier = entries_to_refresh + .into_iter() + .map(|path| local.add_path_prefix_to_scan(path).into_future()) + .collect::>(); + Some(barrier) + })?; + if let Some(barriers) = barrier { + futures::future::join_all(barriers).await; + } + snapshot = worktree.read_with(cx, |this, _| this.snapshot())?; + } + cx.background_executor() + .scoped(|scope| { + scope.spawn(async { + for entry in snapshot.files(include_ignored, 0) { + let (should_scan_tx, should_scan_rx) = oneshot::channel(); + let Ok(_) = tx + .send(InputPath { + entry: entry.clone(), + snapshot: snapshot.clone(), + should_scan_tx, + }) + .await + else { + return; + }; + if results.send(should_scan_rx).await.is_err() { + return; + }; + } + }) + }) + .await; + } + anyhow::Ok(()) + }) + .await; + } + } + + async fn maintain_sorted_search_results( + rx: Receiver>, + paths_for_full_scan: Sender, + limit: usize, + ) { + let mut rx = pin!(rx); + let mut matched = 0; + while let Some(mut next_path_result) = rx.next().await { + let Some(successful_path) = next_path_result.next().await else { + // This math did not produce a match, hence skip it. + continue; + }; + if paths_for_full_scan.send(successful_path).await.is_err() { + return; + }; + matched += 1; + if matched >= limit { + break; + } + } + } + + /// Background workers cannot open buffers by themselves, hence main thread will do it on their behalf. + async fn open_buffers( + buffer_store: &Entity, + rx: Receiver, + find_all_matches_tx: Sender>, + mut cx: AsyncApp, + ) { + let mut rx = pin!(rx.ready_chunks(64)); + _ = maybe!(async move { + while let Some(requested_paths) = rx.next().await { + let mut buffers = buffer_store.update(&mut cx, |this, cx| { + requested_paths + .into_iter() + .map(|path| this.open_buffer(path, None, false, true, cx)) + .collect::>() + })?; + + while let Some(buffer) = buffers.next().await { + if let Some(buffer) = buffer.log_err() { + find_all_matches_tx.send(buffer).await?; + } + } + } + Result::<_, anyhow::Error>::Ok(()) + }) + .await; + } + + async fn grab_buffer_snapshots( + rx: Receiver>, + find_all_matches_tx: Sender<(Entity, BufferSnapshot)>, + mut cx: AsyncApp, + ) { + _ = maybe!(async move { + while let Ok(buffer) = rx.recv().await { + let snapshot = buffer.read_with(&mut cx, |this, _| this.snapshot())?; + find_all_matches_tx.send((buffer, snapshot)).await?; + } + Result::<_, anyhow::Error>::Ok(()) + }) + .await; + } + + fn all_loaded_buffers(&self, search_query: &SearchQuery, cx: &App) -> Vec> { + let worktree_store = self.worktree_store.read(cx); + let mut buffers = search_query + .buffers() + .into_iter() + .flatten() + .filter(|buffer| { + let b = buffer.read(cx); + if let Some(file) = b.file() { + if !search_query.match_path(file.path().as_std_path()) { + return false; + } + if !search_query.include_ignored() + && let Some(entry) = b + .entry_id(cx) + .and_then(|entry_id| worktree_store.entry_for_id(entry_id, cx)) + && entry.is_ignored + { + return false; + } + } + true + }) + .cloned() + .collect::>(); + buffers.sort_by(|a, b| { + let a = a.read(cx); + let b = b.read(cx); + match (a.file(), b.file()) { + (None, None) => a.remote_id().cmp(&b.remote_id()), + (None, Some(_)) => std::cmp::Ordering::Less, + (Some(_), None) => std::cmp::Ordering::Greater, + (Some(a), Some(b)) => compare_rel_paths((a.path(), true), (b.path(), true)), + } + }); + + buffers + } +} + +struct Worker<'search> { + query: &'search SearchQuery, + matched_buffer_count: &'search AtomicUsize, + matches_count: &'search AtomicUsize, + open_buffers: &'search HashSet, + candidates: FindSearchCandidates, + /// Ok, we're back in background: run full scan & find all matches in a given buffer snapshot. + find_all_matches_rx: Receiver<(Entity, BufferSnapshot)>, + /// Cool, we have results; let's share them with the world. + publish_matches: Sender, +} + +impl Worker<'_> { + async fn run(mut self) { + let ( + input_paths_rx, + confirm_contents_will_match_rx, + mut confirm_contents_will_match_tx, + mut get_buffer_for_full_scan_tx, + fs, + ) = match self.candidates { + FindSearchCandidates::Local { + fs, + input_paths_rx, + confirm_contents_will_match_rx, + confirm_contents_will_match_tx, + get_buffer_for_full_scan_tx, + } => ( + input_paths_rx, + confirm_contents_will_match_rx, + confirm_contents_will_match_tx, + get_buffer_for_full_scan_tx, + Some(fs), + ), + FindSearchCandidates::Remote | FindSearchCandidates::OpenBuffersOnly => ( + unbounded().1, + unbounded().1, + unbounded().0, + unbounded().0, + None, + ), + }; + let mut find_all_matches = pin!(self.find_all_matches_rx.fuse()); + let mut find_first_match = pin!(confirm_contents_will_match_rx.fuse()); + let mut scan_path = pin!(input_paths_rx.fuse()); + + loop { + let handler = RequestHandler { + query: self.query, + open_entries: &self.open_buffers, + fs: fs.as_deref(), + matched_buffer_count: self.matched_buffer_count, + matches_count: self.matches_count, + confirm_contents_will_match_tx: &confirm_contents_will_match_tx, + get_buffer_for_full_scan_tx: &get_buffer_for_full_scan_tx, + publish_matches: &self.publish_matches, + }; + // Whenever we notice that some step of a pipeline is closed, we don't want to close subsequent + // steps straight away. Another worker might be about to produce a value that will + // be pushed there, thus we'll replace current worker's pipe with a dummy one. + // That way, we'll only ever close a next-stage channel when ALL workers do so. + select_biased! { + find_all_matches = find_all_matches.next() => { + + if self.publish_matches.is_closed() { + break; + } + let Some(matches) = find_all_matches else { + self.publish_matches = bounded(1).0; + continue; + }; + let result = handler.handle_find_all_matches(matches).await; + if let Some(_should_bail) = result { + + self.publish_matches = bounded(1).0; + continue; + } + }, + find_first_match = find_first_match.next() => { + if let Some(buffer_with_at_least_one_match) = find_first_match { + handler.handle_find_first_match(buffer_with_at_least_one_match).await; + } else { + get_buffer_for_full_scan_tx = bounded(1).0; + } + + }, + scan_path = scan_path.next() => { + if let Some(path_to_scan) = scan_path { + handler.handle_scan_path(path_to_scan).await; + } else { + // If we're the last worker to notice that this is not producing values, close the upstream. + confirm_contents_will_match_tx = bounded(1).0; + } + + } + complete => { + break + }, + + } + } + } +} + +struct RequestHandler<'worker> { + query: &'worker SearchQuery, + fs: Option<&'worker dyn Fs>, + open_entries: &'worker HashSet, + matched_buffer_count: &'worker AtomicUsize, + matches_count: &'worker AtomicUsize, + + confirm_contents_will_match_tx: &'worker Sender, + get_buffer_for_full_scan_tx: &'worker Sender, + publish_matches: &'worker Sender, +} + +struct LimitReached; + +impl RequestHandler<'_> { + async fn handle_find_all_matches( + &self, + (buffer, snapshot): (Entity, BufferSnapshot), + ) -> Option { + let ranges = self + .query + .search(&snapshot, None) + .await + .iter() + .map(|range| snapshot.anchor_before(range.start)..snapshot.anchor_after(range.end)) + .collect::>(); + + let matched_ranges = ranges.len(); + if self.matched_buffer_count.fetch_add(1, Ordering::Release) + > Search::MAX_SEARCH_RESULT_FILES + || self + .matches_count + .fetch_add(matched_ranges, Ordering::Release) + > Search::MAX_SEARCH_RESULT_RANGES + { + _ = self.publish_matches.send(SearchResult::LimitReached).await; + Some(LimitReached) + } else { + _ = self + .publish_matches + .send(SearchResult::Buffer { buffer, ranges }) + .await; + None + } + } + async fn handle_find_first_match(&self, mut entry: MatchingEntry) { + _=maybe!(async move { + let abs_path = entry.worktree_root.join(entry.path.path.as_std_path()); + let Some(file) = self.fs.context("Trying to query filesystem in remote project search")?.open_sync(&abs_path).await.log_err() else { + return anyhow::Ok(()); + }; + + let mut file = BufReader::new(file); + let file_start = file.fill_buf()?; + + if let Err(Some(starting_position)) = + std::str::from_utf8(file_start).map_err(|e| e.error_len()) + { + // Before attempting to match the file content, throw away files that have invalid UTF-8 sequences early on; + // That way we can still match files in a streaming fashion without having look at "obviously binary" files. + log::debug!( + "Invalid UTF-8 sequence in file {abs_path:?} at byte position {starting_position}" + ); + return Ok(()); + } + + if self.query.detect(file).unwrap_or(false) { + // Yes, we should scan the whole file. + entry.should_scan_tx.send(entry.path).await?; + } + Ok(()) + }).await; + } + + async fn handle_scan_path(&self, req: InputPath) { + _ = maybe!(async move { + let InputPath { + entry, + + snapshot, + should_scan_tx, + } = req; + + if entry.is_fifo || !entry.is_file() { + return Ok(()); + } + + if self.query.filters_path() { + let matched_path = if self.query.match_full_paths() { + let mut full_path = snapshot.root_name().as_std_path().to_owned(); + full_path.push(entry.path.as_std_path()); + self.query.match_path(&full_path) + } else { + self.query.match_path(entry.path.as_std_path()) + }; + if !matched_path { + return Ok(()); + } + } + + if self.open_entries.contains(&entry.id) { + // The buffer is already in memory and that's the version we want to scan; + // hence skip the dilly-dally and look for all matches straight away. + self.get_buffer_for_full_scan_tx + .send(ProjectPath { + worktree_id: snapshot.id(), + path: entry.path.clone(), + }) + .await?; + } else { + self.confirm_contents_will_match_tx + .send(MatchingEntry { + should_scan_tx: should_scan_tx, + worktree_root: snapshot.abs_path().clone(), + path: ProjectPath { + worktree_id: snapshot.id(), + path: entry.path.clone(), + }, + }) + .await?; + } + + anyhow::Ok(()) + }) + .await; + } +} + +struct InputPath { + entry: Entry, + snapshot: Snapshot, + should_scan_tx: oneshot::Sender, +} + +struct MatchingEntry { + worktree_root: Arc, + path: ProjectPath, + should_scan_tx: oneshot::Sender, +} diff --git a/crates/project/src/project_tests.rs b/crates/project/src/project_tests.rs index 3dc918d5a757af..cb5b17fcd0efb4 100644 --- a/crates/project/src/project_tests.rs +++ b/crates/project/src/project_tests.rs @@ -12,6 +12,7 @@ use buffer_diff::{ BufferDiffEvent, CALCULATE_DIFF_TASK, DiffHunkSecondaryStatus, DiffHunkStatus, DiffHunkStatusKind, assert_hunks, }; +use encodings::{Encoding, UTF_8}; use fs::FakeFs; use futures::{StreamExt, future}; use git::{ @@ -1459,10 +1460,14 @@ async fn test_reporting_fs_changes_to_language_servers(cx: &mut gpui::TestAppCon ) .await .unwrap(); + + let encoding = Encoding::default(); + fs.save( path!("/the-root/Cargo.lock").as_ref(), &Rope::default(), Default::default(), + encoding.clone(), ) .await .unwrap(); @@ -1470,6 +1475,7 @@ async fn test_reporting_fs_changes_to_language_servers(cx: &mut gpui::TestAppCon path!("/the-stdlib/LICENSE").as_ref(), &Rope::default(), Default::default(), + encoding.clone(), ) .await .unwrap(); @@ -1477,6 +1483,7 @@ async fn test_reporting_fs_changes_to_language_servers(cx: &mut gpui::TestAppCon path!("/the/stdlib/src/string.rs").as_ref(), &Rope::default(), Default::default(), + encoding, ) .await .unwrap(); @@ -3871,7 +3878,12 @@ async fn test_rename_file_to_new_directory(cx: &mut gpui::TestAppContext) { assert_eq!( worktree .update(cx, |worktree, cx| { - worktree.load_file(rel_path("dir1/dir2/dir3/test.txt"), cx) + worktree.load_file( + rel_path("dir1/dir2/dir3/test.txt"), + &Default::default(), + None, + cx, + ) }) .await .unwrap() @@ -3918,7 +3930,12 @@ async fn test_rename_file_to_new_directory(cx: &mut gpui::TestAppContext) { assert_eq!( worktree .update(cx, |worktree, cx| { - worktree.load_file(rel_path("dir1/dir2/test.txt"), cx) + worktree.load_file( + rel_path("dir1/dir2/test.txt"), + &Default::default(), + None, + cx, + ) }) .await .unwrap() @@ -4068,12 +4085,15 @@ async fn test_file_changes_multiple_times_on_disk(cx: &mut gpui::TestAppContext) // the next file change occurs. cx.executor().deprioritize(*language::BUFFER_DIFF_TASK); + let encoding = Encoding::default(); + // Change the buffer's file on disk, and then wait for the file change // to be detected by the worktree, so that the buffer starts reloading. fs.save( path!("/dir/file1").as_ref(), &Rope::from_str("the first contents", cx.background_executor()), Default::default(), + encoding.clone(), ) .await .unwrap(); @@ -4085,6 +4105,7 @@ async fn test_file_changes_multiple_times_on_disk(cx: &mut gpui::TestAppContext) path!("/dir/file1").as_ref(), &Rope::from_str("the second contents", cx.background_executor()), Default::default(), + encoding, ) .await .unwrap(); @@ -4123,12 +4144,15 @@ async fn test_edit_buffer_while_it_reloads(cx: &mut gpui::TestAppContext) { // the next file change occurs. cx.executor().deprioritize(*language::BUFFER_DIFF_TASK); + let encoding = Encoding::new(UTF_8); + // Change the buffer's file on disk, and then wait for the file change // to be detected by the worktree, so that the buffer starts reloading. fs.save( path!("/dir/file1").as_ref(), &Rope::from_str("the first contents", cx.background_executor()), Default::default(), + encoding, ) .await .unwrap(); @@ -4803,10 +4827,14 @@ async fn test_buffer_file_changes_on_disk(cx: &mut gpui::TestAppContext) { let (new_contents, new_offsets) = marked_text_offsets("oneˇ\nthree ˇFOURˇ five\nsixtyˇ seven\n"); + + let encoding = Encoding::new(UTF_8); + fs.save( path!("/dir/the-file").as_ref(), &Rope::from_str(new_contents.as_str(), cx.background_executor()), LineEnding::Unix, + encoding, ) .await .unwrap(); @@ -4834,11 +4862,14 @@ async fn test_buffer_file_changes_on_disk(cx: &mut gpui::TestAppContext) { assert!(!buffer.has_conflict()); }); + let encoding = Encoding::new(UTF_8); + // Change the file on disk again, adding blank lines to the beginning. fs.save( path!("/dir/the-file").as_ref(), &Rope::from_str("\n\n\nAAAA\naaa\nBB\nbbbbb\n", cx.background_executor()), LineEnding::Unix, + encoding, ) .await .unwrap(); @@ -4885,12 +4916,15 @@ async fn test_buffer_line_endings(cx: &mut gpui::TestAppContext) { assert_eq!(buffer.line_ending(), LineEnding::Windows); }); + let encoding = Encoding::new(UTF_8); + // Change a file's line endings on disk from unix to windows. The buffer's // state updates correctly. fs.save( path!("/dir/file1").as_ref(), &Rope::from_str("aaa\nb\nc\n", cx.background_executor()), LineEnding::Windows, + encoding, ) .await .unwrap(); @@ -8979,7 +9013,12 @@ async fn test_ignored_dirs_events(cx: &mut gpui::TestAppContext) { let tree = project.read_with(cx, |project, cx| project.worktrees(cx).next().unwrap()); tree.flush_fs_events(cx).await; tree.update(cx, |tree, cx| { - tree.load_file(rel_path("project/target/debug/important_text.txt"), cx) + tree.load_file( + rel_path("project/target/debug/important_text.txt"), + &Default::default(), + None, + cx, + ) }) .await .unwrap(); @@ -9140,7 +9179,12 @@ async fn test_odd_events_for_ignored_dirs( let tree = project.read_with(cx, |project, cx| project.worktrees(cx).next().unwrap()); tree.update(cx, |tree, cx| { - tree.load_file(rel_path("target/debug/foo.txt"), cx) + tree.load_file( + rel_path("target/debug/foo.txt"), + &Default::default(), + None, + cx, + ) }) .await .unwrap(); diff --git a/crates/remote_server/Cargo.toml b/crates/remote_server/Cargo.toml index 5034b24e0661eb..ac8c680d4e5218 100644 --- a/crates/remote_server/Cargo.toml +++ b/crates/remote_server/Cargo.toml @@ -28,6 +28,7 @@ clap.workspace = true client.workspace = true dap_adapters.workspace = true debug_adapter_extension.workspace = true +encodings.workspace = true env_logger.workspace = true extension.workspace = true extension_host.workspace = true diff --git a/crates/remote_server/src/headless_project.rs b/crates/remote_server/src/headless_project.rs index 5d50853601b394..e6e81ca577865b 100644 --- a/crates/remote_server/src/headless_project.rs +++ b/crates/remote_server/src/headless_project.rs @@ -506,7 +506,7 @@ impl HeadlessProject { let (buffer_store, buffer) = this.update(&mut cx, |this, cx| { let buffer_store = this.buffer_store.clone(); let buffer = this.buffer_store.update(cx, |buffer_store, cx| { - buffer_store.open_buffer(ProjectPath { worktree_id, path }, cx) + buffer_store.open_buffer(ProjectPath { worktree_id, path }, &Default::default(), cx) }); anyhow::Ok((buffer_store, buffer)) })??; @@ -597,6 +597,7 @@ impl HeadlessProject { worktree_id: worktree.read(cx).id(), path: path, }, + &Default::default(), cx, ) }); diff --git a/crates/remote_server/src/remote_editing_tests.rs b/crates/remote_server/src/remote_editing_tests.rs index c7e09e3f681d77..ae8235b6559e40 100644 --- a/crates/remote_server/src/remote_editing_tests.rs +++ b/crates/remote_server/src/remote_editing_tests.rs @@ -6,16 +6,17 @@ use agent::{AgentTool, ReadFileTool, ReadFileToolInput, ToolCallEventStream}; use client::{Client, UserStore}; use clock::FakeSystemClock; use collections::{HashMap, HashSet}; -use language_model::LanguageModelToolResultContent; +use encodings::Encoding; use extension::ExtensionHostProxy; use fs::{FakeFs, Fs}; -use gpui::{AppContext as _, Entity, SemanticVersion, SharedString, TestAppContext}; +use gpui::{AppContext as _, Entity, SemanticVersion, TestAppContext}; use http_client::{BlockedHttpClient, FakeHttpClient}; use language::{ Buffer, FakeLspAdapter, LanguageConfig, LanguageMatcher, LanguageRegistry, LineEnding, Rope, language_settings::{AllLanguageSettings, language_settings}, }; +use language_model::LanguageModelToolResultContent; use lsp::{CompletionContext, CompletionResponse, CompletionTriggerKind, LanguageServerName}; use node_runtime::NodeRuntime; use project::{ @@ -34,6 +35,8 @@ use std::{ use unindent::Unindent as _; use util::{path, rel_path::rel_path}; +use gpui::SharedString; + #[gpui::test] async fn test_basic_remote_editing(cx: &mut TestAppContext, server_cx: &mut TestAppContext) { let fs = FakeFs::new(server_cx.executor()); @@ -122,6 +125,7 @@ async fn test_basic_remote_editing(cx: &mut TestAppContext, server_cx: &mut Test path!("/code/project1/src/main.rs").as_ref(), &Rope::from_str_small("fn main() {}"), Default::default(), + Encoding::default(), ) .await .unwrap(); @@ -768,6 +772,7 @@ async fn test_remote_reload(cx: &mut TestAppContext, server_cx: &mut TestAppCont &PathBuf::from(path!("/code/project1/src/lib.rs")), &Rope::from_str_small("bangles"), LineEnding::Unix, + Encoding::default(), ) .await .unwrap(); @@ -783,6 +788,7 @@ async fn test_remote_reload(cx: &mut TestAppContext, server_cx: &mut TestAppCont &PathBuf::from(path!("/code/project1/src/lib.rs")), &Rope::from_str_small("bloop"), LineEnding::Unix, + Encoding::default(), ) .await .unwrap(); diff --git a/crates/vim/Cargo.toml b/crates/vim/Cargo.toml index 4cea29508f437d..0aa24f40f40da9 100644 --- a/crates/vim/Cargo.toml +++ b/crates/vim/Cargo.toml @@ -54,6 +54,7 @@ vim_mode_setting.workspace = true workspace.workspace = true zed_actions.workspace = true + [dev-dependencies] assets.workspace = true command_palette.workspace = true diff --git a/crates/workspace/Cargo.toml b/crates/workspace/Cargo.toml index d5d3016ab27043..c1d0153fc08441 100644 --- a/crates/workspace/Cargo.toml +++ b/crates/workspace/Cargo.toml @@ -35,6 +35,7 @@ clock.workspace = true collections.workspace = true component.workspace = true db.workspace = true +encodings.workspace = true fs.workspace = true futures.workspace = true gpui.workspace = true diff --git a/crates/workspace/src/invalid_item_view.rs b/crates/workspace/src/invalid_item_view.rs index eb6c8f3299838c..75dcf768a16c2d 100644 --- a/crates/workspace/src/invalid_item_view.rs +++ b/crates/workspace/src/invalid_item_view.rs @@ -1,6 +1,7 @@ use std::{path::Path, sync::Arc}; +use ui::TintColor; -use gpui::{EventEmitter, FocusHandle, Focusable}; +use gpui::{EventEmitter, FocusHandle, Focusable, div}; use ui::{ App, Button, ButtonCommon, ButtonStyle, Clickable, Context, FluentBuilder, InteractiveElement, KeyBinding, Label, LabelCommon, LabelSize, ParentElement, Render, SharedString, Styled as _, @@ -77,6 +78,9 @@ impl Focusable for InvalidItemView { impl Render for InvalidItemView { fn render(&mut self, _window: &mut Window, cx: &mut Context) -> impl gpui::IntoElement { let abs_path = self.abs_path.clone(); + let path0 = self.abs_path.clone(); + let path1 = self.abs_path.clone(); + v_flex() .size_full() .track_focus(&self.focus_handle(cx)) @@ -91,21 +95,70 @@ impl Render for InvalidItemView { .gap_2() .child(h_flex().justify_center().child("Could not open file")) .child( - h_flex() - .justify_center() - .child(Label::new(self.error.clone()).size(LabelSize::Small)), + h_flex().justify_center().child( + div() + .whitespace_normal() + .text_center() + .child(Label::new(self.error.clone()).size(LabelSize::Small)), + ), ) .when(self.is_local, |contents| { - contents.child( - h_flex().justify_center().child( - Button::new("open-with-system", "Open in Default App") - .on_click(move |_, _, cx| { - cx.open_with_system(&abs_path); - }) - .style(ButtonStyle::Outlined) - .key_binding(KeyBinding::for_action(&OpenWithSystem, cx)), - ), - ) + contents + .child( + h_flex().justify_center().child( + Button::new("open-with-system", "Open in Default App") + .on_click(move |_, _, cx| { + cx.open_with_system(&abs_path); + }) + .style(ButtonStyle::Outlined) + .key_binding(KeyBinding::for_action( + &OpenWithSystem, + cx, + )), + ), + ) + .child( + h_flex() + .justify_center() + .child( + Button::new( + "open-with-encoding", + "Open With a Different Encoding", + ) + .style(ButtonStyle::Outlined) + .on_click( + move |_, window, cx| { + window.dispatch_action( + Box::new( + zed_actions::encodings_ui::Toggle( + path0.clone(), + ), + ), + cx, + ) + }, + ), + ) + .child( + Button::new( + "accept-risk-and-open", + "Accept the Risk and Open", + ) + .style(ButtonStyle::Tinted(TintColor::Warning)) + .on_click( + move |_, window, cx| { + window.dispatch_action( + Box::new( + zed_actions::encodings_ui::ForceOpen( + path1.clone(), + ), + ), + cx, + ); + }, + ), + ), + ) }), ), ) diff --git a/crates/workspace/src/workspace.rs b/crates/workspace/src/workspace.rs index 62e29f215146c0..8fc2432429f6b9 100644 --- a/crates/workspace/src/workspace.rs +++ b/crates/workspace/src/workspace.rs @@ -19,6 +19,8 @@ mod workspace_settings; pub use crate::notifications::NotificationFrame; pub use dock::Panel; +use encodings::Encoding; + pub use path_list::PathList; pub use toast_layer::{ToastAction, ToastLayer, ToastView}; @@ -30,6 +32,8 @@ use client::{ }; use collections::{HashMap, HashSet, hash_map}; use dock::{Dock, DockPosition, PanelButtons, PanelHandle, RESIZE_HANDLE_SIZE}; +use encodings::EncodingOptions; + use futures::{ Future, FutureExt, StreamExt, channel::{ @@ -621,6 +625,7 @@ type BuildProjectItemForPathFn = fn( &Entity, &ProjectPath, + Option, &mut Window, &mut App, ) -> Option, WorkspaceItemBuilder)>>>; @@ -642,8 +647,12 @@ impl ProjectItemRegistry { }, ); self.build_project_item_for_path_fns - .push(|project, project_path, window, cx| { + .push(|project, project_path, encoding, window, cx| { let project_path = project_path.clone(); + let encoding = encoding.unwrap_or_default(); + + project.update(cx, |project, _| project.encoding_options.encoding.set(encoding.get())); + let is_file = project .read(cx) .entry_for_path(&project_path, cx) @@ -712,14 +721,17 @@ impl ProjectItemRegistry { &self, project: &Entity, path: &ProjectPath, + encoding: Option, window: &mut Window, cx: &mut App, ) -> Task, WorkspaceItemBuilder)>> { - let Some(open_project_item) = self - .build_project_item_for_path_fns - .iter() - .rev() - .find_map(|open_project_item| open_project_item(project, path, window, cx)) + let Some(open_project_item) = + self.build_project_item_for_path_fns + .iter() + .rev() + .find_map(|open_project_item| { + open_project_item(project, path, encoding.clone(), window, cx) + }) else { return Task::ready(Err(anyhow!("cannot open file {:?}", path.path))); }; @@ -1179,6 +1191,7 @@ pub struct Workspace { session_id: Option, scheduled_tasks: Vec>, last_open_dock_positions: Vec, + pub encoding_options: EncodingOptions, } impl EventEmitter for Workspace {} @@ -1519,9 +1532,9 @@ impl Workspace { serializable_items_tx, _items_serializer, session_id: Some(session_id), - scheduled_tasks: Vec::new(), last_open_dock_positions: Vec::new(), + encoding_options: Default::default(), } } @@ -1932,6 +1945,10 @@ impl Workspace { window: &mut Window, cx: &mut Context, ) -> Task> { + // This is done so that we would get an error when we try to open the file with wrong encoding, + // and not silently use the previously set encoding. + self.encoding_options.reset(); + let to_load = if let Some(pane) = pane.upgrade() { pane.update(cx, |pane, cx| { window.focus(&pane.focus_handle(cx)); @@ -3561,8 +3578,25 @@ impl Workspace { window: &mut Window, cx: &mut App, ) -> Task, WorkspaceItemBuilder)>> { + let project = self.project(); + + project.update(cx, |project, _| { + project.encoding_options.force.store( + self.encoding_options + .force + .load(std::sync::atomic::Ordering::Acquire), + std::sync::atomic::Ordering::Release, + ); + }); + let registry = cx.default_global::().clone(); - registry.open_path(self.project(), &path, window, cx) + registry.open_path( + project, + &path, + Some((*self.encoding_options.encoding).clone()), + window, + cx, + ) } pub fn find_project_item( @@ -7586,8 +7620,15 @@ pub fn create_and_open_local_file( let fs = workspace.read_with(cx, |workspace, _| workspace.app_state().fs.clone())?; if !fs.is_file(path).await { fs.create_file(path, Default::default()).await?; - fs.save(path, &default_content(cx), Default::default()) - .await?; + fs.save( + path, + &default_content(cx), + + + Default::default(), + Default::default(), + ) + .await?; } let mut items = workspace diff --git a/crates/worktree/Cargo.toml b/crates/worktree/Cargo.toml index 6d132fbd2cb8c7..46ad794b2886a8 100644 --- a/crates/worktree/Cargo.toml +++ b/crates/worktree/Cargo.toml @@ -27,6 +27,7 @@ anyhow.workspace = true async-lock.workspace = true clock.workspace = true collections.workspace = true +encodings.workspace = true fs.workspace = true futures.workspace = true fuzzy.workspace = true @@ -47,6 +48,8 @@ smol.workspace = true sum_tree.workspace = true text.workspace = true util.workspace = true +encoding = "0.2.33" + [dev-dependencies] clock = { workspace = true, features = ["test-support"] } diff --git a/crates/worktree/src/worktree.rs b/crates/worktree/src/worktree.rs index a4d3f61141c8b0..b4ceceda3f6f32 100644 --- a/crates/worktree/src/worktree.rs +++ b/crates/worktree/src/worktree.rs @@ -7,6 +7,7 @@ use ::ignore::gitignore::{Gitignore, GitignoreBuilder}; use anyhow::{Context as _, Result, anyhow}; use clock::ReplicaId; use collections::{HashMap, HashSet, VecDeque}; +use encodings::{Encoding, EncodingOptions}; use fs::{Fs, MTime, PathEvent, RemoveOptions, Watcher, copy_recursive, read_dir_items}; use futures::{ FutureExt as _, Stream, StreamExt, @@ -703,9 +704,15 @@ impl Worktree { } } - pub fn load_file(&self, path: &RelPath, cx: &Context) -> Task> { + pub fn load_file( + &self, + path: &RelPath, + options: &EncodingOptions, + buffer_encoding: Option>, + cx: &Context, + ) -> Task> { match self { - Worktree::Local(this) => this.load_file(path, cx), + Worktree::Local(this) => this.load_file(path, options, buffer_encoding, cx), Worktree::Remote(_) => { Task::ready(Err(anyhow!("remote worktrees can't yet load files"))) } @@ -730,10 +737,11 @@ impl Worktree { path: Arc, text: Rope, line_ending: LineEnding, + encoding: Encoding, cx: &Context, ) -> Task>> { match self { - Worktree::Local(this) => this.write_file(path, text, line_ending, cx), + Worktree::Local(this) => this.write_file(path, text, line_ending, cx, encoding), Worktree::Remote(_) => { Task::ready(Err(anyhow!("remote worktree can't yet write files"))) } @@ -1303,6 +1311,7 @@ impl LocalWorktree { }, is_local: true, is_private, + encoding: None, }) } }; @@ -1311,12 +1320,20 @@ impl LocalWorktree { }) } - fn load_file(&self, path: &RelPath, cx: &Context) -> Task> { + fn load_file( + &self, + path: &RelPath, + options: &EncodingOptions, + buffer_encoding: Option>, + cx: &Context, + ) -> Task> { let path = Arc::from(path); let abs_path = self.absolutize(&path); let fs = self.fs.clone(); let entry = self.refresh_entry(path.clone(), None, cx); let is_private = self.is_path_private(path.as_ref()); + let options = options.clone(); + let encoding = options.encoding.clone(); let this = cx.weak_entity(); cx.background_spawn(async move { @@ -1334,7 +1351,9 @@ impl LocalWorktree { anyhow::bail!("File is too large to load"); } } - let text = fs.load(&abs_path).await?; + let text = fs + .load_with_encoding(&abs_path, &options, buffer_encoding.clone()) + .await?; let worktree = this.upgrade().context("worktree was dropped")?; let file = match entry.await? { @@ -1358,6 +1377,7 @@ impl LocalWorktree { }, is_local: true, is_private, + encoding: Some(encoding), }) } }; @@ -1446,6 +1466,7 @@ impl LocalWorktree { text: Rope, line_ending: LineEnding, cx: &Context, + encoding: Encoding, ) -> Task>> { let fs = self.fs.clone(); let is_private = self.is_path_private(&path); @@ -1454,7 +1475,10 @@ impl LocalWorktree { let write = cx.background_spawn({ let fs = fs.clone(); let abs_path = abs_path.clone(); - async move { fs.save(&abs_path, &text, line_ending).await } + { + let encoding = encoding.clone(); + async move { fs.save(&abs_path, &text, line_ending, encoding).await } + } }); cx.spawn(async move |this, cx| { @@ -1488,6 +1512,7 @@ impl LocalWorktree { entry_id: None, is_local: true, is_private, + encoding: Some(Arc::new(encoding)), })) } }) @@ -3041,7 +3066,7 @@ impl fmt::Debug for Snapshot { } } -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone)] pub struct File { pub worktree: Entity, pub path: Arc, @@ -3049,8 +3074,35 @@ pub struct File { pub entry_id: Option, pub is_local: bool, pub is_private: bool, + pub encoding: Option>, } +impl PartialEq for File { + fn eq(&self, other: &Self) -> bool { + if self.worktree == other.worktree + && self.path == other.path + && self.disk_state == other.disk_state + && self.entry_id == other.entry_id + && self.is_local == other.is_local + && self.is_private == other.is_private + && (if let Some(encoding) = &self.encoding + && let Some(other_encoding) = &other.encoding + { + if encoding.get() != other_encoding.get() { + false + } else { + true + } + } else { + true + }) + { + true + } else { + false + } + } +} impl language::File for File { fn as_local(&self) -> Option<&dyn language::LocalFile> { if self.is_local { Some(self) } else { None } @@ -3097,6 +3149,13 @@ impl language::File for File { fn path_style(&self, cx: &App) -> PathStyle { self.worktree.read(cx).path_style() } + fn encoding(&self) -> Option> { + if let Some(encoding) = &self.encoding { + Some(encoding.clone()) + } else { + None + } + } } impl language::LocalFile for File { @@ -3104,11 +3163,30 @@ impl language::LocalFile for File { self.worktree.read(cx).absolutize(&self.path) } - fn load(&self, cx: &App) -> Task> { + fn load( + &self, + cx: &App, + options: &EncodingOptions, + buffer_encoding: Option>, + ) -> Task> { let worktree = self.worktree.read(cx).as_local().unwrap(); let abs_path = worktree.absolutize(&self.path); let fs = worktree.fs.clone(); - cx.background_spawn(async move { fs.load(&abs_path).await }) + let options = EncodingOptions { + encoding: options.encoding.clone(), + force: std::sync::atomic::AtomicBool::new( + options.force.load(std::sync::atomic::Ordering::Acquire), + ), + detect_utf16: std::sync::atomic::AtomicBool::new( + options + .detect_utf16 + .load(std::sync::atomic::Ordering::Acquire), + ), + }; + cx.background_spawn(async move { + fs.load_with_encoding(&abs_path, &options, buffer_encoding) + .await + }) } fn load_bytes(&self, cx: &App) -> Task>> { @@ -3132,6 +3210,7 @@ impl File { entry_id: Some(entry.id), is_local: true, is_private: entry.is_private, + encoding: None, }) } @@ -3162,6 +3241,7 @@ impl File { entry_id: proto.entry_id.map(ProjectEntryId::from_proto), is_local: false, is_private: false, + encoding: None, }) } diff --git a/crates/worktree/src/worktree_tests.rs b/crates/worktree/src/worktree_tests.rs index 1cce23712ae88f..673764bbba9ca2 100644 --- a/crates/worktree/src/worktree_tests.rs +++ b/crates/worktree/src/worktree_tests.rs @@ -467,7 +467,12 @@ async fn test_open_gitignored_files(cx: &mut TestAppContext) { let prev_read_dir_count = fs.read_dir_call_count(); let loaded = tree .update(cx, |tree, cx| { - tree.load_file(rel_path("one/node_modules/b/b1.js"), cx) + tree.load_file( + rel_path("one/node_modules/b/b1.js"), + &Default::default(), + None, + cx, + ) }) .await .unwrap(); @@ -507,7 +512,12 @@ async fn test_open_gitignored_files(cx: &mut TestAppContext) { let prev_read_dir_count = fs.read_dir_call_count(); let loaded = tree .update(cx, |tree, cx| { - tree.load_file(rel_path("one/node_modules/a/a2.js"), cx) + tree.load_file( + rel_path("one/node_modules/a/a2.js"), + &Default::default(), + None, + cx, + ) }) .await .unwrap(); @@ -651,6 +661,7 @@ async fn test_dirs_no_longer_ignored(cx: &mut TestAppContext) { "/root/.gitignore".as_ref(), &Rope::from_str("e", cx.background_executor()), Default::default(), + Default::default(), ) .await .unwrap(); @@ -723,6 +734,7 @@ async fn test_write_file(cx: &mut TestAppContext) { rel_path("tracked-dir/file.txt").into(), Rope::from_str("hello", cx.background_executor()), Default::default(), + Default::default(), cx, ) }) @@ -734,6 +746,7 @@ async fn test_write_file(cx: &mut TestAppContext) { rel_path("ignored-dir/file.txt").into(), Rope::from_str("world", cx.background_executor()), Default::default(), + Default::default(), cx, ) }) @@ -1769,6 +1782,7 @@ fn randomly_mutate_worktree( Rope::default(), Default::default(), cx, + Default::default(), ); cx.background_spawn(async move { task.await?; @@ -1861,6 +1875,7 @@ async fn randomly_mutate_fs( &ignore_path, &Rope::from_str(ignore_contents.as_str(), executor), Default::default(), + Default::default(), ) .await .unwrap(); diff --git a/crates/zed/Cargo.toml b/crates/zed/Cargo.toml index 9f6196c1482bcf..73681693e91c59 100644 --- a/crates/zed/Cargo.toml +++ b/crates/zed/Cargo.toml @@ -52,6 +52,8 @@ debugger_ui.workspace = true diagnostics.workspace = true editor.workspace = true zeta2_tools.workspace = true +encodings.workspace = true +encodings_ui.workspace = true env_logger.workspace = true extension.workspace = true extension_host.workspace = true diff --git a/crates/zed/src/main.rs b/crates/zed/src/main.rs index b873a58d3b6133..576e8cd41a587d 100644 --- a/crates/zed/src/main.rs +++ b/crates/zed/src/main.rs @@ -630,6 +630,7 @@ pub fn main() { zeta::init(cx); inspector_ui::init(app_state.clone(), cx); json_schema_store::init(cx); + encodings_ui::init(cx); cx.observe_global::({ let http = app_state.client.http_client(); diff --git a/crates/zed/src/zed.rs b/crates/zed/src/zed.rs index bd0a600ce52a26..134c0eb452fb52 100644 --- a/crates/zed/src/zed.rs +++ b/crates/zed/src/zed.rs @@ -443,6 +443,10 @@ pub fn initialize_workspace( } }); + let encoding_indicator = cx.new(|_cx| { + encodings_ui::EncodingIndicator::new(None, workspace.weak_handle(), None, None) + }); + let cursor_position = cx.new(|_| go_to_line::cursor_position::CursorPosition::new(workspace)); let line_ending_indicator = @@ -458,6 +462,7 @@ pub fn initialize_workspace( status_bar.add_right_item(line_ending_indicator, window, cx); status_bar.add_right_item(vim_mode_indicator, window, cx); status_bar.add_right_item(cursor_position, window, cx); + status_bar.add_right_item(encoding_indicator, window, cx); status_bar.add_right_item(image_info, window, cx); }); @@ -2172,6 +2177,8 @@ mod tests { use assets::Assets; use collections::HashSet; use editor::{DisplayPoint, Editor, SelectionEffects, display_map::DisplayRow}; + use encodings::Encoding; + use gpui::{ Action, AnyWindowHandle, App, AssetSource, BorrowAppContext, SemanticVersion, TestAppContext, UpdateGlobal, VisualTestContext, WindowHandle, actions, @@ -4377,6 +4384,7 @@ mod tests { "/settings.json".as_ref(), &Rope::from_str_small(r#"{"base_keymap": "Atom"}"#), Default::default(), + Encoding::default(), ) .await .unwrap(); @@ -4387,6 +4395,7 @@ mod tests { "/keymap.json".as_ref(), &Rope::from_str_small(r#"[{"bindings": {"backspace": "test_only::ActionA"}}]"#), Default::default(), + Encoding::default(), ) .await .unwrap(); @@ -4435,6 +4444,7 @@ mod tests { "/keymap.json".as_ref(), &Rope::from_str_small(r#"[{"bindings": {"backspace": "test_only::ActionB"}}]"#), Default::default(), + Encoding::default(), ) .await .unwrap(); @@ -4455,6 +4465,7 @@ mod tests { "/settings.json".as_ref(), &Rope::from_str_small(r#"{"base_keymap": "JetBrains"}"#), Default::default(), + Encoding::default(), ) .await .unwrap(); @@ -4495,6 +4506,7 @@ mod tests { "/settings.json".as_ref(), &Rope::from_str_small(r#"{"base_keymap": "Atom"}"#), Default::default(), + Encoding::default(), ) .await .unwrap(); @@ -4504,6 +4516,7 @@ mod tests { "/keymap.json".as_ref(), &Rope::from_str_small(r#"[{"bindings": {"backspace": "test_only::ActionA"}}]"#), Default::default(), + Encoding::default(), ) .await .unwrap(); @@ -4547,6 +4560,7 @@ mod tests { "/keymap.json".as_ref(), &Rope::from_str_small(r#"[{"bindings": {"backspace": null}}]"#), Default::default(), + Encoding::default(), ) .await .unwrap(); @@ -4567,6 +4581,7 @@ mod tests { "/settings.json".as_ref(), &Rope::from_str_small(r#"{"base_keymap": "JetBrains"}"#), Default::default(), + Encoding::default(), ) .await .unwrap(); diff --git a/crates/zed/src/zed/open_listener.rs b/crates/zed/src/zed/open_listener.rs index bc6c25105e69eb..790c3d3ba2b567 100644 --- a/crates/zed/src/zed/open_listener.rs +++ b/crates/zed/src/zed/open_listener.rs @@ -653,6 +653,7 @@ mod tests { ipc::{self}, }; use editor::Editor; + use encodings::Encoding; use gpui::TestAppContext; use language::LineEnding; use remote::SshConnectionOptions; @@ -863,6 +864,7 @@ mod tests { Path::new(file1_path), &Rope::from_str("content1", cx.background_executor()), LineEnding::Unix, + Encoding::default(), ) .await .unwrap(); @@ -877,6 +879,7 @@ mod tests { Path::new(file2_path), &Rope::from_str("content2", cx.background_executor()), LineEnding::Unix, + Encoding::default(), ) .await .unwrap(); diff --git a/crates/zed_actions/src/lib.rs b/crates/zed_actions/src/lib.rs index 5cb2903fa653fc..5c60efd96f6d9f 100644 --- a/crates/zed_actions/src/lib.rs +++ b/crates/zed_actions/src/lib.rs @@ -299,6 +299,20 @@ pub mod settings_profile_selector { pub struct Toggle; } +pub mod encodings_ui { + use std::sync::Arc; + + use gpui::Action; + use schemars::JsonSchema; + use serde::Deserialize; + + #[derive(PartialEq, Debug, Clone, Action, JsonSchema, Deserialize)] + pub struct Toggle(pub Arc); + + #[derive(PartialEq, Debug, Clone, Action, JsonSchema, Deserialize)] + pub struct ForceOpen(pub Arc); +} + pub mod agent { use gpui::actions; diff --git a/crates/zeta/src/zeta.rs b/crates/zeta/src/zeta.rs index ca2edd0682e181..d95e62be5e4ceb 100644 --- a/crates/zeta/src/zeta.rs +++ b/crates/zeta/src/zeta.rs @@ -1986,7 +1986,7 @@ mod tests { .worktree_for_root_name("closed_source_worktree", cx) .unwrap(); worktree2.update(cx, |worktree2, cx| { - worktree2.load_file(rel_path("main.rs"), cx) + worktree2.load_file(rel_path("main.rs"), &Default::default(), None, cx) }) }) .await diff --git a/docs/src/configuring-zed.md b/docs/src/configuring-zed.md index 054b6b1b5c812b..eddaf1084b03c9 100644 --- a/docs/src/configuring-zed.md +++ b/docs/src/configuring-zed.md @@ -1541,7 +1541,8 @@ Positive `integer` value between 1 and 32. Values outside of this range will be "status_bar": { "active_language_button": true, "cursor_position_button": true, - "line_endings_button": false + "line_endings_button": false, + "encoding_indicator": true, }, ```