Yehowshua/ZIm
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add support for saving and opening files in different encodings. The implementation is now complete.

Browse source
This commit is contained in:
R Aadarsh 2025-08-26 22:38:47 +05:30
parent 0e1f9f689c
commit a3f5e91f0f
36 changed files with 362 additions and 92 deletions
Download patch file Download diff file Expand all files Collapse all files

10
Cargo.lock generated
View file

@ -228,6 +228,7 @@ dependencies = [
"ctor",
"db",
"editor",
"encoding",
"env_logger 0.11.8",
"fs",
"futures 0.3.31",
@ -961,6 +962,7 @@ dependencies = [
"derive_more",
"diffy",
"editor",
"encoding",
"feature_flags",
"fs",
"futures 0.3.31",
@ -3302,6 +3304,7 @@ dependencies = [
"dashmap 6.1.0",
"debugger_ui",
"editor",
"encoding",
"envy",
"extension",
"file_finder",
@ -5272,6 +5275,7 @@ dependencies = [
name = "encodings"
version = "0.1.0"
dependencies = [
"anyhow",
"editor",
"encoding",
"fuzzy",
@ -5626,6 +5630,7 @@ dependencies = [
"criterion",
"ctor",
"dap",
"encoding",
"extension",
"fs",
"futures 0.3.31",
@ -6136,6 +6141,7 @@ dependencies = [
"paths",
"proto",
"rope",
"schemars",
"serde",
"serde_json",
"smol",
@ -6561,6 +6567,7 @@ dependencies = [
"ctor",
"db",
"editor",
"encoding",
"futures 0.3.31",
"fuzzy",
"git",
@ -12644,6 +12651,7 @@ dependencies = [
"context_server",
"dap",
"dap_adapters",
"encoding",
"extension",
"fancy-regex 0.14.0",
"fs",
@ -13567,6 +13575,7 @@ dependencies = [
"dap_adapters",
"debug_adapter_extension",
"editor",
"encoding",
"env_logger 0.11.8",
"extension",
"extension_host",
@ -19878,6 +19887,7 @@ dependencies = [
"component",
"dap",
"db",
"encoding",
"fs",
"futures 0.3.31",
"gpui",

1
Cargo.toml
View file

@ -477,6 +477,7 @@ documented = "0.9.1"
dotenvy = "0.15.0"
ec4rs = "1.1"
emojis = "0.6.1"
encoding = "0.2.33"
env_logger = "0.11"
exec = "0.3.1"
fancy-regex = "0.14.0"

2
crates/agent2/Cargo.toml
View file

@ -32,6 +32,7 @@ cloud_llm_client.workspace = true
collections.workspace = true
context_server.workspace = true
db.workspace = true
encoding.workspace = true
fs.workspace = true
futures.workspace = true
git.workspace = true
@ -72,6 +73,7 @@ which.workspace = true
workspace-hack.workspace = true
zstd.workspace = true
[dev-dependencies]
agent = { workspace = true, "features" = ["test-support"] }
agent_servers = { workspace = true, "features" = ["test-support"] }

5
crates/agent2/src/tools/edit_file_tool.rs
View file

@ -523,7 +523,8 @@ mod tests {
use super::*;
use crate::{ContextServerRegistry, Templates};
use client::TelemetrySettings;
use fs::Fs;
use encoding::all::UTF_8;
use fs::{Fs, encodings::EncodingWrapper};
use gpui::{TestAppContext, UpdateGlobal};
use language_model::fake_provider::FakeLanguageModel;
use prompt_store::ProjectContext;
@ -705,6 +706,7 @@ mod tests {
path!("/root/src/main.rs").as_ref(),
&"initial content".into(),
language::LineEnding::Unix,
EncodingWrapper::new(UTF_8),
)
.await
.unwrap();
@ -873,6 +875,7 @@ mod tests {
path!("/root/src/main.rs").as_ref(),
&"initial content".into(),
language::LineEnding::Unix,
EncodingWrapper::new(UTF_8),
)
.await
.unwrap();

1
crates/assistant_tools/Cargo.toml
View file

@ -28,6 +28,7 @@ component.workspace = true
derive_more.workspace = true
diffy = "0.4.2"
editor.workspace = true
encoding.workspace = true
feature_flags.workspace = true
futures.workspace = true
gpui.workspace = true

5
crates/assistant_tools/src/edit_file_tool.rs
View file

@ -1231,8 +1231,9 @@ async fn build_buffer_diff(
#[cfg(test)]
mod tests {
use super::*;
use ::fs::Fs;
use ::fs::{Fs, encodings::EncodingWrapper};
use client::TelemetrySettings;
use encoding::all::UTF_8;
use gpui::{TestAppContext, UpdateGlobal};
use language_model::fake_provider::FakeLanguageModel;
use serde_json::json;
@ -1501,6 +1502,7 @@ mod tests {
path!("/root/src/main.rs").as_ref(),
&"initial content".into(),
language::LineEnding::Unix,
EncodingWrapper::new(UTF_8),
)
.await
.unwrap();
@ -1670,6 +1672,7 @@ mod tests {
path!("/root/src/main.rs").as_ref(),
&"initial content".into(),
language::LineEnding::Unix,
EncodingWrapper::new(UTF_8),
)
.await
.unwrap();

1
crates/collab/Cargo.toml
View file

@ -31,6 +31,7 @@ chrono.workspace = true
clock.workspace = true
collections.workspace = true
dashmap.workspace = true
encoding.workspace = true
envy = "0.4.2"
futures.workspace = true
gpui.workspace = true

5
crates/collab/src/tests/integration_tests.rs
View file

@ -12,7 +12,8 @@ use buffer_diff::{DiffHunkSecondaryStatus, DiffHunkStatus, assert_hunks};
use call::{ActiveCall, ParticipantLocation, Room, room};
use client::{RECEIVE_TIMEOUT, User};
use collections::{HashMap, HashSet};
use fs::{FakeFs, Fs as _, RemoveOptions};
use encoding::all::UTF_8;
use fs::{FakeFs, Fs as _, RemoveOptions, encodings::EncodingWrapper};
use futures::{StreamExt as _, channel::mpsc};
use git::status::{FileStatus, StatusCode, TrackedStatus, UnmergedStatus, UnmergedStatusCode};
use gpui::{
@ -3706,6 +3707,7 @@ async fn test_buffer_reloading(
path!("/dir/a.txt").as_ref(),
&new_contents,
LineEnding::Windows,
EncodingWrapper::new(UTF_8),
)
.await
.unwrap();
@ -4472,6 +4474,7 @@ async fn test_reloading_buffer_manually(
path!("/a/a.rs").as_ref(),
&Rope::from("let seven = 7;"),
LineEnding::Unix,
EncodingWrapper::new(UTF_8),
)
.await
.unwrap();

10
crates/collab/src/tests/random_project_collaboration_tests.rs
View file

@ -5,7 +5,8 @@ use async_trait::async_trait;
use call::ActiveCall;
use collections::{BTreeMap, HashMap};
use editor::Bias;
use fs::{FakeFs, Fs as _};
use encoding::all::UTF_8;
use fs::{FakeFs, Fs as _, encodings::EncodingWrapper};
use git::status::{FileStatus, StatusCode, TrackedStatus, UnmergedStatus, UnmergedStatusCode};
use gpui::{BackgroundExecutor, Entity, TestAppContext};
use language::{
@ -924,7 +925,12 @@ impl RandomizedTest for ProjectCollaborationTest {
client
.fs()
.save(&path, &content.as_str().into(), text::LineEnding::Unix)
.save(
&path,
&content.as_str().into(),
text::LineEnding::Unix,
EncodingWrapper::new(UTF_8),
)
.await
.unwrap();
}

2
crates/copilot/Cargo.toml
View file

@ -30,6 +30,7 @@ client.workspace = true
collections.workspace = true
command_palette_hooks.workspace = true
dirs.workspace = true
encoding.workspace = true
fs.workspace = true
futures.workspace = true
gpui.workspace = true
@ -53,7 +54,6 @@ util.workspace = true
workspace.workspace = true
workspace-hack.workspace = true
itertools.workspace = true
encoding = "0.2.33"
[target.'cfg(windows)'.dependencies]

3
crates/encodings/Cargo.toml
View file

@ -5,6 +5,7 @@ publish.workspace = true
edition.workspace = true
[dependencies]
anyhow.workspace = true
ui.workspace = true
workspace.workspace = true
gpui.workspace = true
@ -12,7 +13,7 @@ picker.workspace = true
util.workspace = true
fuzzy.workspace = true
editor.workspace = true
encoding = "0.2.33"
encoding.workspace = true
language.workspace = true
[lints]

50
crates/encodings/src/lib.rs
View file

@ -1,3 +1,4 @@
///! A crate for handling file encodings in the text editor.
use editor::Editor;
use encoding::Encoding;
use encoding::all::{
@ -12,7 +13,7 @@ use ui::{Button, ButtonCommon, Context, LabelSize, Render, Tooltip, Window, div}
use ui::{Clickable, ParentElement};
use workspace::{ItemHandle, StatusItemView, Workspace};
use crate::selectors::save_or_reopen::{EncodingSaveOrReopenSelector, get_current_encoding};
use crate::selectors::save_or_reopen::EncodingSaveOrReopenSelector;
/// A status bar item that shows the current file encoding and allows changing it.
pub struct EncodingIndicator {
@ -44,8 +45,6 @@ impl Render for EncodingIndicator {
}
impl EncodingIndicator {
pub fn get_current_encoding(&self, cx: &mut Context<Self>, editor: WeakEntity<Editor>) {}
pub fn new(
encoding: Option<&'static dyn encoding::Encoding>,
workspace: WeakEntity<Workspace>,
@ -187,3 +186,48 @@ pub fn encoding_from_index(index: usize) -> &'static dyn Encoding {
_ => UTF_8,
}
}
/// Get an encoding from its name.
pub fn encoding_from_name(name: &str) -> &'static dyn Encoding {
match name {
"UTF-8" => UTF_8,
"UTF-16 LE" => UTF_16LE,
"UTF-16 BE" => UTF_16BE,
"IBM866" => IBM866,
"ISO 8859-1" => ISO_8859_1,
"ISO 8859-2" => ISO_8859_2,
"ISO 8859-3" => ISO_8859_3,
"ISO 8859-4" => ISO_8859_4,
"ISO 8859-5" => ISO_8859_5,
"ISO 8859-6" => ISO_8859_6,
"ISO 8859-7" => ISO_8859_7,
"ISO 8859-8" => ISO_8859_8,
"ISO 8859-10" => ISO_8859_10,
"ISO 8859-13" => ISO_8859_13,
"ISO 8859-14" => ISO_8859_14,
"ISO 8859-15" => ISO_8859_15,
"ISO 8859-16" => ISO_8859_16,
"KOI8-R" => KOI8_R,
"KOI8-U" => KOI8_U,
"MacRoman" => MAC_ROMAN,
"Mac Cyrillic" => MAC_CYRILLIC,
"Windows-874" => WINDOWS_874,
"Windows-1250" => WINDOWS_1250,
"Windows-1251" => WINDOWS_1251,
"Windows-1252" => WINDOWS_1252,
"Windows-1253" => WINDOWS_1253,
"Windows-1254" => WINDOWS_1254,
"Windows-1255" => WINDOWS_1255,
"Windows-1256" => WINDOWS_1256,
"Windows-1257" => WINDOWS_1257,
"Windows-1258" => WINDOWS_1258,
"Windows-949" => WINDOWS_949,
"EUC-JP" => EUC_JP,
"ISO 2022-JP" => ISO_2022_JP,
"GBK" => GBK,
"GB18030" => GB18030,
"Big5" => BIG5_2003,
"HZ-GB-2312" => HZ,
_ => UTF_8, // Default to UTF-8 for unknown names
}
}

115
crates/encodings/src/selectors.rs
View file

@ -1,30 +1,28 @@
/// This module contains the encoding selectors for saving or reopening files with a different encoding.
/// It provides a modal view that allows the user to choose between saving with a different encoding
/// or reopening with a different encoding, and then selecting the desired encoding from a list.
pub mod save_or_reopen {
use editor::Editor;
use gpui::Styled;
use gpui::{AppContext, ParentElement};
use picker::Picker;
use picker::PickerDelegate;
use std::cell::RefCell;
use std::ops::{Deref, DerefMut};
use std::rc::Rc;
use std::sync::Arc;
use std::sync::atomic::AtomicBool;
use util::ResultExt;
use fuzzy::{StringMatch, StringMatchCandidate};
use gpui::{DismissEvent, Entity, EventEmitter, Focusable, WeakEntity};
use ui::{Context, HighlightedLabel, Label, ListItem, Render, Window, rems, v_flex};
use ui::{Context, HighlightedLabel, ListItem, Render, Window, rems, v_flex};
use workspace::{ModalView, Workspace};
use crate::selectors::encoding::{Action, EncodingSelector, EncodingSelectorDelegate};
use crate::selectors::encoding::{Action, EncodingSelector};
/// A modal view that allows the user to select between saving with a different encoding or
/// reopening with a different encoding.
pub struct EncodingSaveOrReopenSelector {
picker: Entity<Picker<EncodingSaveOrReopenDelegate>>,
pub current_selection: usize,
workspace: WeakEntity<Workspace>,
}
impl EncodingSaveOrReopenSelector {
@ -41,7 +39,6 @@ pub mod save_or_reopen {
Self {
picker,
current_selection: 0,
workspace,
}
}
@ -119,9 +116,17 @@ pub mod save_or_reopen {
.read(cx)
.active_excerpt(cx)?;
let weak_workspace = workspace.read(cx).weak_handle();
workspace.update(cx, |workspace, cx| {
workspace.toggle_modal(window, cx, |window, cx| {
EncodingSelector::new(window, cx, Action::Save, buffer.downgrade())
EncodingSelector::new(
window,
cx,
Action::Save,
buffer.downgrade(),
weak_workspace,
)
})
});
}
@ -134,9 +139,17 @@ pub mod save_or_reopen {
.read(cx)
.active_excerpt(cx)?;
let weak_workspace = workspace.read(cx).weak_handle();
workspace.update(cx, |workspace, cx| {
workspace.toggle_modal(window, cx, |window, cx| {
EncodingSelector::new(window, cx, Action::Reopen, buffer.downgrade())
EncodingSelector::new(
window,
cx,
Action::Reopen,
buffer.downgrade(),
weak_workspace,
)
})
});
}
@ -165,7 +178,7 @@ pub mod save_or_reopen {
) {
self.current_selection = ix;
self.selector
.update(cx, |selector, cx| {
.update(cx, |selector, _cx| {
selector.current_selection = ix;
})
.log_err();
@ -217,7 +230,7 @@ pub mod save_or_reopen {
.min(delegate.matches.len().saturating_sub(1));
delegate
.selector
.update(cx, |selector, cx| {
.update(cx, |selector, _cx| {
selector.current_selection = delegate.current_selection
})
.log_err();
@ -263,33 +276,27 @@ pub mod save_or_reopen {
}
}
/// This module contains the encoding selector for choosing an encoding to save or reopen a file with.
pub mod encoding {
use std::{
ops::DerefMut,
rc::{Rc, Weak},
sync::{Arc, atomic::AtomicBool},
};
use std::sync::atomic::AtomicBool;
use fuzzy::{StringMatch, StringMatchCandidate};
use gpui::{
AppContext, BackgroundExecutor, DismissEvent, Entity, EventEmitter, Focusable, Length,
WeakEntity, actions,
};
use gpui::{AppContext, DismissEvent, Entity, EventEmitter, Focusable, WeakEntity};
use language::Buffer;
use picker::{Picker, PickerDelegate};
use ui::{
Context, DefiniteLength, HighlightedLabel, Label, ListItem, ListItemSpacing, ParentElement,
Render, Styled, Window, rems, v_flex,
Context, HighlightedLabel, ListItem, ListItemSpacing, ParentElement, Render, Styled,
Window, rems, v_flex,
};
use util::{ResultExt, TryFutureExt};
use workspace::{ModalView, Workspace};
use crate::encoding_from_index;
use crate::encoding_from_name;
/// A modal view that allows the user to select an encoding from a list of encodings.
pub struct EncodingSelector {
picker: Entity<Picker<EncodingSelectorDelegate>>,
action: Action,
workspace: WeakEntity<Workspace>,
}
pub struct EncodingSelectorDelegate {
@ -298,12 +305,14 @@ pub mod encoding {
matches: Vec<StringMatch>,
selector: WeakEntity<EncodingSelector>,
buffer: WeakEntity<Buffer>,
action: Action,
}
impl EncodingSelectorDelegate {
pub fn new(
selector: WeakEntity<EncodingSelector>,
buffer: WeakEntity<Buffer>,
action: Action,
) -> EncodingSelectorDelegate {
EncodingSelectorDelegate {
current_selection: 0,
@ -350,6 +359,7 @@ pub mod encoding {
matches: Vec::new(),
selector,
buffer,
action,
}
}
}
@ -365,12 +375,7 @@ pub mod encoding {
self.current_selection
}
fn set_selected_index(
&mut self,
ix: usize,
window: &mut Window,
cx: &mut Context<Picker<Self>>,
) {
fn set_selected_index(&mut self, ix: usize, _: &mut Window, _: &mut Context<Picker<Self>>) {
self.current_selection = ix;
}
@ -427,21 +432,40 @@ pub mod encoding {
})
}
fn confirm(
&mut self,
secondary: bool,
window: &mut Window,
cx: &mut Context<Picker<Self>>,
) {
fn confirm(&mut self, _: bool, window: &mut Window, cx: &mut Context<Picker<Self>>) {
if let Some(buffer) = self.buffer.upgrade() {
buffer.update(cx, |buffer, cx| {
buffer.encoding = encoding_from_index(self.current_selection)
buffer.encoding =
encoding_from_name(self.matches[self.current_selection].string.as_str());
if self.action == Action::Reopen {
let executor = cx.background_executor().clone();
executor.spawn(buffer.reload(cx)).detach();
} else if self.action == Action::Save {
let executor = cx.background_executor().clone();
let workspace = self
.selector
.upgrade()
.unwrap()
.read(cx)
.workspace
.upgrade()
.unwrap();
executor
.spawn(workspace.update(cx, |workspace, cx| {
workspace
.save_active_item(workspace::SaveIntent::Save, window, cx)
.log_err()
}))
.detach();
}
});
}
self.dismissed(window, cx);
}
fn dismissed(&mut self, window: &mut Window, cx: &mut Context<Picker<Self>>) {
fn dismissed(&mut self, _: &mut Window, cx: &mut Context<Picker<Self>>) {
self.selector
.update(cx, |_, cx| cx.emit(DismissEvent))
.log_err();
@ -450,9 +474,9 @@ pub mod encoding {
fn render_match(
&self,
ix: usize,
selected: bool,
window: &mut Window,
cx: &mut Context<Picker<Self>>,
_: bool,
_: &mut Window,
_: &mut Context<Picker<Self>>,
) -> Option<Self::ListItem> {
Some(
ListItem::new(ix)
@ -466,6 +490,7 @@ pub mod encoding {
}
/// The action to perform after selecting an encoding.
#[derive(PartialEq, Clone)]
pub enum Action {
Save,
Reopen,
@ -477,11 +502,13 @@ pub mod encoding {
cx: &mut Context<EncodingSelector>,
action: Action,
buffer: WeakEntity<Buffer>,
workspace: WeakEntity<Workspace>,
) -> EncodingSelector {
let delegate = EncodingSelectorDelegate::new(cx.entity().downgrade(), buffer);
let delegate =
EncodingSelectorDelegate::new(cx.entity().downgrade(), buffer, action.clone());
let picker = cx.new(|cx| Picker::uniform_list(delegate, window, cx));
EncodingSelector { picker, action }
EncodingSelector { picker, workspace }
}
}

1
crates/extension_host/Cargo.toml
View file

@ -23,6 +23,7 @@ async-trait.workspace = true
client.workspace = true
collections.workspace = true
dap.workspace = true
encoding.workspace = true
extension.workspace = true
fs.workspace = true
futures.workspace = true

10
crates/extension_host/src/extension_host.rs
View file

@ -12,6 +12,7 @@ use async_tar::Archive;
use client::ExtensionProvides;
use client::{Client, ExtensionMetadata, GetExtensionsResponse, proto, telemetry::Telemetry};
use collections::{BTreeMap, BTreeSet, HashMap, HashSet, btree_map};
use encoding::all::UTF_8;
pub use extension::ExtensionManifest;
use extension::extension_builder::{CompileExtensionOptions, ExtensionBuilder};
use extension::{
@ -20,6 +21,7 @@ use extension::{
ExtensionLanguageServerProxy, ExtensionSlashCommandProxy, ExtensionSnippetProxy,
ExtensionThemeProxy,
};
use fs::encodings::EncodingWrapper;
use fs::{Fs, RemoveOptions};
use futures::future::join_all;
use futures::{
@ -1468,7 +1470,12 @@ impl ExtensionStore {
}
if let Ok(index_json) = serde_json::to_string_pretty(&index) {
fs.save(&index_path, &index_json.as_str().into(), Default::default())
fs.save(
&index_path,
&index_json.as_str().into(),
Default::default(),
EncodingWrapper::new(UTF_8),
)
.await
.context("failed to save extension index")
.log_err();
@ -1636,6 +1643,7 @@ impl ExtensionStore {
&tmp_dir.join(EXTENSION_TOML),
&Rope::from(manifest_toml),
language::LineEnding::Unix,
EncodingWrapper::new(UTF_8),
)
.await?;
} else {

4
crates/fs/Cargo.toml
View file

@ -16,6 +16,7 @@ anyhow.workspace = true
async-tar.workspace = true
async-trait.workspace = true
collections.workspace = true
encoding.workspace = true
futures.workspace = true
git.workspace = true
gpui.workspace = true
@ -34,7 +35,8 @@ text.workspace = true
time.workspace = true
util.workspace = true
workspace-hack.workspace = true
encoding = "0.2.33"
schemars.workspace = true
[target.'cfg(target_os = "macos")'.dependencies]

58
crates/fs/src/encodings.rs
View file

@ -1,14 +1,70 @@
use anyhow::{Error, Result};
//! Encoding and decoding utilities using the `encoding` crate.
use std::fmt::Debug;
use anyhow::{Error, Result};
use encoding::Encoding;
use serde::{Deserialize, de::Visitor};
/// A wrapper around `encoding::Encoding` to implement `Send` and `Sync`.
/// Since the reference is static, it is safe to send it across threads.
pub struct EncodingWrapper(&'static dyn Encoding);
impl Debug for EncodingWrapper {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_tuple("EncodingWrapper")
.field(&self.0.name())
.finish()
}
}
pub struct EncodingWrapperVisitor;
impl<'vi> Visitor<'vi> for EncodingWrapperVisitor {
type Value = EncodingWrapper;
fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
formatter.write_str("a valid encoding name")
}
fn visit_str<E: serde::de::Error>(self, encoding: &str) -> Result<EncodingWrapper, E> {
Ok(EncodingWrapper(
encoding::label::encoding_from_whatwg_label(encoding)
.ok_or_else(|| serde::de::Error::custom("Invalid Encoding"))?,
))
}
fn visit_string<E: serde::de::Error>(self, encoding: String) -> Result<EncodingWrapper, E> {
Ok(EncodingWrapper(
encoding::label::encoding_from_whatwg_label(&encoding)
.ok_or_else(|| serde::de::Error::custom("Invalid Encoding"))?,
))
}
}
impl<'de> Deserialize<'de> for EncodingWrapper {
fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
where
D: serde::Deserializer<'de>,
{
deserializer.deserialize_str(EncodingWrapperVisitor)
}
}
impl PartialEq for EncodingWrapper {
fn eq(&self, other: &Self) -> bool {
self.0.name() == other.0.name()
}
}
unsafe impl Send for EncodingWrapper {}
unsafe impl Sync for EncodingWrapper {}
impl Clone for EncodingWrapper {
fn clone(&self) -> Self {
EncodingWrapper(self.0)
}
}
impl EncodingWrapper {
pub fn new(encoding: &'static dyn Encoding) -> EncodingWrapper {
EncodingWrapper(encoding)

33
crates/fs/src/fs.rs
View file

@ -56,6 +56,7 @@ use smol::io::AsyncReadExt;
use std::ffi::OsStr;
use crate::encodings::EncodingWrapper;
use crate::encodings::from_utf8;
pub trait Watcher: Send + Sync {
fn add(&self, path: &Path) -> Result<()>;
@ -123,7 +124,13 @@ pub trait Fs: Send + Sync {
async fn load_bytes(&self, path: &Path) -> Result<Vec<u8>>;
async fn atomic_write(&self, path: PathBuf, text: String) -> Result<()>;
async fn save(&self, path: &Path, text: &Rope, line_ending: LineEnding) -> Result<()>;
async fn save(
&self,
path: &Path,
text: &Rope,
line_ending: LineEnding,
encoding: EncodingWrapper,
) -> Result<()>;
async fn write(&self, path: &Path, content: &[u8]) -> Result<()>;
async fn canonicalize(&self, path: &Path) -> Result<PathBuf>;
async fn is_file(&self, path: &Path) -> bool;
@ -611,7 +618,13 @@ impl Fs for RealFs {
Ok(())
}
async fn save(&self, path: &Path, text: &Rope, line_ending: LineEnding) -> Result<()> {
async fn save(
&self,
path: &Path,
text: &Rope,
line_ending: LineEnding,
encoding: EncodingWrapper,
) -> Result<()> {
let buffer_size = text.summary().len.min(10 * 1024);
if let Some(path) = path.parent() {
self.create_dir(path).await?;
@ -619,7 +632,9 @@ impl Fs for RealFs {
let file = smol::fs::File::create(path).await?;
let mut writer = smol::io::BufWriter::with_capacity(buffer_size, file);
for chunk in chunks(text, line_ending) {
writer.write_all(chunk.as_bytes()).await?;
writer
.write_all(&from_utf8(chunk.to_string(), encoding.clone()).await?)
.await?;
}
writer.flush().await?;
Ok(())
@ -2290,14 +2305,22 @@ impl Fs for FakeFs {
Ok(())
}
async fn save(&self, path: &Path, text: &Rope, line_ending: LineEnding) -> Result<()> {
async fn save(
&self,
path: &Path,
text: &Rope,
line_ending: LineEnding,
encoding: EncodingWrapper,
) -> Result<()> {
use crate::encodings::from_utf8;
self.simulate_random_delay().await;
let path = normalize_path(path);
let content = chunks(text, line_ending).collect::<String>();
if let Some(path) = path.parent() {
self.create_dir(path).await?;
}
self.write_file_internal(path, content.into_bytes(), false)?;
self.write_file_internal(path, from_utf8(content, encoding).await?, false)?;
Ok(())
}

1
crates/git_ui/Cargo.toml
View file

@ -29,6 +29,7 @@ command_palette_hooks.workspace = true
component.workspace = true
db.workspace = true
editor.workspace = true
encoding.workspace = true
futures.workspace = true
fuzzy.workspace = true
git.workspace = true

5
crates/git_ui/src/file_diff_view.rs
View file

@ -362,8 +362,9 @@ impl Render for FileDiffView {
mod tests {
use super::*;
use editor::test::editor_test_context::assert_state_with_diff;
use encoding::all::UTF_8;
use gpui::TestAppContext;
use project::{FakeFs, Fs, Project};
use project::{FakeFs, Fs, Project, encodings::EncodingWrapper};
use settings::{Settings, SettingsStore};
use std::path::PathBuf;
use unindent::unindent;
@ -444,6 +445,7 @@ mod tests {
)
.into(),
Default::default(),
EncodingWrapper::new(UTF_8),
)
.await
.unwrap();
@ -479,6 +481,7 @@ mod tests {
)
.into(),
Default::default(),
EncodingWrapper::new(UTF_8),
)
.await
.unwrap();

4
crates/language/Cargo.toml
View file

@ -31,7 +31,9 @@ anyhow.workspace = true
async-trait.workspace = true
clock.workspace = true
collections.workspace = true
diffy = "0.4.2"
ec4rs.workspace = true
encoding.workspace = true
fs.workspace = true
futures.workspace = true
fuzzy.workspace = true
@ -69,8 +71,6 @@ unicase = "2.6"
util.workspace = true
watch.workspace = true
workspace-hack.workspace = true
diffy = "0.4.2"
encoding = "0.2.33"
[dev-dependencies]
collections = { workspace = true, features = ["test-support"] }

10
crates/language/src/buffer.rs
View file

@ -22,7 +22,7 @@ pub use clock::ReplicaId;
use clock::{AGENT_REPLICA_ID, Lamport};
use collections::HashMap;
use encoding::Encoding;
use fs::{Fs, MTime, RealFs};
use fs::MTime;
use futures::channel::oneshot;
use gpui::{
App, AppContext as _, Context, Entity, EventEmitter, HighlightStyle, SharedString, StyledText,
@ -1281,7 +1281,7 @@ impl Buffer {
/// Reloads the contents of the buffer from disk.
pub fn reload(&mut self, cx: &Context<Self>) -> oneshot::Receiver<Option<Transaction>> {
let (tx, rx) = futures::channel::oneshot::channel();
let encoding = self.encoding.clone();
let encoding = self.encoding;
let prev_version = self.text.version();
self.reload_task = Some(cx.spawn(async move |this, cx| {
let Some((new_mtime, new_text)) = this.update(cx, |this, cx| {
@ -4976,11 +4976,7 @@ impl LocalFile for TestFile {
unimplemented!()
}
fn load_with_encoding(
&self,
cx: &App,
encoding: &'static dyn Encoding,
) -> Task<Result<String>> {
fn load_with_encoding(&self, _: &App, _: &'static dyn Encoding) -> Task<Result<String>> {
unimplemented!()
}
}

2
crates/project/Cargo.toml
View file

@ -39,6 +39,7 @@ clock.workspace = true
collections.workspace = true
context_server.workspace = true
dap.workspace = true
encoding.workspace = true
extension.workspace = true
fancy-regex.workspace = true
fs.workspace = true
@ -90,6 +91,7 @@ worktree.workspace = true
zlog.workspace = true
workspace-hack.workspace = true
[dev-dependencies]
client = { workspace = true, features = ["test-support"] }
collections = { workspace = true, features = ["test-support"] }

4
crates/project/src/buffer_store.rs
View file

@ -372,6 +372,8 @@ impl LocalBufferStore {
let version = buffer.version();
let buffer_id = buffer.remote_id();
let file = buffer.file().cloned();
let encoding = buffer.encoding;
if file
.as_ref()
.is_some_and(|file| file.disk_state() == DiskState::New)
@ -380,7 +382,7 @@ impl LocalBufferStore {
}
let save = worktree.update(cx, |worktree, cx| {
worktree.write_file(path.as_ref(), text, line_ending, cx)
worktree.write_file(path.as_ref(), text, line_ending, cx, encoding)
});
cx.spawn(async move |this, cx| {

5
crates/project/src/prettier_store.rs
View file

@ -7,7 +7,8 @@ use std::{
use anyhow::{Context as _, Result, anyhow};
use collections::{HashMap, HashSet};
use fs::Fs;
use encoding::all::UTF_8;
use fs::{Fs, encodings::EncodingWrapper};
use futures::{
FutureExt,
future::{self, Shared},
@ -941,10 +942,12 @@ async fn install_prettier_packages(
async fn save_prettier_server_file(fs: &dyn Fs) -> anyhow::Result<()> {
let prettier_wrapper_path = default_prettier_dir().join(prettier::PRETTIER_SERVER_FILE);
let encoding_wrapper = EncodingWrapper::new(UTF_8);
fs.save(
&prettier_wrapper_path,
&text::Rope::from(prettier::PRETTIER_SERVER_JS),
text::LineEnding::Unix,
encoding_wrapper,
)
.await
.with_context(|| {

26
crates/project/src/project_tests.rs
View file

@ -9,7 +9,8 @@ use buffer_diff::{
BufferDiffEvent, CALCULATE_DIFF_TASK, DiffHunkSecondaryStatus, DiffHunkStatus,
DiffHunkStatusKind, assert_hunks,
};
use fs::FakeFs;
use encoding::all::UTF_8;
use fs::{FakeFs, encodings::EncodingWrapper};
use futures::{StreamExt, future};
use git::{
GitHostingProviderRegistry,
@ -1448,10 +1449,14 @@ async fn test_reporting_fs_changes_to_language_servers(cx: &mut gpui::TestAppCon
)
.await
.unwrap();
let encoding_wrapper = EncodingWrapper::new(UTF_8);
fs.save(
path!("/the-root/Cargo.lock").as_ref(),
&"".into(),
Default::default(),
encoding_wrapper.clone(),
)
.await
.unwrap();
@ -1459,6 +1464,7 @@ async fn test_reporting_fs_changes_to_language_servers(cx: &mut gpui::TestAppCon
path!("/the-stdlib/LICENSE").as_ref(),
&"".into(),
Default::default(),
encoding_wrapper.clone(),
)
.await
.unwrap();
@ -1466,6 +1472,7 @@ async fn test_reporting_fs_changes_to_language_servers(cx: &mut gpui::TestAppCon
path!("/the/stdlib/src/string.rs").as_ref(),
&"".into(),
Default::default(),
encoding_wrapper,
)
.await
.unwrap();
@ -3941,12 +3948,15 @@ async fn test_file_changes_multiple_times_on_disk(cx: &mut gpui::TestAppContext)
// the next file change occurs.
cx.executor().deprioritize(*language::BUFFER_DIFF_TASK);
let encoding_wrapper = EncodingWrapper::new(UTF_8);
// Change the buffer's file on disk, and then wait for the file change
// to be detected by the worktree, so that the buffer starts reloading.
fs.save(
path!("/dir/file1").as_ref(),
&"the first contents".into(),
Default::default(),
encoding_wrapper.clone(),
)
.await
.unwrap();
@ -3958,6 +3968,7 @@ async fn test_file_changes_multiple_times_on_disk(cx: &mut gpui::TestAppContext)
path!("/dir/file1").as_ref(),
&"the second contents".into(),
Default::default(),
encoding_wrapper,
)
.await
.unwrap();
@ -3996,12 +4007,15 @@ async fn test_edit_buffer_while_it_reloads(cx: &mut gpui::TestAppContext) {
// the next file change occurs.
cx.executor().deprioritize(*language::BUFFER_DIFF_TASK);
let encoding_wrapper = EncodingWrapper::new(UTF_8);
// Change the buffer's file on disk, and then wait for the file change
// to be detected by the worktree, so that the buffer starts reloading.
fs.save(
path!("/dir/file1").as_ref(),
&"the first contents".into(),
Default::default(),
encoding_wrapper,
)
.await
.unwrap();
@ -4603,10 +4617,14 @@ async fn test_buffer_file_changes_on_disk(cx: &mut gpui::TestAppContext) {
let (new_contents, new_offsets) =
marked_text_offsets("oneˇ\nthree ˇFOURˇ five\nsixtyˇ seven\n");
let encoding_wrapper = EncodingWrapper::new(UTF_8);
fs.save(
path!("/dir/the-file").as_ref(),
&new_contents.as_str().into(),
LineEnding::Unix,
encoding_wrapper,
)
.await
.unwrap();
@ -4634,11 +4652,14 @@ async fn test_buffer_file_changes_on_disk(cx: &mut gpui::TestAppContext) {
assert!(!buffer.has_conflict());
});
let encoding_wrapper = EncodingWrapper::new(UTF_8);
// Change the file on disk again, adding blank lines to the beginning.
fs.save(
path!("/dir/the-file").as_ref(),
&"\n\n\nAAAA\naaa\nBB\nbbbbb\n".into(),
LineEnding::Unix,
encoding_wrapper,
)
.await
.unwrap();
@ -4685,12 +4706,15 @@ async fn test_buffer_line_endings(cx: &mut gpui::TestAppContext) {
assert_eq!(buffer.line_ending(), LineEnding::Windows);
});
let encoding_wrapper = EncodingWrapper::new(UTF_8);
// Change a file's line endings on disk from unix to windows. The buffer's
// state updates correctly.
fs.save(
path!("/dir/file1").as_ref(),
&"aaa\nb\nc\n".into(),
LineEnding::Windows,
encoding_wrapper,
)
.await
.unwrap();

1
crates/remote_server/Cargo.toml
View file

@ -30,6 +30,7 @@ clap.workspace = true
client.workspace = true
dap_adapters.workspace = true
debug_adapter_extension.workspace = true
encoding.workspace = true
env_logger.workspace = true
extension.workspace = true
extension_host.workspace = true

6
crates/remote_server/src/remote_editing_tests.rs
View file

@ -6,10 +6,11 @@ use assistant_tool::{Tool as _, ToolResultContent};
use assistant_tools::{ReadFileTool, ReadFileToolInput};
use client::{Client, UserStore};
use clock::FakeSystemClock;
use encoding::all::UTF_8;
use language_model::{LanguageModelRequest, fake_provider::FakeLanguageModel};
use extension::ExtensionHostProxy;
use fs::{FakeFs, Fs};
use fs::{FakeFs, Fs, encodings::EncodingWrapper};
use gpui::{AppContext as _, Entity, SemanticVersion, TestAppContext};
use http_client::{BlockedHttpClient, FakeHttpClient};
use language::{
@ -123,6 +124,7 @@ async fn test_basic_remote_editing(cx: &mut TestAppContext, server_cx: &mut Test
path!("/code/project1/src/main.rs").as_ref(),
&"fn main() {}".into(),
Default::default(),
EncodingWrapper::new(UTF_8),
)
.await
.unwrap();
@ -763,6 +765,7 @@ async fn test_remote_reload(cx: &mut TestAppContext, server_cx: &mut TestAppCont
&PathBuf::from(path!("/code/project1/src/lib.rs")),
&("bangles".to_string().into()),
LineEnding::Unix,
EncodingWrapper::new(UTF_8),
)
.await
.unwrap();
@ -778,6 +781,7 @@ async fn test_remote_reload(cx: &mut TestAppContext, server_cx: &mut TestAppCont
&PathBuf::from(path!("/code/project1/src/lib.rs")),
&("bloop".to_string().into()),
LineEnding::Unix,
EncodingWrapper::new(UTF_8),
)
.await
.unwrap();

1
crates/vim/Cargo.toml
View file

@ -52,6 +52,7 @@ workspace.workspace = true
zed_actions.workspace = true
workspace-hack.workspace = true
[dev-dependencies]
assets.workspace = true
command_palette.workspace = true

1
crates/workspace/Cargo.toml
View file

@ -35,6 +35,7 @@ clock.workspace = true
collections.workspace = true
component.workspace = true
db.workspace = true
encoding.workspace = true
fs.workspace = true
futures.workspace = true
gpui.workspace = true

11
crates/workspace/src/workspace.rs
View file

@ -19,6 +19,8 @@ mod workspace_settings;
pub use crate::notifications::NotificationFrame;
pub use dock::Panel;
use encoding::all::UTF_8;
use fs::encodings::EncodingWrapper;
pub use path_list::PathList;
pub use toast_layer::{ToastAction, ToastLayer, ToastView};
@ -118,7 +120,6 @@ use crate::persistence::{
model::{DockData, DockStructure, SerializedItem, SerializedPane, SerializedPaneGroup},
};
pub const SERIALIZATION_THROTTLE_TIME: Duration = Duration::from_millis(200);
static ZED_WINDOW_SIZE: LazyLock<Option<Size<Pixels>>> = LazyLock::new(|| {
@ -7233,7 +7234,13 @@ pub fn create_and_open_local_file(
let fs = workspace.read_with(cx, |workspace, _| workspace.app_state().fs.clone())?;
if !fs.is_file(path).await {
fs.create_file(path, Default::default()).await?;
fs.save(path, &default_content(), Default::default())
let encoding_wrapper = EncodingWrapper::new(UTF_8);
fs.save(
path,
&default_content(),
Default::default(),
encoding_wrapper,
)
.await?;
}

2
crates/worktree/Cargo.toml
View file

@ -26,6 +26,7 @@ test-support = [
anyhow.workspace = true
clock.workspace = true
collections.workspace = true
encoding.workspace = true
fs.workspace = true
futures.workspace = true
fuzzy.workspace = true
@ -48,7 +49,6 @@ sum_tree.workspace = true
text.workspace = true
util.workspace = true
workspace-hack.workspace = true
encoding = "0.2.33"
[dev-dependencies]

11
crates/worktree/src/worktree.rs
View file

@ -835,9 +835,10 @@ impl Worktree {
text: Rope,
line_ending: LineEnding,
cx: &Context<Worktree>,
encoding: &'static dyn Encoding,
) -> Task<Result<Arc<File>>> {
match self {
Worktree::Local(this) => this.write_file(path, text, line_ending, cx),
Worktree::Local(this) => this.write_file(path, text, line_ending, cx, encoding),
Worktree::Remote(_) => {
Task::ready(Err(anyhow!("remote worktree can't yet write files")))
}
@ -1648,6 +1649,7 @@ impl LocalWorktree {
text: Rope,
line_ending: LineEnding,
cx: &Context<Worktree>,
encoding: &'static dyn Encoding,
) -> Task<Result<Arc<File>>> {
let path = path.into();
let fs = self.fs.clone();
@ -1656,10 +1658,15 @@ impl LocalWorktree {
return Task::ready(Err(anyhow!("invalid path {path:?}")));
};
let encoding_wrapper = EncodingWrapper::new(encoding);
let write = cx.background_spawn({
let fs = fs.clone();
let abs_path = abs_path.clone();
async move { fs.save(&abs_path, &text, line_ending).await }
async move {
fs.save(&abs_path, &text, line_ending, encoding_wrapper)
.await
}
});
cx.spawn(async move |this, cx| {

24
crates/worktree/src/worktree_tests.rs
View file

@ -3,7 +3,8 @@ use crate::{
worktree_settings::WorktreeSettings,
};
use anyhow::Result;
use fs::{FakeFs, Fs, RealFs, RemoveOptions};
use encoding::all::UTF_8;
use fs::{FakeFs, Fs, RealFs, RemoveOptions, encodings::EncodingWrapper};
use git::GITIGNORE;
use gpui::{AppContext as _, BackgroundExecutor, BorrowAppContext, Context, Task, TestAppContext};
use parking_lot::Mutex;
@ -642,7 +643,13 @@ async fn test_dirs_no_longer_ignored(cx: &mut TestAppContext) {
// Update the gitignore so that node_modules is no longer ignored,
// but a subdirectory is ignored
fs.save("/root/.gitignore".as_ref(), &"e".into(), Default::default())
let encoding_wrapper = fs::encodings::EncodingWrapper::new(UTF_8);
fs.save(
"/root/.gitignore".as_ref(),
&"e".into(),
Default::default(),
encoding_wrapper,
)
.await
.unwrap();
cx.executor().run_until_parked();
@ -715,6 +722,7 @@ async fn test_write_file(cx: &mut TestAppContext) {
"hello".into(),
Default::default(),
cx,
UTF_8,
)
})
.await
@ -726,6 +734,7 @@ async fn test_write_file(cx: &mut TestAppContext) {
"world".into(),
Default::default(),
cx,
UTF_8,
)
})
.await
@ -1746,8 +1755,13 @@ fn randomly_mutate_worktree(
})
} else {
log::info!("overwriting file {:?} ({})", entry.path, entry.id.0);
let task =
worktree.write_file(entry.path.clone(), "".into(), Default::default(), cx);
let task = worktree.write_file(
entry.path.clone(),
"".into(),
Default::default(),
cx,
UTF_8,
);
cx.background_spawn(async move {
task.await?;
Ok(())
@ -1834,10 +1848,12 @@ async fn randomly_mutate_fs(
ignore_path.strip_prefix(root_path).unwrap(),
ignore_contents
);
let encoding_wrapper = EncodingWrapper::new(UTF_8);
fs.save(
&ignore_path,
&ignore_contents.as_str().into(),
Default::default(),
encoding_wrapper,
)
.await
.unwrap();

2
crates/zed/Cargo.toml
View file

@ -55,6 +55,7 @@ debugger_tools.workspace = true
debugger_ui.workspace = true
diagnostics.workspace = true
editor.workspace = true
encoding.workspace = true
encodings.workspace = true
env_logger.workspace = true
extension.workspace = true
@ -168,7 +169,6 @@ zed_actions.workspace = true
zeta.workspace = true
zlog.workspace = true
zlog_settings.workspace = true
encoding = "0.2.33"
[target.'cfg(target_os = "windows")'.dependencies]
windows.workspace = true

10
crates/zed/src/zed.rs
View file

@ -1937,6 +1937,8 @@ mod tests {
use assets::Assets;
use collections::HashSet;
use editor::{DisplayPoint, Editor, SelectionEffects, display_map::DisplayRow};
use encoding::all::UTF_8;
use fs::encodings::EncodingWrapper;
use gpui::{
Action, AnyWindowHandle, App, AssetSource, BorrowAppContext, SemanticVersion,
TestAppContext, UpdateGlobal, VisualTestContext, WindowHandle, actions,
@ -4128,6 +4130,7 @@ mod tests {
"/settings.json".as_ref(),
&r#"{"base_keymap": "Atom"}"#.into(),
Default::default(),
EncodingWrapper::new(UTF_8),
)
.await
.unwrap();
@ -4138,6 +4141,7 @@ mod tests {
"/keymap.json".as_ref(),
&r#"[{"bindings": {"backspace": "test_only::ActionA"}}]"#.into(),
Default::default(),
EncodingWrapper::new(UTF_8),
)
.await
.unwrap();
@ -4186,6 +4190,7 @@ mod tests {
"/keymap.json".as_ref(),
&r#"[{"bindings": {"backspace": "test_only::ActionB"}}]"#.into(),
Default::default(),
EncodingWrapper::new(UTF_8),
)
.await
.unwrap();
@ -4206,6 +4211,7 @@ mod tests {
"/settings.json".as_ref(),
&r#"{"base_keymap": "JetBrains"}"#.into(),
Default::default(),
EncodingWrapper::new(UTF_8),
)
.await
.unwrap();
@ -4246,6 +4252,7 @@ mod tests {
"/settings.json".as_ref(),
&r#"{"base_keymap": "Atom"}"#.into(),
Default::default(),
EncodingWrapper::new(UTF_8),
)
.await
.unwrap();
@ -4255,6 +4262,7 @@ mod tests {
"/keymap.json".as_ref(),
&r#"[{"bindings": {"backspace": "test_only::ActionA"}}]"#.into(),
Default::default(),
EncodingWrapper::new(UTF_8),
)
.await
.unwrap();
@ -4298,6 +4306,7 @@ mod tests {
"/keymap.json".as_ref(),
&r#"[{"bindings": {"backspace": null}}]"#.into(),
Default::default(),
EncodingWrapper::new(UTF_8),
)
.await
.unwrap();
@ -4318,6 +4327,7 @@ mod tests {
"/settings.json".as_ref(),
&r#"{"base_keymap": "JetBrains"}"#.into(),
Default::default(),
EncodingWrapper::new(UTF_8),
)
.await
.unwrap();