Introduce rating for assistant threads (#26780)
Release Notes: - N/A --------- Co-authored-by: Richard Feldman <oss@rtfeldman.com> Co-authored-by: Agus Zubiaga <hi@aguz.me>
This commit is contained in:
Antonio Scandurra
GitHub
parent
c62210b178
commit
f68a475eca
12 changed files with 378 additions and 117 deletions
|
|
@ -5,7 +5,9 @@ use anyhow::{Context as _, Result};
|
|||
use assistant_tool::ToolWorkingSet;
|
||||
use chrono::{DateTime, Utc};
|
||||
use collections::{BTreeMap, HashMap, HashSet};
|
||||
use futures::StreamExt as _;
|
||||
use futures::future::Shared;
|
||||
use futures::{FutureExt, StreamExt as _};
|
||||
use git;
|
||||
use gpui::{App, AppContext, Context, Entity, EventEmitter, SharedString, Task};
|
||||
use language_model::{
|
||||
LanguageModel, LanguageModelCompletionEvent, LanguageModelRegistry, LanguageModelRequest,
|
||||
|
|
@ -21,7 +23,9 @@ use util::{post_inc, ResultExt, TryFutureExt as _};
|
|||
use uuid::Uuid;
|
||||
|
||||
use crate::context::{attach_context_to_message, ContextId, ContextSnapshot};
|
||||
use crate::thread_store::SavedThread;
|
||||
use crate::thread_store::{
|
||||
SerializedMessage, SerializedThread, SerializedToolResult, SerializedToolUse,
|
||||
};
|
||||
use crate::tool_use::{PendingToolUse, ToolUse, ToolUseState};
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
|
|
@ -63,6 +67,27 @@ pub struct Message {
|
|||
pub text: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ProjectSnapshot {
|
||||
pub worktree_snapshots: Vec<WorktreeSnapshot>,
|
||||
pub unsaved_buffer_paths: Vec<String>,
|
||||
pub timestamp: DateTime<Utc>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct WorktreeSnapshot {
|
||||
pub worktree_path: String,
|
||||
pub git_state: Option<GitState>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct GitState {
|
||||
pub remote_url: Option<String>,
|
||||
pub head_sha: Option<String>,
|
||||
pub current_branch: Option<String>,
|
||||
pub diff: Option<String>,
|
||||
}
|
||||
|
||||
/// A thread of conversation with the LLM.
|
||||
pub struct Thread {
|
||||
id: ThreadId,
|
||||
|
|
@ -81,6 +106,7 @@ pub struct Thread {
|
|||
tool_use: ToolUseState,
|
||||
scripting_session: Entity<ScriptingSession>,
|
||||
scripting_tool_use: ToolUseState,
|
||||
initial_project_snapshot: Shared<Task<Option<Arc<ProjectSnapshot>>>>,
|
||||
cumulative_token_usage: TokenUsage,
|
||||
}
|
||||
|
||||
|
|
@ -91,8 +117,6 @@ impl Thread {
|
|||
prompt_builder: Arc<PromptBuilder>,
|
||||
cx: &mut Context<Self>,
|
||||
) -> Self {
|
||||
let scripting_session = cx.new(|cx| ScriptingSession::new(project.clone(), cx));
|
||||
|
||||
Self {
|
||||
id: ThreadId::new(),
|
||||
updated_at: Utc::now(),
|
||||
|
|
@ -104,43 +128,52 @@ impl Thread {
|
|||
context_by_message: HashMap::default(),
|
||||
completion_count: 0,
|
||||
pending_completions: Vec::new(),
|
||||
project,
|
||||
project: project.clone(),
|
||||
prompt_builder,
|
||||
tools,
|
||||
tool_use: ToolUseState::new(),
|
||||
scripting_session,
|
||||
scripting_session: cx.new(|cx| ScriptingSession::new(project.clone(), cx)),
|
||||
scripting_tool_use: ToolUseState::new(),
|
||||
initial_project_snapshot: {
|
||||
let project_snapshot = Self::project_snapshot(project, cx);
|
||||
cx.foreground_executor()
|
||||
.spawn(async move { Some(project_snapshot.await) })
|
||||
.shared()
|
||||
},
|
||||
cumulative_token_usage: TokenUsage::default(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn from_saved(
|
||||
pub fn deserialize(
|
||||
id: ThreadId,
|
||||
saved: SavedThread,
|
||||
serialized: SerializedThread,
|
||||
project: Entity<Project>,
|
||||
tools: Arc<ToolWorkingSet>,
|
||||
prompt_builder: Arc<PromptBuilder>,
|
||||
cx: &mut Context<Self>,
|
||||
) -> Self {
|
||||
let next_message_id = MessageId(
|
||||
saved
|
||||
serialized
|
||||
.messages
|
||||
.last()
|
||||
.map(|message| message.id.0 + 1)
|
||||
.unwrap_or(0),
|
||||
);
|
||||
let tool_use =
|
||||
ToolUseState::from_saved_messages(&saved.messages, |name| name != ScriptingTool::NAME);
|
||||
let tool_use = ToolUseState::from_serialized_messages(&serialized.messages, |name| {
|
||||
name != ScriptingTool::NAME
|
||||
});
|
||||
let scripting_tool_use =
|
||||
ToolUseState::from_saved_messages(&saved.messages, |name| name == ScriptingTool::NAME);
|
||||
ToolUseState::from_serialized_messages(&serialized.messages, |name| {
|
||||
name == ScriptingTool::NAME
|
||||
});
|
||||
let scripting_session = cx.new(|cx| ScriptingSession::new(project.clone(), cx));
|
||||
|
||||
Self {
|
||||
id,
|
||||
updated_at: saved.updated_at,
|
||||
summary: Some(saved.summary),
|
||||
updated_at: serialized.updated_at,
|
||||
summary: Some(serialized.summary),
|
||||
pending_summary: Task::ready(None),
|
||||
messages: saved
|
||||
messages: serialized
|
||||
.messages
|
||||
.into_iter()
|
||||
.map(|message| Message {
|
||||
|
|
@ -160,6 +193,7 @@ impl Thread {
|
|||
tool_use,
|
||||
scripting_session,
|
||||
scripting_tool_use,
|
||||
initial_project_snapshot: Task::ready(serialized.initial_project_snapshot).shared(),
|
||||
// TODO: persist token usage?
|
||||
cumulative_token_usage: TokenUsage::default(),
|
||||
}
|
||||
|
|
@ -349,6 +383,47 @@ impl Thread {
|
|||
text
|
||||
}
|
||||
|
||||
/// Serializes this thread into a format for storage or telemetry.
|
||||
pub fn serialize(&self, cx: &mut Context<Self>) -> Task<Result<SerializedThread>> {
|
||||
let initial_project_snapshot = self.initial_project_snapshot.clone();
|
||||
cx.spawn(|this, cx| async move {
|
||||
let initial_project_snapshot = initial_project_snapshot.await;
|
||||
this.read_with(&cx, |this, _| SerializedThread {
|
||||
summary: this.summary_or_default(),
|
||||
updated_at: this.updated_at(),
|
||||
messages: this
|
||||
.messages()
|
||||
.map(|message| SerializedMessage {
|
||||
id: message.id,
|
||||
role: message.role,
|
||||
text: message.text.clone(),
|
||||
tool_uses: this
|
||||
.tool_uses_for_message(message.id)
|
||||
.into_iter()
|
||||
.chain(this.scripting_tool_uses_for_message(message.id))
|
||||
.map(|tool_use| SerializedToolUse {
|
||||
id: tool_use.id,
|
||||
name: tool_use.name,
|
||||
input: tool_use.input,
|
||||
})
|
||||
.collect(),
|
||||
tool_results: this
|
||||
.tool_results_for_message(message.id)
|
||||
.into_iter()
|
||||
.chain(this.scripting_tool_results_for_message(message.id))
|
||||
.map(|tool_result| SerializedToolResult {
|
||||
tool_use_id: tool_result.tool_use_id.clone(),
|
||||
is_error: tool_result.is_error,
|
||||
content: tool_result.content.clone(),
|
||||
})
|
||||
.collect(),
|
||||
})
|
||||
.collect(),
|
||||
initial_project_snapshot,
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
pub fn send_to_model(
|
||||
&mut self,
|
||||
model: Arc<dyn LanguageModel>,
|
||||
|
|
@ -807,6 +882,133 @@ impl Thread {
|
|||
}
|
||||
}
|
||||
|
||||
/// Reports feedback about the thread and stores it in our telemetry backend.
|
||||
pub fn report_feedback(&self, is_positive: bool, cx: &mut Context<Self>) -> Task<Result<()>> {
|
||||
let final_project_snapshot = Self::project_snapshot(self.project.clone(), cx);
|
||||
let serialized_thread = self.serialize(cx);
|
||||
let thread_id = self.id().clone();
|
||||
let client = self.project.read(cx).client();
|
||||
|
||||
cx.background_spawn(async move {
|
||||
let final_project_snapshot = final_project_snapshot.await;
|
||||
let serialized_thread = serialized_thread.await?;
|
||||
let thread_data =
|
||||
serde_json::to_value(serialized_thread).unwrap_or_else(|_| serde_json::Value::Null);
|
||||
|
||||
let rating = if is_positive { "positive" } else { "negative" };
|
||||
telemetry::event!(
|
||||
"Assistant Thread Rated",
|
||||
rating,
|
||||
thread_id,
|
||||
thread_data,
|
||||
final_project_snapshot
|
||||
);
|
||||
client.telemetry().flush_events();
|
||||
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
|
||||
/// Create a snapshot of the current project state including git information and unsaved buffers.
|
||||
fn project_snapshot(
|
||||
project: Entity<Project>,
|
||||
cx: &mut Context<Self>,
|
||||
) -> Task<Arc<ProjectSnapshot>> {
|
||||
let worktree_snapshots: Vec<_> = project
|
||||
.read(cx)
|
||||
.visible_worktrees(cx)
|
||||
.map(|worktree| Self::worktree_snapshot(worktree, cx))
|
||||
.collect();
|
||||
|
||||
cx.spawn(move |_, cx| async move {
|
||||
let worktree_snapshots = futures::future::join_all(worktree_snapshots).await;
|
||||
|
||||
let mut unsaved_buffers = Vec::new();
|
||||
cx.update(|app_cx| {
|
||||
let buffer_store = project.read(app_cx).buffer_store();
|
||||
for buffer_handle in buffer_store.read(app_cx).buffers() {
|
||||
let buffer = buffer_handle.read(app_cx);
|
||||
if buffer.is_dirty() {
|
||||
if let Some(file) = buffer.file() {
|
||||
let path = file.path().to_string_lossy().to_string();
|
||||
unsaved_buffers.push(path);
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
.ok();
|
||||
|
||||
Arc::new(ProjectSnapshot {
|
||||
worktree_snapshots,
|
||||
unsaved_buffer_paths: unsaved_buffers,
|
||||
timestamp: Utc::now(),
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
fn worktree_snapshot(worktree: Entity<project::Worktree>, cx: &App) -> Task<WorktreeSnapshot> {
|
||||
cx.spawn(move |cx| async move {
|
||||
// Get worktree path and snapshot
|
||||
let worktree_info = cx.update(|app_cx| {
|
||||
let worktree = worktree.read(app_cx);
|
||||
let path = worktree.abs_path().to_string_lossy().to_string();
|
||||
let snapshot = worktree.snapshot();
|
||||
(path, snapshot)
|
||||
});
|
||||
|
||||
let Ok((worktree_path, snapshot)) = worktree_info else {
|
||||
return WorktreeSnapshot {
|
||||
worktree_path: String::new(),
|
||||
git_state: None,
|
||||
};
|
||||
};
|
||||
|
||||
// Extract git information
|
||||
let git_state = match snapshot.repositories().first() {
|
||||
None => None,
|
||||
Some(repo_entry) => {
|
||||
// Get branch information
|
||||
let current_branch = repo_entry.branch().map(|branch| branch.name.to_string());
|
||||
|
||||
// Get repository info
|
||||
let repo_result = worktree.read_with(&cx, |worktree, _cx| {
|
||||
if let project::Worktree::Local(local_worktree) = &worktree {
|
||||
local_worktree.get_local_repo(repo_entry).map(|local_repo| {
|
||||
let repo = local_repo.repo();
|
||||
(repo.remote_url("origin"), repo.head_sha(), repo.clone())
|
||||
})
|
||||
} else {
|
||||
None
|
||||
}
|
||||
});
|
||||
|
||||
match repo_result {
|
||||
Ok(Some((remote_url, head_sha, repository))) => {
|
||||
// Get diff asynchronously
|
||||
let diff = repository
|
||||
.diff(git::repository::DiffType::HeadToWorktree, cx)
|
||||
.await
|
||||
.ok();
|
||||
|
||||
Some(GitState {
|
||||
remote_url,
|
||||
head_sha,
|
||||
current_branch,
|
||||
diff,
|
||||
})
|
||||
}
|
||||
Err(_) | Ok(None) => None,
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
WorktreeSnapshot {
|
||||
worktree_path,
|
||||
git_state,
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
pub fn to_markdown(&self) -> Result<String> {
|
||||
let mut markdown = Vec::new();
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue