Fix some semantic index issues (#11216)

* [x] Fixed an issue where embeddings would be assigned incorrectly to
files if a subset of embedding batches failed
* [x] Added a command to debug which paths are present in the semantic
index
* [x] Determine why so many paths are often missing from the semantic
index
* we erroring out if an embedding batch contained multiple texts that
were the same, which can happen if a worktree contains multiple copies
of the same text (e.g. a license).

Release Notes:

- N/A

---------

Co-authored-by: Marshall <marshall@zed.dev>
Co-authored-by: Nathan <nathan@zed.dev>
Co-authored-by: Kyle <kylek@zed.dev>
Co-authored-by: Kyle Kelley <rgbkrk@gmail.com>

crates/assistant2/examples/chat_with_functions.rs

@@ -365,7 +365,7 @@ impl Example {
     ) -> Self {
         Self {
             assistant_panel: cx.new_view(|cx| {
-                AssistantPanel::new(language_registry, tool_registry, user_store, cx)
+                AssistantPanel::new(language_registry, tool_registry, user_store, None, cx)
             }),
         }
     }

crates/assistant2/src/assistant2.rs

@@ -19,7 +19,7 @@ use gpui::{
 use language::{language_settings::SoftWrap, LanguageRegistry};
 use open_ai::{FunctionContent, ToolCall, ToolCallContent};
 use rich_text::RichText;
-use semantic_index::{CloudEmbeddingProvider, SemanticIndex};
+use semantic_index::{CloudEmbeddingProvider, ProjectIndex, SemanticIndex};
 use serde::Deserialize;
 use settings::Settings;
 use std::sync::Arc;
@@ -51,7 +51,7 @@ pub enum SubmitMode {
     Codebase,
 }
 
-gpui::actions!(assistant2, [Cancel, ToggleFocus]);
+gpui::actions!(assistant2, [Cancel, ToggleFocus, DebugProjectIndex]);
 gpui::impl_actions!(assistant2, [Submit]);
 
 pub fn init(client: Arc<Client>, cx: &mut AppContext) {
@@ -131,7 +131,13 @@ impl AssistantPanel {
 
                 let tool_registry = Arc::new(tool_registry);
 
-                Self::new(app_state.languages.clone(), tool_registry, user_store, cx)
+                Self::new(
+                    app_state.languages.clone(),
+                    tool_registry,
+                    user_store,
+                    Some(project_index),
+                    cx,
+                )
             })
         })
     }
@@ -140,6 +146,7 @@ impl AssistantPanel {
         language_registry: Arc<LanguageRegistry>,
         tool_registry: Arc<ToolRegistry>,
         user_store: Model<UserStore>,
+        project_index: Option<Model<ProjectIndex>>,
         cx: &mut ViewContext<Self>,
     ) -> Self {
         let chat = cx.new_view(|cx| {
@@ -147,6 +154,7 @@ impl AssistantPanel {
                 language_registry.clone(),
                 tool_registry.clone(),
                 user_store,
+                project_index,
                 cx,
             )
         });
@@ -225,6 +233,7 @@ struct AssistantChat {
     collapsed_messages: HashMap<MessageId, bool>,
     pending_completion: Option<Task<()>>,
     tool_registry: Arc<ToolRegistry>,
+    project_index: Option<Model<ProjectIndex>>,
 }
 
 impl AssistantChat {
@@ -232,6 +241,7 @@ impl AssistantChat {
         language_registry: Arc<LanguageRegistry>,
         tool_registry: Arc<ToolRegistry>,
         user_store: Model<UserStore>,
+        project_index: Option<Model<ProjectIndex>>,
         cx: &mut ViewContext<Self>,
     ) -> Self {
         let model = CompletionProvider::get(cx).default_model();
@@ -258,6 +268,7 @@ impl AssistantChat {
             list_state,
             user_store,
             language_registry,
+            project_index,
             next_message_id: MessageId(0),
             collapsed_messages: HashMap::default(),
             pending_completion: None,
@@ -342,6 +353,14 @@ impl AssistantChat {
         self.pending_completion.is_none()
     }
 
+    fn debug_project_index(&mut self, _: &DebugProjectIndex, cx: &mut ViewContext<Self>) {
+        if let Some(index) = &self.project_index {
+            index.update(cx, |project_index, cx| {
+                project_index.debug(cx).detach_and_log_err(cx)
+            });
+        }
+    }
+
     async fn request_completion(
         this: WeakView<Self>,
         mode: SubmitMode,
@@ -686,6 +705,7 @@ impl Render for AssistantChat {
             .key_context("AssistantChat")
             .on_action(cx.listener(Self::submit))
             .on_action(cx.listener(Self::cancel))
+            .on_action(cx.listener(Self::debug_project_index))
             .text_color(Color::Default.color(cx))
             .child(list(self.list_state.clone()).flex_1())
             .child(Composer::new(