assistant: Stream tool uses as structured data (#17322)

This PR adjusts the approach we use to encoding tool uses in the
completion response to use a structured format rather than simply
injecting it into the response stream as text.

In #17170 we would encode the tool uses as XML and insert them as text.
This would require then re-parsing the tool uses out of the buffer in
order to use them.

The approach taken in this PR is to make `stream_completion` return a
stream of `LanguageModelCompletionEvent`s. Each of these events can be
either text, or a tool use.

A new `stream_completion_text` method has been added to `LanguageModel`
for scenarios where we only care about textual content (currently,
everywhere that isn't the Assistant context editor).

Release Notes:

- N/A

crates/language_model/src/provider/anthropic.rs

@@ -3,6 +3,7 @@ use crate::{
     LanguageModelId, LanguageModelName, LanguageModelProvider, LanguageModelProviderId,
     LanguageModelProviderName, LanguageModelProviderState, LanguageModelRequest, RateLimiter, Role,
 };
+use crate::{LanguageModelCompletionEvent, LanguageModelToolUse};
 use anthropic::AnthropicError;
 use anyhow::{anyhow, Context as _, Result};
 use collections::BTreeMap;
@@ -364,7 +365,7 @@ impl LanguageModel for AnthropicModel {
         &self,
         request: LanguageModelRequest,
         cx: &AsyncAppContext,
-    ) -> BoxFuture<'static, Result<BoxStream<'static, Result<String>>>> {
+    ) -> BoxFuture<'static, Result<BoxStream<'static, Result<LanguageModelCompletionEvent>>>> {
         let request =
             request.into_anthropic(self.model.id().into(), self.model.max_output_tokens());
         let request = self.stream_completion(request, cx);
@@ -375,7 +376,22 @@ impl LanguageModel for AnthropicModel {
         async move {
             Ok(future
                 .await?
-                .map(|result| result.map_err(|err| anyhow!(err)))
+                .map(|result| {
+                    result
+                        .map(|content| match content {
+                            anthropic::ResponseContent::Text { text } => {
+                                LanguageModelCompletionEvent::Text(text)
+                            }
+                            anthropic::ResponseContent::ToolUse { id, name, input } => {
+                                LanguageModelCompletionEvent::ToolUse(LanguageModelToolUse {
+                                    id,
+                                    name,
+                                    input,
+                                })
+                            }
+                        })
+                        .map_err(|err| anyhow!(err))
+                })
                 .boxed())
         }
         .boxed()