move queuing to embedding_queue functionality and update embedding provider to include trait items for max tokens per batch"

Co-authored-by: Max <max@zed.dev>

crates/util/src/util.rs

@@ -260,11 +260,22 @@ pub fn defer<F: FnOnce()>(f: F) -> impl Drop {
     Defer(Some(f))
 }
 
-pub struct RandomCharIter<T: Rng>(T);
+pub struct RandomCharIter<T: Rng> {
+    rng: T,
+    simple_text: bool,
+}
 
 impl<T: Rng> RandomCharIter<T> {
     pub fn new(rng: T) -> Self {
-        Self(rng)
+        Self {
+            rng,
+            simple_text: std::env::var("SIMPLE_TEXT").map_or(false, |v| !v.is_empty()),
+        }
+    }
+
+    pub fn with_simple_text(mut self) -> Self {
+        self.simple_text = true;
+        self
     }
 }
 
@@ -272,25 +283,27 @@ impl<T: Rng> Iterator for RandomCharIter<T> {
     type Item = char;
 
     fn next(&mut self) -> Option<Self::Item> {
-        if std::env::var("SIMPLE_TEXT").map_or(false, |v| !v.is_empty()) {
-            return if self.0.gen_range(0..100) < 5 {
+        if self.simple_text {
+            return if self.rng.gen_range(0..100) < 5 {
                 Some('\n')
             } else {
-                Some(self.0.gen_range(b'a'..b'z' + 1).into())
+                Some(self.rng.gen_range(b'a'..b'z' + 1).into())
             };
         }
 
-        match self.0.gen_range(0..100) {
+        match self.rng.gen_range(0..100) {
             // whitespace
-            0..=19 => [' ', '\n', '\r', '\t'].choose(&mut self.0).copied(),
+            0..=19 => [' ', '\n', '\r', '\t'].choose(&mut self.rng).copied(),
             // two-byte greek letters
-            20..=32 => char::from_u32(self.0.gen_range(('α' as u32)..('ω' as u32 + 1))),
+            20..=32 => char::from_u32(self.rng.gen_range(('α' as u32)..('ω' as u32 + 1))),
             // // three-byte characters
-            33..=45 => ['✋', '✅', '❌', '❎', '⭐'].choose(&mut self.0).copied(),
+            33..=45 => ['✋', '✅', '❌', '❎', '⭐']
+                .choose(&mut self.rng)
+                .copied(),
             // // four-byte characters
-            46..=58 => ['🍐', '🏀', '🍗', '🎉'].choose(&mut self.0).copied(),
+            46..=58 => ['🍐', '🏀', '🍗', '🎉'].choose(&mut self.rng).copied(),
             // ascii letters
-            _ => Some(self.0.gen_range(b'a'..b'z' + 1).into()),
+            _ => Some(self.rng.gen_range(b'a'..b'z' + 1).into()),
         }
     }
 }