debugger: Fix attaching with DebugPy (#34706)

@cole-miller found a root cause of our struggles with attach scenarios;
we did not fetch .so files necessary for attaching to work,
as we were downloading DebugPy source tarballs from GitHub.

This PR does away with it by setting up a virtualenv instead that has
debugpy installed.

Closes #34660
Closes #34575

Release Notes:

- debugger: Fixed attaching with DebugPy. DebugPy is now installed
automatically from pip (instead of GitHub), unless it is present in
active virtual environment. Additionally this should resolve any startup
issues with missing .so on Linux.

Cargo.lock

@@ -4248,6 +4248,7 @@ dependencies = [
  "serde",
  "serde_json",
  "shlex",
+ "smol",
  "task",
  "util",
  "workspace-hack",

crates/dap_adapters/Cargo.toml

@@ -36,6 +36,7 @@ paths.workspace = true
 serde.workspace = true
 serde_json.workspace = true
 shlex.workspace = true
+smol.workspace = true
 task.workspace = true
 util.workspace = true
 workspace-hack.workspace = true

crates/dap_adapters/src/dap_adapters.rs

@@ -13,7 +13,6 @@ use dap::{
     DapRegistry,
     adapters::{
         self, AdapterVersion, DapDelegate, DebugAdapter, DebugAdapterBinary, DebugAdapterName,
-        GithubRepo,
     },
     configure_tcp_connection,
 };

crates/dap_adapters/src/python.rs

@@ -1,31 +1,39 @@
 use crate::*;
 use anyhow::Context as _;
-use dap::adapters::latest_github_release;
 use dap::{DebugRequest, StartDebuggingRequestArguments, adapters::DebugTaskDefinition};
-use gpui::{AppContext, AsyncApp, SharedString};
+use gpui::{AsyncApp, SharedString};
 use json_dotpath::DotPaths;
-use language::{LanguageName, Toolchain};
+use language::LanguageName;
+use paths::debug_adapters_dir;
 use serde_json::Value;
+use smol::lock::OnceCell;
 use std::net::Ipv4Addr;
 use std::{
     collections::HashMap,
     ffi::OsStr,
     path::{Path, PathBuf},
-    sync::OnceLock,
 };
-use util::ResultExt;
 
 #[derive(Default)]
 pub(crate) struct PythonDebugAdapter {
-    checked: OnceLock<()>,
+    python_venv_base: OnceCell<Result<Arc<Path>, String>>,
 }
 
 impl PythonDebugAdapter {
     const ADAPTER_NAME: &'static str = "Debugpy";
     const DEBUG_ADAPTER_NAME: DebugAdapterName =
         DebugAdapterName(SharedString::new_static(Self::ADAPTER_NAME));
-    const ADAPTER_PACKAGE_NAME: &'static str = "debugpy";
-    const ADAPTER_PATH: &'static str = "src/debugpy/adapter";
+    const PYTHON_ADAPTER_IN_VENV: &'static str = if cfg!(target_os = "windows") {
+        "Scripts/python3"
+    } else {
+        "bin/python3"
+    };
+    const ADAPTER_PATH: &'static str = if cfg!(target_os = "windows") {
+        "debugpy-venv/Scripts/debugpy-adapter"
+    } else {
+        "debugpy-venv/bin/debugpy-adapter"
+    };
+
     const LANGUAGE_NAME: &'static str = "Python";
 
     async fn generate_debugpy_arguments(
@@ -46,25 +54,12 @@ impl PythonDebugAdapter {
             vec!["-m".to_string(), "debugpy.adapter".to_string()]
         } else {
             let adapter_path = paths::debug_adapters_dir().join(Self::DEBUG_ADAPTER_NAME.as_ref());
-            let file_name_prefix = format!("{}_", Self::ADAPTER_NAME);
-
-            let debugpy_dir =
-                util::fs::find_file_name_in_dir(adapter_path.as_path(), |file_name| {
-                    file_name.starts_with(&file_name_prefix)
-                })
-                .await
-                .context("Debugpy directory not found")?;
-
-            log::debug!(
-                "Using GitHub-downloaded debugpy adapter from: {}",
-                debugpy_dir.display()
-            );
-            vec![
-                debugpy_dir
+            let path = adapter_path
                 .join(Self::ADAPTER_PATH)
                 .to_string_lossy()
-                    .to_string(),
-            ]
+                .into_owned();
+            log::debug!("Using pip debugpy adapter from: {path}");
+            vec![path]
         };
 
         args.extend(if let Some(args) = user_args {
@@ -100,44 +95,67 @@ impl PythonDebugAdapter {
             request,
         })
     }
-    async fn fetch_latest_adapter_version(
-        &self,
-        delegate: &Arc<dyn DapDelegate>,
-    ) -> Result<AdapterVersion> {
-        let github_repo = GithubRepo {
-            repo_name: Self::ADAPTER_PACKAGE_NAME.into(),
-            repo_owner: "microsoft".into(),
-        };
 
-        fetch_latest_adapter_version_from_github(github_repo, delegate.as_ref()).await
+    async fn ensure_venv(delegate: &dyn DapDelegate) -> Result<Arc<Path>> {
+        let python_path = Self::find_base_python(delegate)
+            .await
+            .context("Could not find Python installation for DebugPy")?;
+        let work_dir = debug_adapters_dir().join(Self::ADAPTER_NAME);
+        let mut path = work_dir.clone();
+        path.push("debugpy-venv");
+        if !path.exists() {
+            util::command::new_smol_command(python_path)
+                .arg("-m")
+                .arg("venv")
+                .arg("debugpy-venv")
+                .current_dir(work_dir)
+                .spawn()?
+                .output()
+                .await?;
         }
 
-    async fn install_binary(
-        adapter_name: DebugAdapterName,
-        version: AdapterVersion,
-        delegate: Arc<dyn DapDelegate>,
-    ) -> Result<()> {
-        let version_path = adapters::download_adapter_from_github(
-            adapter_name,
-            version,
-            adapters::DownloadedFileType::GzipTar,
-            delegate.as_ref(),
-        )
-        .await?;
-        // only needed when you install the latest version for the first time
-        if let Some(debugpy_dir) =
-            util::fs::find_file_name_in_dir(version_path.as_path(), |file_name| {
-                file_name.starts_with("microsoft-debugpy-")
+        Ok(path.into())
+    }
+
+    // Find "baseline", user python version from which we'll create our own venv.
+    async fn find_base_python(delegate: &dyn DapDelegate) -> Option<PathBuf> {
+        for path in ["python3", "python"] {
+            if let Some(path) = delegate.which(path.as_ref()).await {
+                return Some(path);
+            }
+        }
+        None
+    }
+
+    async fn base_venv(&self, delegate: &dyn DapDelegate) -> Result<Arc<Path>, String> {
+        const BINARY_DIR: &str = if cfg!(target_os = "windows") {
+            "Scripts"
+        } else {
+            "bin"
+        };
+        self.python_venv_base
+            .get_or_init(move || async move {
+                let venv_base = Self::ensure_venv(delegate)
+                    .await
+                    .map_err(|e| format!("{e}"))?;
+                let pip_path = venv_base.join(BINARY_DIR).join("pip3");
+                let installation_succeeded = util::command::new_smol_command(pip_path.as_path())
+                    .arg("install")
+                    .arg("debugpy")
+                    .arg("-U")
+                    .output()
+                    .await
+                    .map_err(|e| format!("{e}"))?
+                    .status
+                    .success();
+                if !installation_succeeded {
+                    return Err("debugpy installation failed".into());
+                }
+
+                Ok(venv_base)
             })
             .await
-        {
-            // TODO Debugger: Rename folder instead of moving all files to another folder
-            // We're doing unnecessary IO work right now
-            util::fs::move_folder_files_to_folder(debugpy_dir.as_path(), version_path.as_path())
-                .await?;
-        }
-
-        Ok(())
+            .clone()
     }
 
     async fn get_installed_binary(
@@ -146,15 +164,15 @@ impl PythonDebugAdapter {
         config: &DebugTaskDefinition,
         user_installed_path: Option<PathBuf>,
         user_args: Option<Vec<String>>,
-        toolchain: Option<Toolchain>,
+        python_from_toolchain: Option<String>,
         installed_in_venv: bool,
     ) -> Result<DebugAdapterBinary> {
         const BINARY_NAMES: [&str; 3] = ["python3", "python", "py"];
         let tcp_connection = config.tcp_connection.clone().unwrap_or_default();
         let (host, port, timeout) = crate::configure_tcp_connection(tcp_connection).await?;
 
-        let python_path = if let Some(toolchain) = toolchain {
-            Some(toolchain.path.to_string())
+        let python_path = if let Some(toolchain) = python_from_toolchain {
+            Some(toolchain)
         } else {
             let mut name = None;
 
@@ -635,24 +653,27 @@ impl DebugAdapter for PythonDebugAdapter {
                             &config,
                             None,
                             user_args,
-                            Some(toolchain.clone()),
+                            Some(toolchain.path.to_string()),
                             true,
                         )
                         .await;
                 }
             }
         }
-
-        if self.checked.set(()).is_ok() {
-            delegate.output_to_console(format!("Checking latest version of {}...", self.name()));
-            if let Some(version) = self.fetch_latest_adapter_version(delegate).await.log_err() {
-                cx.background_spawn(Self::install_binary(self.name(), version, delegate.clone()))
+        let toolchain = self
+            .base_venv(&**delegate)
             .await
-                    .context("Failed to install debugpy")?;
-            }
-        }
+            .map_err(|e| anyhow::anyhow!(e))?
+            .join(Self::PYTHON_ADAPTER_IN_VENV);
 
-        self.get_installed_binary(delegate, &config, None, user_args, toolchain, false)
+        self.get_installed_binary(
+            delegate,
+            &config,
+            None,
+            user_args,
+            Some(toolchain.to_string_lossy().into_owned()),
+            false,
+        )
         .await
     }
 
@@ -666,24 +687,6 @@ impl DebugAdapter for PythonDebugAdapter {
     }
 }
 
-async fn fetch_latest_adapter_version_from_github(
-    github_repo: GithubRepo,
-    delegate: &dyn DapDelegate,
-) -> Result<AdapterVersion> {
-    let release = latest_github_release(
-        &format!("{}/{}", github_repo.repo_owner, github_repo.repo_name),
-        false,
-        false,
-        delegate.http_client(),
-    )
-    .await?;
-
-    Ok(AdapterVersion {
-        tag_name: release.tag_name,
-        url: release.tarball_url,
-    })
-}
-
 #[cfg(test)]
 mod tests {
     use super::*;