Remove vendored files from ruff binary

.github/workflows/sync_typeshed.yaml

@@ -37,13 +37,13 @@ jobs:
       - name: Sync typeshed
         id: sync
         run: |
-          rm -rf ruff/crates/red_knot_python_semantic/vendor/typeshed
-          mkdir ruff/crates/red_knot_python_semantic/vendor/typeshed
-          cp typeshed/README.md ruff/crates/red_knot_python_semantic/vendor/typeshed
-          cp typeshed/LICENSE ruff/crates/red_knot_python_semantic/vendor/typeshed
-          cp -r typeshed/stdlib ruff/crates/red_knot_python_semantic/vendor/typeshed/stdlib
-          rm -rf ruff/crates/red_knot_python_semantic/vendor/typeshed/stdlib/@tests
-          git -C typeshed rev-parse HEAD > ruff/crates/red_knot_python_semantic/vendor/typeshed/source_commit.txt
+          rm -rf ruff/crates/ruff_vendored/vendor/typeshed
+          mkdir ruff/crates/ruff_vendored/vendor/typeshed
+          cp typeshed/README.md ruff/crates/ruff_vendored/vendor/typeshed
+          cp typeshed/LICENSE ruff/crates/ruff_vendored/vendor/typeshed
+          cp -r typeshed/stdlib ruff/crates/ruff_vendored/vendor/typeshed/stdlib
+          rm -rf ruff/crates/ruff_vendored/vendor/typeshed/stdlib/@tests
+          git -C typeshed rev-parse HEAD > ruff/crates/ruff_vendored/vendor/typeshed/source_commit.txt
       - name: Commit the changes
         id: commit
         if: ${{ steps.sync.outcome == 'success' }}

.pre-commit-config.yaml

@@ -2,7 +2,7 @@ fail_fast: true
 
 exclude: |
   (?x)^(
-    crates/red_knot_python_semantic/vendor/.*|
+    crates/ruff_vendored/vendor/.*|
     crates/red_knot_workspace/resources/.*|
     crates/ruff_linter/resources/.*|
     crates/ruff_linter/src/rules/.*/snapshots/.*|

Cargo.toml

@@ -14,7 +14,7 @@ license = "MIT"
 [workspace.dependencies]
 ruff = { path = "crates/ruff" }
 ruff_cache = { path = "crates/ruff_cache" }
-ruff_db = { path = "crates/ruff_db" }
+ruff_db = { path = "crates/ruff_db", default-features = false }
 ruff_diagnostics = { path = "crates/ruff_diagnostics" }
 ruff_formatter = { path = "crates/ruff_formatter" }
 ruff_graph = { path = "crates/ruff_graph" }
@@ -34,11 +34,12 @@ ruff_python_trivia = { path = "crates/ruff_python_trivia" }
 ruff_server = { path = "crates/ruff_server" }
 ruff_source_file = { path = "crates/ruff_source_file" }
 ruff_text_size = { path = "crates/ruff_text_size" }
+ruff_vendored = { path = "crates/ruff_vendored" }
 ruff_workspace = { path = "crates/ruff_workspace" }
 
 red_knot_python_semantic = { path = "crates/red_knot_python_semantic" }
 red_knot_server = { path = "crates/red_knot_server" }
-red_knot_workspace = { path = "crates/red_knot_workspace" }
+red_knot_workspace = { path = "crates/red_knot_workspace", default-features = false }
 
 aho-corasick = { version = "1.1.3" }
 annotate-snippets = { version = "0.9.2", features = ["color"] }

crates/red_knot/Cargo.toml

@@ -13,9 +13,8 @@ license.workspace = true
 
 [dependencies]
 red_knot_python_semantic = { workspace = true }
-red_knot_workspace = { workspace = true }
+red_knot_workspace = { workspace = true, features = ["zstd"] }
 red_knot_server = { workspace = true }
-
 ruff_db = { workspace = true, features = ["os", "cache"] }
 
 anyhow = { workspace = true }

crates/red_knot_python_semantic/Cargo.toml

@@ -24,7 +24,6 @@ bitflags = { workspace = true }
 camino = { workspace = true }
 compact_str = { workspace = true }
 countme = { workspace = true }
-once_cell = { workspace = true }
 ordermap = { workspace = true }
 salsa = { workspace = true }
 thiserror = { workspace = true }
@@ -35,20 +34,14 @@ smallvec = { workspace = true }
 static_assertions = { workspace = true }
 test-case = { workspace = true }
 
-[build-dependencies]
-path-slash = { workspace = true }
-walkdir = { workspace = true }
-zip = { workspace = true, features = ["zstd", "deflate"] }
-
 [dev-dependencies]
 ruff_db = { workspace = true, features = ["os", "testing"] }
 ruff_python_parser = { workspace = true }
+ruff_vendored = { workspace = true }
 
 anyhow = { workspace = true }
 insta = { workspace = true }
 tempfile = { workspace = true }
-walkdir = { workspace = true }
-zip = { workspace = true }
 
 [lints]
 workspace = true

crates/red_knot_python_semantic/src/db.rs

@@ -11,7 +11,6 @@ pub trait Db: SourceDb + Upcast<dyn SourceDb> {
 pub(crate) mod tests {
     use std::sync::Arc;
 
-    use crate::module_resolver::vendored_typeshed_stubs;
     use ruff_db::files::{File, Files};
     use ruff_db::system::{DbWithTestSystem, System, TestSystem};
     use ruff_db::vendored::VendoredFileSystem;
@@ -33,7 +32,7 @@ pub(crate) mod tests {
             Self {
                 storage: salsa::Storage::default(),
                 system: TestSystem::default(),
-                vendored: vendored_typeshed_stubs().clone(),
+                vendored: ruff_vendored::file_system().clone(),
                 events: std::sync::Arc::default(),
                 files: Files::default(),
             }

crates/red_knot_python_semantic/src/lib.rs

@@ -4,9 +4,7 @@ use rustc_hash::FxHasher;
 
 pub use db::Db;
 pub use module_name::ModuleName;
-pub use module_resolver::{
-    resolve_module, system_module_search_paths, vendored_typeshed_stubs, Module,
-};
+pub use module_resolver::{resolve_module, system_module_search_paths, Module};
 pub use program::{Program, ProgramSettings, SearchPathSettings, SitePackages};
 pub use python_version::PythonVersion;
 pub use semantic_model::{HasTy, SemanticModel};

crates/red_knot_python_semantic/src/module_resolver/mod.rs

@@ -4,7 +4,6 @@ pub use module::Module;
 pub use resolver::resolve_module;
 pub(crate) use resolver::{file_to_module, SearchPaths};
 use ruff_db::system::SystemPath;
-pub use typeshed::vendored_typeshed_stubs;
 
 use crate::module_resolver::resolver::search_paths;
 use crate::Db;

crates/red_knot_python_semantic/src/module_resolver/typeshed/versions.rs → crates/red_knot_python_semantic/src/module_resolver/typeshed.rs

@@ -390,7 +390,8 @@ mod tests {
     fn typeshed_versions_consistent_with_vendored_stubs() {
         let db = TestDb::new();
         let vendored_typeshed_versions = vendored_typeshed_versions(&db);
-        let vendored_typeshed_dir = Path::new("vendor/typeshed").canonicalize().unwrap();
+        let vendored_typeshed_dir =
+            Path::new(env!("CARGO_MANIFEST_DIR")).join("../ruff_vendored/vendor/typeshed");
 
         let mut empty_iterator = true;
 

crates/red_knot_python_semantic/src/types.rs

@@ -701,12 +701,12 @@ enum CallOutcome<'db> {
 
 impl<'db> CallOutcome<'db> {
     /// Create a new `CallOutcome::Callable` with given return type.
-    fn callable(return_ty: Type<'db>) -> CallOutcome {
+    fn callable(return_ty: Type<'db>) -> CallOutcome<'db> {
         CallOutcome::Callable { return_ty }
     }
 
     /// Create a new `CallOutcome::NotCallable` with given not-callable type.
-    fn not_callable(not_callable_ty: Type<'db>) -> CallOutcome {
+    fn not_callable(not_callable_ty: Type<'db>) -> CallOutcome<'db> {
         CallOutcome::NotCallable { not_callable_ty }
     }
 
@@ -719,7 +719,10 @@ impl<'db> CallOutcome<'db> {
     }
 
     /// Create a new `CallOutcome::Union` with given wrapped outcomes.
-    fn union(called_ty: Type<'db>, outcomes: impl Into<Box<[CallOutcome<'db>]>>) -> CallOutcome {
+    fn union(
+        called_ty: Type<'db>,
+        outcomes: impl Into<Box<[CallOutcome<'db>]>>,
+    ) -> CallOutcome<'db> {
         CallOutcome::Union {
             called_ty,
             outcomes: outcomes.into(),

crates/red_knot_wasm/Cargo.toml

@@ -20,9 +20,9 @@ default = ["console_error_panic_hook"]
 
 [dependencies]
 red_knot_python_semantic = { workspace = true }
-red_knot_workspace = { workspace = true }
+red_knot_workspace = { workspace = true, default-features = false, features = ["deflate"] }
 
-ruff_db = { workspace = true }
+ruff_db = { workspace = true, features = [] }
 ruff_notebook = { workspace = true }
 
 console_error_panic_hook = { workspace = true, optional = true }

crates/red_knot_workspace/Cargo.toml

@@ -18,6 +18,7 @@ ruff_cache = { workspace = true }
 ruff_db = { workspace = true, features = ["os", "cache"] }
 ruff_python_ast = { workspace = true }
 ruff_text_size = { workspace = true }
+ruff_vendored = { workspace = true }
 
 anyhow = { workspace = true }
 crossbeam = { workspace = true }
@@ -31,5 +32,10 @@ tracing = { workspace = true }
 ruff_db = { workspace = true, features = ["testing"] }
 tempfile = { workspace = true }
 
+[features]
+default = ["zstd"]
+zstd = ["ruff_vendored/zstd"]
+deflate = ["ruff_vendored/deflate"]
+
 [lints]
 workspace = true

crates/red_knot_workspace/src/db.rs

@@ -4,7 +4,7 @@ use std::sync::Arc;
 use salsa::plumbing::ZalsaDatabase;
 use salsa::{Cancelled, Event};
 
-use red_knot_python_semantic::{vendored_typeshed_stubs, Db as SemanticDb, Program};
+use red_knot_python_semantic::{Db as SemanticDb, Program};
 use ruff_db::files::{File, Files};
 use ruff_db::system::System;
 use ruff_db::vendored::VendoredFileSystem;
@@ -124,7 +124,7 @@ impl SemanticDb for RootDatabase {
 #[salsa::db]
 impl SourceDb for RootDatabase {
     fn vendored(&self) -> &VendoredFileSystem {
-        vendored_typeshed_stubs()
+        ruff_vendored::file_system()
     }
 
     fn system(&self) -> &dyn System {
@@ -161,7 +161,7 @@ pub(crate) mod tests {
 
     use salsa::Event;
 
-    use red_knot_python_semantic::{vendored_typeshed_stubs, Db as SemanticDb};
+    use red_knot_python_semantic::Db as SemanticDb;
     use ruff_db::files::Files;
     use ruff_db::system::{DbWithTestSystem, System, TestSystem};
     use ruff_db::vendored::VendoredFileSystem;
@@ -183,7 +183,7 @@ pub(crate) mod tests {
             Self {
                 storage: salsa::Storage::default(),
                 system: TestSystem::default(),
-                vendored: vendored_typeshed_stubs().clone(),
+                vendored: ruff_vendored::file_system().clone(),
                 files: Files::default(),
                 events: Arc::default(),
             }

crates/ruff/Cargo.toml

@@ -14,7 +14,7 @@ default-run = "ruff"
 
 [dependencies]
 ruff_cache = { workspace = true }
-ruff_db = { workspace = true }
+ruff_db = { workspace = true, default-features = false, features = ["os"] }
 ruff_diagnostics = { workspace = true }
 ruff_graph = { workspace = true, features = ["serde", "clap"] }
 ruff_linter = { workspace = true, features = ["clap"] }

crates/ruff_db/Cargo.toml

@@ -33,19 +33,17 @@ tracing = { workspace = true }
 tracing-subscriber = { workspace = true, optional = true }
 tracing-tree = { workspace = true, optional = true }
 rustc-hash = { workspace = true }
-
-[target.'cfg(not(target_arch="wasm32"))'.dependencies]
-zip = { workspace = true, features = ["zstd"] }
+zip = { workspace = true }
 
 [target.'cfg(target_arch="wasm32")'.dependencies]
 web-time = { version = "1.1.0" }
-zip = { workspace = true, features = ["deflate"] }
 
 [dev-dependencies]
 insta = { workspace = true }
 tempfile = { workspace = true }
 
 [features]
+default = ["os"]
 cache = ["ruff_cache"]
 os = ["ignore"]
 serde = ["dep:serde", "camino/serde1"]

crates/ruff_db/src/.vendored.rs.pending-snap

@@ -0,0 +1,3 @@
+{"run_id":"1726924292-321604","line":421,"new":{"module_name":"ruff_db__vendored__tests","snapshot_name":"filesystem_debug_implementation_alternate","metadata":{"source":"crates/ruff_db/src/vendored.rs","assertion_line":421,"expression":"format!(\"{:#?}\", mock_typeshed())"},"snapshot":"VendoredFileSystem {\n    inner_mutex_poisoned: false,\n    paths: [\n        \"stdlib/\",\n        \"stdlib/asyncio/\",\n        \"stdlib/asyncio/tasks.pyi\",\n        \"stdlib/functools.pyi\",\n    ],\n    data_by_path: {\n        \"stdlib/\": ZipFileDebugInfo {\n            crc32_hash: 0,\n            compressed_size: 0,\n            uncompressed_size: 0,\n            kind: Directory,\n        },\n        \"stdlib/asyncio/\": ZipFileDebugInfo {\n            crc32_hash: 0,\n            compressed_size: 0,\n            uncompressed_size: 0,\n            kind: Directory,\n        },\n        \"stdlib/asyncio/tasks.pyi\": ZipFileDebugInfo {\n            crc32_hash: 2826547428,\n            compressed_size: 15,\n            uncompressed_size: 15,\n            kind: File,\n        },\n        \"stdlib/functools.pyi\": ZipFileDebugInfo {\n            crc32_hash: 1099005079,\n            compressed_size: 25,\n            uncompressed_size: 25,\n            kind: File,\n        },\n    },\n}"},"old":{"module_name":"ruff_db__vendored__tests","metadata":{},"snapshot":"VendoredFileSystem {\n    inner_mutex_poisoned: false,\n    paths: [\n        \"stdlib/\",\n        \"stdlib/asyncio/\",\n        \"stdlib/asyncio/tasks.pyi\",\n        \"stdlib/functools.pyi\",\n    ],\n    data_by_path: {\n        \"stdlib/\": ZipFileDebugInfo {\n            crc32_hash: 0,\n            compressed_size: 0,\n            uncompressed_size: 0,\n            kind: Directory,\n        },\n        \"stdlib/asyncio/\": ZipFileDebugInfo {\n            crc32_hash: 0,\n            compressed_size: 0,\n            uncompressed_size: 0,\n            kind: Directory,\n        },\n        \"stdlib/asyncio/tasks.pyi\": ZipFileDebugInfo {\n            crc32_hash: 2826547428,\n            compressed_size: 24,\n            uncompressed_size: 15,\n            kind: File,\n        },\n        \"stdlib/functools.pyi\": ZipFileDebugInfo {\n            crc32_hash: 1099005079,\n            compressed_size: 34,\n            uncompressed_size: 25,\n            kind: File,\n        },\n    },\n}"}}
+{"run_id":"1726924355-481144814","line":413,"new":null,"old":null}
+{"run_id":"1726924355-481144814","line":421,"new":null,"old":null}

crates/ruff_db/src/files.rs

@@ -503,7 +503,8 @@ mod tests {
     use crate::files::{system_path_to_file, vendored_path_to_file, FileError};
     use crate::system::DbWithTestSystem;
     use crate::tests::TestDb;
-    use crate::vendored::tests::VendoredFileSystemBuilder;
+    use crate::vendored::VendoredFileSystemBuilder;
+    use zip::CompressionMethod;
 
     #[test]
     fn system_existing_file() -> crate::system::Result<()> {
@@ -548,7 +549,7 @@ mod tests {
     fn stubbed_vendored_file() -> crate::system::Result<()> {
         let mut db = TestDb::new();
 
-        let mut vendored_builder = VendoredFileSystemBuilder::new();
+        let mut vendored_builder = VendoredFileSystemBuilder::new(CompressionMethod::Stored);
         vendored_builder
             .add_file("test.pyi", "def foo() -> str")
             .unwrap();

crates/ruff_db/src/parsed.rs

@@ -79,8 +79,9 @@ mod tests {
     use crate::parsed::parsed_module;
     use crate::system::{DbWithTestSystem, SystemPath, SystemVirtualPath};
     use crate::tests::TestDb;
-    use crate::vendored::{tests::VendoredFileSystemBuilder, VendoredPath};
+    use crate::vendored::{VendoredFileSystemBuilder, VendoredPath};
     use crate::Db;
+    use zip::CompressionMethod;
 
     #[test]
     fn python_file() -> crate::system::Result<()> {
@@ -150,7 +151,7 @@ mod tests {
     fn vendored_file() {
         let mut db = TestDb::new();
 
-        let mut vendored_builder = VendoredFileSystemBuilder::new();
+        let mut vendored_builder = VendoredFileSystemBuilder::new(CompressionMethod::Stored);
         vendored_builder
             .add_file(
                 "path.pyi",