mirror of
https://github.com/BurntSushi/ripgrep.git
synced 2024-12-12 19:18:24 +02:00
139f186e57
This replaces the use of channels in the parallel directory traversal with a simple stack. The primary motivation for this change is to reduce peak memory usage. In particular, when using a channel (which is a queue), we wind up visiting files in a breadth first fashion. Using a stack switches us to a depth first traversal. While there are no real intrinsic differences, depth first traversal generally tends to use less memory because directory trees are more commonly wide than they are deep. In particular, the queue/stack size itself is not the only concern. In one recent case documented in #1550, a user wanted to search all Rust crates. The directory structure was shallow but extremely wide, with a single directory containing all crates. This in turn results is in descending into each of those directories and building a gitignore matcher for each (since most crates have `.gitignore` files) before ever searching a single file. This means that ripgrep has all such matchers in memory simultaneously, which winds up using quite a bit of memory. In a depth first traversal, peak memory usage is much lower because gitignore matches are built and discarded more quickly. In the case of searching all crates, the peak memory usage decrease is dramatic. On my system, it shrinks by an order magnitude, from almost 1GB to 50MB. The decline in peak memory usage is consistent across other use cases as well, but is typically more modest. For example, searching the Linux repo has a 50% decrease in peak memory usage and searching the Chromium repo has a 25% decrease in peak memory usage. Search times generally remain unchanged, although some ad hoc benchmarks that I typically run have gotten a bit slower. As far as I can tell, this appears to be result of scheduling changes. Namely, the depth first traversal seems to result in searching some very large files towards the end of the search, which reduces the effectiveness of parallelism and makes the overall search take longer. This seems to suggest that a stack isn't optimal. It would instead perhaps be better to prioritize searching larger files first, but it's not quite clear how to do this without introducing more overhead (getting the file size for each file requires a stat call). Fixes #1550
39 lines
943 B
TOML
39 lines
943 B
TOML
[package]
|
|
name = "ignore"
|
|
version = "0.4.14" #:version
|
|
authors = ["Andrew Gallant <jamslam@gmail.com>"]
|
|
description = """
|
|
A fast library for efficiently matching ignore files such as `.gitignore`
|
|
against file paths.
|
|
"""
|
|
documentation = "https://docs.rs/ignore"
|
|
homepage = "https://github.com/BurntSushi/ripgrep/tree/master/crates/ignore"
|
|
repository = "https://github.com/BurntSushi/ripgrep/tree/master/crates/ignore"
|
|
readme = "README.md"
|
|
keywords = ["glob", "ignore", "gitignore", "pattern", "file"]
|
|
license = "Unlicense/MIT"
|
|
|
|
[lib]
|
|
name = "ignore"
|
|
bench = false
|
|
|
|
[dependencies]
|
|
crossbeam-utils = "0.7.0"
|
|
globset = { version = "0.4.3", path = "../globset" }
|
|
lazy_static = "1.1"
|
|
log = "0.4.5"
|
|
memchr = "2.1"
|
|
regex = "1.1"
|
|
same-file = "1.0.4"
|
|
thread_local = "1"
|
|
walkdir = "2.2.7"
|
|
|
|
[target.'cfg(windows)'.dependencies.winapi-util]
|
|
version = "0.1.2"
|
|
|
|
[dev-dependencies]
|
|
crossbeam-channel = "0.4.0"
|
|
|
|
[features]
|
|
simd-accel = ["globset/simd-accel"]
|