mirror of
https://github.com/meilisearch/meilisearch.git
synced 2024-11-22 01:57:41 +08:00
Replace the procfs by libproc
This commit is contained in:
Clément Renault
Louis Dureuil
parent
d78ada07b5
commit
02dcaf07db
106
Cargo.lock
generated
106
Cargo.lock
generated
@ -535,6 +535,26 @@ dependencies = [
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bindgen"
|
||||
version = "0.68.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "726e4313eb6ec35d2730258ad4e15b547ee75d6afaa1361a922e78e59b7d8078"
|
||||
dependencies = [
|
||||
"bitflags 2.4.1",
|
||||
"cexpr",
|
||||
"clang-sys",
|
||||
"lazy_static",
|
||||
"lazycell",
|
||||
"peeking_take_while",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"regex",
|
||||
"rustc-hash",
|
||||
"shlex",
|
||||
"syn 2.0.48",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bit-set"
|
||||
version = "0.5.3"
|
||||
@ -831,6 +851,15 @@ dependencies = [
|
||||
"smallvec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cexpr"
|
||||
version = "0.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766"
|
||||
dependencies = [
|
||||
"nom",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cfg-if"
|
||||
version = "1.0.0"
|
||||
@ -915,6 +944,17 @@ dependencies = [
|
||||
"inout",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clang-sys"
|
||||
version = "1.7.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "67523a3b4be3ce1989d607a828d036249522dd9c1c8de7f4dd2dae43a37369d1"
|
||||
dependencies = [
|
||||
"glob",
|
||||
"libc",
|
||||
"libloading",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clap"
|
||||
version = "4.4.17"
|
||||
@ -3119,6 +3159,12 @@ version = "1.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
|
||||
|
||||
[[package]]
|
||||
name = "lazycell"
|
||||
version = "1.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
|
||||
|
||||
[[package]]
|
||||
name = "levenshtein_automata"
|
||||
version = "0.2.1"
|
||||
@ -3146,6 +3192,16 @@ dependencies = [
|
||||
"pkg-config",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "libloading"
|
||||
version = "0.8.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c571b676ddfc9a8c12f1f3d3085a7b163966a8fd8098a90640953ce5f6170161"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"windows-sys 0.48.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "libm"
|
||||
version = "0.2.7"
|
||||
@ -3162,6 +3218,17 @@ dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "libproc"
|
||||
version = "0.14.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "229004ebba9d1d5caf41623f1523b6d52abb47d9f6ab87f7e6fc992e3b854aef"
|
||||
dependencies = [
|
||||
"bindgen",
|
||||
"errno",
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "libz-sys"
|
||||
version = "1.1.12"
|
||||
@ -4177,6 +4244,12 @@ dependencies = [
|
||||
"sha2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "peeking_take_while"
|
||||
version = "0.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099"
|
||||
|
||||
[[package]]
|
||||
name = "pem"
|
||||
version = "1.1.1"
|
||||
@ -4432,29 +4505,6 @@ dependencies = [
|
||||
"rustix 0.36.16",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "procfs"
|
||||
version = "0.16.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "731e0d9356b0c25f16f33b5be79b1c57b562f141ebfcdb0ad8ac2c13a24293b4"
|
||||
dependencies = [
|
||||
"bitflags 2.4.1",
|
||||
"hex",
|
||||
"lazy_static",
|
||||
"procfs-core",
|
||||
"rustix 0.38.26",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "procfs-core"
|
||||
version = "0.16.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2d3554923a69f4ce04c4a754260c338f505ce22642d3830e049a399fc2059a29"
|
||||
dependencies = [
|
||||
"bitflags 2.4.1",
|
||||
"hex",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "prometheus"
|
||||
version = "0.13.3"
|
||||
@ -4467,7 +4517,7 @@ dependencies = [
|
||||
"libc",
|
||||
"memchr",
|
||||
"parking_lot",
|
||||
"procfs 0.14.2",
|
||||
"procfs",
|
||||
"protobuf",
|
||||
"thiserror",
|
||||
]
|
||||
@ -5079,6 +5129,12 @@ dependencies = [
|
||||
"lazy_static",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "shlex"
|
||||
version = "1.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
|
||||
|
||||
[[package]]
|
||||
name = "signal-hook-registry"
|
||||
version = "1.4.1"
|
||||
@ -5722,7 +5778,7 @@ dependencies = [
|
||||
"byte-unit",
|
||||
"color-spantrace",
|
||||
"fxprof-processed-profile",
|
||||
"procfs 0.16.0",
|
||||
"libproc",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"tokio",
|
||||
|
||||
@ -19,5 +19,5 @@ byte-unit = { version = "4.0.19", default-features = false, features = [
|
||||
] }
|
||||
tokio = { version = "1.35.1", features = ["sync"] }
|
||||
|
||||
[target.'cfg(target_os = "linux")'.dependencies]
|
||||
procfs = { version = "0.16.0", default-features = false }
|
||||
[target.'cfg(any(target_os = "linux", target_os = "macos"))'.dependencies]
|
||||
libproc = "0.14.2"
|
||||
|
||||
@ -100,46 +100,29 @@ pub struct SpanClose {
|
||||
pub time: std::time::Duration,
|
||||
}
|
||||
|
||||
/// A struct with a lot of memory allocation stats akin
|
||||
/// to the `procfs::Process::StatsM` one plus the OOM score.
|
||||
///
|
||||
/// Note that all the values are in bytes not in pages.
|
||||
/// A struct with a memory allocation stat.
|
||||
#[derive(Debug, Default, Clone, Copy, Serialize, Deserialize)]
|
||||
pub struct MemoryStats {
|
||||
/// Resident set size, measured in bytes.
|
||||
/// (same as VmRSS in /proc/<pid>/status).
|
||||
pub resident: u64,
|
||||
/// Number of resident shared bytes (i.e., backed by a file).
|
||||
/// (same as RssFile+RssShmem in /proc/<pid>/status).
|
||||
pub shared: u64,
|
||||
/// The current score that the kernel gives to this process
|
||||
/// for the purpose of selecting a process for the OOM-killer
|
||||
///
|
||||
/// A higher score means that the process is more likely to be selected
|
||||
/// by the OOM-killer. The basis for this score is the amount of memory used
|
||||
/// by the process, plus other factors.
|
||||
///
|
||||
/// (Since linux 2.6.11)
|
||||
pub oom_score: u32,
|
||||
}
|
||||
|
||||
impl MemoryStats {
|
||||
#[cfg(target_os = "linux")]
|
||||
pub fn fetch() -> procfs::ProcResult<Self> {
|
||||
let process = procfs::process::Process::myself().unwrap();
|
||||
let procfs::process::StatM { resident, shared, .. } = process.statm()?;
|
||||
let oom_score = process.oom_score()?;
|
||||
let page_size = procfs::page_size();
|
||||
#[cfg(any(target_os = "linux", target_os = "macos"))]
|
||||
pub fn fetch() -> Option<Self> {
|
||||
use libproc::libproc::pid_rusage::{pidrusage, RUsageInfoV0};
|
||||
|
||||
Ok(MemoryStats { resident: resident * page_size, shared: shared * page_size, oom_score })
|
||||
match pidrusage(std::process::id() as i32) {
|
||||
Ok(RUsageInfoV0 { ri_resident_size, .. }) => {
|
||||
Some(MemoryStats { resident: ri_resident_size })
|
||||
}
|
||||
Err(_) => None, /* ignoring error to avoid spamming */
|
||||
}
|
||||
}
|
||||
|
||||
pub fn checked_sub(self, other: Self) -> Option<Self> {
|
||||
Some(Self {
|
||||
resident: self.resident.checked_sub(other.resident)?,
|
||||
shared: self.shared.checked_sub(other.shared)?,
|
||||
oom_score: self.oom_score.checked_sub(other.oom_score)?,
|
||||
})
|
||||
Some(Self { resident: self.resident.checked_sub(other.resident)? })
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -96,12 +96,12 @@ impl TraceLayer {
|
||||
self.start_time.elapsed()
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
#[cfg(any(target_os = "linux", target_os = "macos"))]
|
||||
fn memory_stats(&self) -> Option<MemoryStats> {
|
||||
Some(MemoryStats::fetch().unwrap())
|
||||
}
|
||||
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
#[cfg(not(any(target_os = "linux", target_os = "macos")))]
|
||||
fn memory_stats(&self) -> Option<MemoryStats> {
|
||||
None
|
||||
}
|
||||
|
||||
@ -322,18 +322,6 @@ impl<'a> ProfilerMarker for SpanMarker<'a> {
|
||||
format: MarkerFieldFormat::Bytes,
|
||||
searchable: false,
|
||||
}),
|
||||
MarkerSchemaField::Dynamic(MarkerDynamicField {
|
||||
key: "shared",
|
||||
label: "Number of resident shared pages (i.e., backed by a file) while this function was executing",
|
||||
format: MarkerFieldFormat::Bytes,
|
||||
searchable: false,
|
||||
}),
|
||||
MarkerSchemaField::Dynamic(MarkerDynamicField {
|
||||
key: "oom_score",
|
||||
label: "The current score that the kernel gives to this process for the purpose of selecting a process for the OOM-killer while this function was executing",
|
||||
format: MarkerFieldFormat::Integer,
|
||||
searchable: false,
|
||||
}),
|
||||
];
|
||||
|
||||
MarkerSchema {
|
||||
@ -366,10 +354,8 @@ impl<'a> ProfilerMarker for SpanMarker<'a> {
|
||||
"thread_id": thread_id,
|
||||
});
|
||||
|
||||
if let Some(MemoryStats { resident, shared, oom_score }) = self.memory_delta {
|
||||
if let Some(MemoryStats { resident }) = self.memory_delta {
|
||||
value["resident"] = json!(resident);
|
||||
value["shared"] = json!(shared);
|
||||
value["oom_score"] = json!(oom_score);
|
||||
}
|
||||
|
||||
value
|
||||
@ -423,18 +409,6 @@ impl<'a> ProfilerMarker for EventMarker<'a> {
|
||||
format: MarkerFieldFormat::Bytes,
|
||||
searchable: false,
|
||||
}),
|
||||
MarkerSchemaField::Dynamic(MarkerDynamicField {
|
||||
key: "shared",
|
||||
label: "Number of resident shared pages (i.e., backed by a file) while this function was executing",
|
||||
format: MarkerFieldFormat::Bytes,
|
||||
searchable: false,
|
||||
}),
|
||||
MarkerSchemaField::Dynamic(MarkerDynamicField {
|
||||
key: "oom_score",
|
||||
label: "The current score that the kernel gives to this process for the purpose of selecting a process for the OOM-killer while this function was executing",
|
||||
format: MarkerFieldFormat::Integer,
|
||||
searchable: false,
|
||||
}),
|
||||
];
|
||||
|
||||
MarkerSchema {
|
||||
@ -467,10 +441,8 @@ impl<'a> ProfilerMarker for EventMarker<'a> {
|
||||
"thread_id": thread_id,
|
||||
});
|
||||
|
||||
if let Some(MemoryStats { resident, shared, oom_score }) = self.memory_delta {
|
||||
if let Some(MemoryStats { resident }) = self.memory_delta {
|
||||
value["resident"] = json!(resident);
|
||||
value["shared"] = json!(shared);
|
||||
value["oom_score"] = json!(oom_score);
|
||||
}
|
||||
|
||||
value
|
||||
|
||||
@ -188,9 +188,8 @@ fn print_duration(duration: std::time::Duration) -> String {
|
||||
}
|
||||
|
||||
/// Format only the allocated bytes, deallocated bytes and reallocated bytes in GiB, MiB, KiB, Bytes.
|
||||
fn print_memory(MemoryStats { resident, shared, oom_score }: MemoryStats) -> String {
|
||||
fn print_memory(MemoryStats { resident }: MemoryStats) -> String {
|
||||
use byte_unit::Byte;
|
||||
let rss_bytes = Byte::from_bytes(resident).get_appropriate_unit(true);
|
||||
let shared_bytes = Byte::from_bytes(shared).get_appropriate_unit(true);
|
||||
format!("RSS {rss_bytes:.2}, Shared {shared_bytes:.2}, OOM score {oom_score}")
|
||||
format!("RSS {rss_bytes:.2}")
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user