From 103246486d08320f36c39e3b3100eea1711bbb29 Mon Sep 17 00:00:00 2001 From: Douglas Raillard Date: Wed, 19 Jun 2024 14:55:07 +0100 Subject: [PATCH 1/3] tools/trace-parser/trace-tools: Make trace.dat support a cargo feature FEATURE Prepare the ground for adding support to more trace formats by making trace.dat support optional. --- .../trace-tools/src/bin/trace-dump.rs | 54 ++++++++----------- .../trace-parser/trace-tools/src/lib/error.rs | 1 + 2 files changed, 23 insertions(+), 32 deletions(-) diff --git a/tools/trace-parser/trace-tools/src/bin/trace-dump.rs b/tools/trace-parser/trace-tools/src/bin/trace-dump.rs index ccf9b6bb5..28344849e 100644 --- a/tools/trace-parser/trace-tools/src/bin/trace-dump.rs +++ b/tools/trace-parser/trace-tools/src/bin/trace-dump.rs @@ -14,7 +14,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::{error::Error, fs::File, io::Write, path::PathBuf, process::ExitCode}; +use std::{ + error::Error, fs::File, io::Write, path::PathBuf, process::ExitCode, +}; #[cfg(target_arch = "x86_64")] use mimalloc::MiMalloc; @@ -23,7 +25,9 @@ use mimalloc::MiMalloc; static GLOBAL: MiMalloc = MiMalloc; use clap::{Parser, Subcommand, ValueEnum}; -use lib::error::DynMultiError; +use lib::{ + error::DynMultiError, +}; use parquet::basic::{Compression as ParquetCompression, ZstdLevel}; #[derive(Parser)] @@ -135,10 +139,10 @@ enum Command { impl Command { fn trace_format(&self) -> Option { match self { - Command::HumanReadable { trace_format, .. } => Some(trace_format.clone()), - Command::Parquet { trace_format, .. } => Some(trace_format.clone()), - Command::CheckHeader { trace_format, .. } => Some(trace_format.clone()), - Command::Metadata { trace_format, .. } => Some(trace_format.clone()), + Command::HumanReadable {trace_format, ..} => Some(trace_format.clone()), + Command::Parquet {trace_format, ..} => Some(trace_format.clone()), + Command::CheckHeader {trace_format, ..} => Some(trace_format.clone()), + Command::Metadata {trace_format, ..} => Some(trace_format.clone()), } } } @@ -153,7 +157,7 @@ fn _main() -> Result<(), Box> { #[cfg(feature = "tracedat")] Some(TraceFormat::TraceDat) => tracedat_main(&cli, &mut out), - _ => Err(DynMultiError::from_string("File format not handled".into())), + _ => Err(DynMultiError::from_string("File format not handled".into())) }; out.flush()?; @@ -195,28 +199,27 @@ fn main() -> ExitCode { #[cfg(feature = "tracedat")] fn tracedat_main(cli: &Cli, out: &mut W) -> Result<(), DynMultiError> -where - W: Write, + where W: Write { use std::ops::DerefMut as _; + use traceevent::{ + io::MmapFile, + header, + header::{Header, Timestamp}, + }; use lib::tracedat::{ check::check_header, parquet::dump::{dump_events, dump_metadata}, print::print_events, }; - use traceevent::{ - header, - header::{Header, Timestamp}, - io::MmapFile, - }; let open_trace = |path| -> Result<(Header, Box<_>), DynMultiError> { let file = std::fs::File::open(path)?; let reader = unsafe { MmapFile::new(file) }?; let mut reader = Box::new(reader); let header = header::header(reader.deref_mut())?; - Ok((header, reader)) + return Ok((header, reader)); }; match &cli.command { @@ -230,7 +233,6 @@ where unique_timestamps, compression, row_group_size, - max_errors, .. } => { let (header, reader) = open_trace(trace)?; @@ -250,15 +252,7 @@ where }; let compression = compression.clone().into(); - match dump_events( - &header, - reader, - make_ts, - event.clone(), - *row_group_size, - compression, - *max_errors, - ) { + match dump_events(&header, reader, make_ts, event.clone(), *row_group_size, compression) { Ok(metadata) => Ok(metadata.dump(File::create("meta.json")?)?), Err(err) => Err(err), } @@ -267,14 +261,10 @@ where let (header, _) = open_trace(trace)?; check_header(&header, out) } - Command::Metadata { - trace, - key, - max_errors, - .. - } => { + Command::Metadata { trace, key, .. } => { let (header, reader) = open_trace(trace)?; - dump_metadata(&header, reader, out, key.clone(), *max_errors) + dump_metadata(&header, reader, out, key.clone()) } } } + diff --git a/tools/trace-parser/trace-tools/src/lib/error.rs b/tools/trace-parser/trace-tools/src/lib/error.rs index ed38ea70f..526db809b 100644 --- a/tools/trace-parser/trace-tools/src/lib/error.rs +++ b/tools/trace-parser/trace-tools/src/lib/error.rs @@ -80,3 +80,4 @@ impl From for DynMultiError { }) } } + -- GitLab From a6a6034f1f049713cf4b50072ae8ad13710b9c5b Mon Sep 17 00:00:00 2001 From: Douglas Raillard Date: Wed, 19 Jun 2024 15:51:51 +0100 Subject: [PATCH 2/3] tools/trace-parser/trace-tools: Add Perfetto trace support --- install_base.sh | 2 +- tools/trace-parser/trace-tools/Cargo.toml | 11 ++- tools/trace-parser/trace-tools/build.rs | 36 ++++++++ .../trace-tools/src/bin/trace-dump.rs | 84 +++++++++++++++++++ .../trace-tools/src/lib/perfetto/mod.rs | 17 ++++ .../trace-tools/src/lib/perfetto/parquet.rs | 16 ++++ 6 files changed, 163 insertions(+), 3 deletions(-) create mode 100644 tools/trace-parser/trace-tools/build.rs create mode 100644 tools/trace-parser/trace-tools/src/lib/perfetto/mod.rs create mode 100644 tools/trace-parser/trace-tools/src/lib/perfetto/parquet.rs diff --git a/install_base.sh b/install_base.sh index 893689ded..677dc0bfd 100755 --- a/install_base.sh +++ b/install_base.sh @@ -258,7 +258,7 @@ for arg in "${args[@]}"; do pacman_packages+=(base-devel aarch64-linux-gnu-gcc flex) # Build dependencies of some assets - apt_packages+=(autopoint autoconf libtool bison flex cmake) + apt_packages+=(autopoint autoconf libtool bison flex cmake protobuf-compiler) # gettext for autopoint pacman_packages+=(gettext autoconf libtool bison cmake) diff --git a/tools/trace-parser/trace-tools/Cargo.toml b/tools/trace-parser/trace-tools/Cargo.toml index 729f4fa35..f10e2cc88 100644 --- a/tools/trace-parser/trace-tools/Cargo.toml +++ b/tools/trace-parser/trace-tools/Cargo.toml @@ -25,10 +25,17 @@ serde = { version = "1.0", features = ["derive"] } nom = "8.0" bytemuck = "1.13" clap = { version = "4.4", features = ["derive"] } +prost = { version = "0.12", optional = true } +bytes = { version = "1.6", optional = true } +memmap2 = { version = "0.9", optional = true } + +[build-dependencies] +prost-build = { version = "0.12", optional = true } [features] -default = ["tracedat"] -tracedat = ["dep:traceevent"] +default = [ "tracedat", "perfetto" ] +tracedat = [ "dep:traceevent" ] +perfetto = [ "dep:prost", "dep:prost-build", "dep:bytes", "dep:memmap2" ] [target.'cfg(target_arch = "x86_64")'.dependencies] mimalloc = {version = "0.1", default-features = false } diff --git a/tools/trace-parser/trace-tools/build.rs b/tools/trace-parser/trace-tools/build.rs new file mode 100644 index 000000000..974f2a84c --- /dev/null +++ b/tools/trace-parser/trace-tools/build.rs @@ -0,0 +1,36 @@ +// SPDX-License-Identifier: Apache-2.0 +// +// Copyright (C) 2024, ARM Limited and contributors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::io::Result; +use std::path::Path; +use std::env; + +fn main() -> Result<()> { + #[cfg(feature = "perfetto")] + protobuf()?; + + Ok(()) +} + +#[cfg(feature = "perfetto")] +fn protobuf() -> Result<()> { + let proto_root = env::var("PERFETTO_PROTO_ROOT").expect("PERFETTO_PROTO_ROOT env var must be set"); + let proto_root = Path::new(&proto_root); + + let proto = proto_root.join("perfetto_trace.proto"); + prost_build::compile_protos(&[proto], &[proto_root])?; + Ok(()) +} diff --git a/tools/trace-parser/trace-tools/src/bin/trace-dump.rs b/tools/trace-parser/trace-tools/src/bin/trace-dump.rs index 28344849e..ce43edb56 100644 --- a/tools/trace-parser/trace-tools/src/bin/trace-dump.rs +++ b/tools/trace-parser/trace-tools/src/bin/trace-dump.rs @@ -68,6 +68,7 @@ impl From for ParquetCompression { #[clap(rename_all = "lower")] enum TraceFormat { TraceDat, + Perfetto, } #[derive(Subcommand)] @@ -157,6 +158,9 @@ fn _main() -> Result<(), Box> { #[cfg(feature = "tracedat")] Some(TraceFormat::TraceDat) => tracedat_main(&cli, &mut out), + #[cfg(feature = "perfetto")] + Some(TraceFormat::Perfetto) => perfetto_main(&cli, &mut out), + _ => Err(DynMultiError::from_string("File format not handled".into())) }; @@ -268,3 +272,83 @@ fn tracedat_main(cli: &Cli, out: &mut W) -> Result<(), DynMultiError> } } +#[cfg(feature = "perfetto")] +fn perfetto_main(cli: &Cli, out: &mut W) -> Result<(), DynMultiError> +where + W: Write +{ + let trace = match &cli.command { + Command::Parquet { + trace, + .. + } => Ok(trace), + _ => Err(DynMultiError::from_string("File format not handled".into())) + }?; + + let mut f = File::open(trace)?; + // let mut content: Vec = Vec::new(); + // use std::io::Read; + // f.read_to_end(&mut content)?; + + + //SAFETY: mmap is inherently unsafe as the memory content could change + // without notice if the backing file is modified. We have to rely on + // the user/OS being nice to us and not do that, or we might crash, + // there is no way around it unfortunately. + let mmap = unsafe { + memmap2::MmapOptions::new() + // .offset(0) + // .len(len) + .map(&f) + }?; + + // This MADV_WILLNEED is equivalent to MAP_POPULATE in terms of enabling read-ahead but + // will not trigger a complete read in memory upon creation of the mapping. This + // dramatically lowers the reported RES memory consumption at no performance cost. + let _ = mmap.advise(memmap2::Advice::WillNeed); + let _ = mmap.advise(memmap2::Advice::Sequential); + + let mut content = &mmap[..]; + + + pub mod perfettoproto { + include!(concat!(env!("OUT_DIR"), "/perfetto.protos.rs")); + } + + // Count 94232 + // let trace = perfettoproto::Trace::decode(&*content)?; + // for packet in trace.packet { + // writeln!(out, "{packet:#?}")?; + // } + // // writeln!(out, "{trace:#?}")?; + // return Ok(()); + + + use prost::Message; + let mut packet: perfettoproto::TracePacket = Default::default(); + + // FtraceEvent count: 94232 + use bytes::Buf; + let mut buf = &mut content; + while buf.len() != 0 { + use prost::encoding::decode_varint; + + // TracePacket are encoded as if they were a repeated field in a parent Trace message. + // That means the field tag in the parent Trace is encoded, followed by the varint + // specifying the length of the submessage, followed by the TracePacket messsage itself. + let tag = decode_varint(&mut buf)?; + let len = decode_varint(&mut buf)?; + let len: usize = len.try_into().unwrap(); + packet.merge(&buf[..len])?; + buf.advance(len); + + writeln!(out, "{packet:#?}")?; + + packet.clear(); + } + + // let trace = perfettoproto::Trace::decode(content)?; + // writeln!(out, "{trace:#?}")?; + + Ok(()) +} diff --git a/tools/trace-parser/trace-tools/src/lib/perfetto/mod.rs b/tools/trace-parser/trace-tools/src/lib/perfetto/mod.rs new file mode 100644 index 000000000..419e0671e --- /dev/null +++ b/tools/trace-parser/trace-tools/src/lib/perfetto/mod.rs @@ -0,0 +1,17 @@ +// SPDX-License-Identifier: Apache-2.0 +// +// Copyright (C) 2024, ARM Limited and contributors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +pub mod parquet; diff --git a/tools/trace-parser/trace-tools/src/lib/perfetto/parquet.rs b/tools/trace-parser/trace-tools/src/lib/perfetto/parquet.rs new file mode 100644 index 000000000..20d250ea1 --- /dev/null +++ b/tools/trace-parser/trace-tools/src/lib/perfetto/parquet.rs @@ -0,0 +1,16 @@ +// SPDX-License-Identifier: Apache-2.0 +// +// Copyright (C) 2024, ARM Limited and contributors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + -- GitLab From f30cac0fcf19cfe6c0210643854a8ca73deaabbc Mon Sep 17 00:00:00 2001 From: Douglas Raillard Date: Mon, 1 Jul 2024 11:16:32 +0100 Subject: [PATCH 3/3] WIPprocmacro --- .../trace-tools-proc-macro/Cargo.lock | 25 +++++++++++++++++++ .../trace-tools-proc-macro/Cargo.toml | 10 ++++++++ .../trace-tools-proc-macro/src/lib.rs | 3 +++ 3 files changed, 38 insertions(+) create mode 100644 tools/trace-parser/trace-tools-proc-macro/Cargo.lock create mode 100644 tools/trace-parser/trace-tools-proc-macro/Cargo.toml create mode 100644 tools/trace-parser/trace-tools-proc-macro/src/lib.rs diff --git a/tools/trace-parser/trace-tools-proc-macro/Cargo.lock b/tools/trace-parser/trace-tools-proc-macro/Cargo.lock new file mode 100644 index 000000000..a5da7ffae --- /dev/null +++ b/tools/trace-parser/trace-tools-proc-macro/Cargo.lock @@ -0,0 +1,25 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "proc-macro2" +version = "1.0.86" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "trace-tools-proc-macro" +version = "0.1.0" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "unicode-ident" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" diff --git a/tools/trace-parser/trace-tools-proc-macro/Cargo.toml b/tools/trace-parser/trace-tools-proc-macro/Cargo.toml new file mode 100644 index 000000000..4b10f63b4 --- /dev/null +++ b/tools/trace-parser/trace-tools-proc-macro/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "trace-tools-proc-macro" +version = "0.1.0" +edition = "2021" + +[lib] +proc-macro = true + +[dependencies] +proc-macro2 = "1.0" diff --git a/tools/trace-parser/trace-tools-proc-macro/src/lib.rs b/tools/trace-parser/trace-tools-proc-macro/src/lib.rs new file mode 100644 index 000000000..e7a11a969 --- /dev/null +++ b/tools/trace-parser/trace-tools-proc-macro/src/lib.rs @@ -0,0 +1,3 @@ +fn main() { + println!("Hello, world!"); +} -- GitLab