diff --git a/install_base.sh b/install_base.sh index 893689deda50163ef78b6dd825fce7ae58fc49c4..677dc0bfd90ad65ee4210a1a548cab9b48c42971 100755 --- a/install_base.sh +++ b/install_base.sh @@ -258,7 +258,7 @@ for arg in "${args[@]}"; do pacman_packages+=(base-devel aarch64-linux-gnu-gcc flex) # Build dependencies of some assets - apt_packages+=(autopoint autoconf libtool bison flex cmake) + apt_packages+=(autopoint autoconf libtool bison flex cmake protobuf-compiler) # gettext for autopoint pacman_packages+=(gettext autoconf libtool bison cmake) diff --git a/tools/trace-parser/trace-tools-proc-macro/Cargo.lock b/tools/trace-parser/trace-tools-proc-macro/Cargo.lock new file mode 100644 index 0000000000000000000000000000000000000000..a5da7ffae295af65adaa019a9c9da1cc5739f7fd --- /dev/null +++ b/tools/trace-parser/trace-tools-proc-macro/Cargo.lock @@ -0,0 +1,25 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "proc-macro2" +version = "1.0.86" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "trace-tools-proc-macro" +version = "0.1.0" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "unicode-ident" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" diff --git a/tools/trace-parser/trace-tools-proc-macro/Cargo.toml b/tools/trace-parser/trace-tools-proc-macro/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..4b10f63b4443de40fc4ee4c8d38601ffff06b57d --- /dev/null +++ b/tools/trace-parser/trace-tools-proc-macro/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "trace-tools-proc-macro" +version = "0.1.0" +edition = "2021" + +[lib] +proc-macro = true + +[dependencies] +proc-macro2 = "1.0" diff --git a/tools/trace-parser/trace-tools-proc-macro/src/lib.rs b/tools/trace-parser/trace-tools-proc-macro/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..e7a11a969c037e00a796aafeff6258501ec15e9a --- /dev/null +++ b/tools/trace-parser/trace-tools-proc-macro/src/lib.rs @@ -0,0 +1,3 @@ +fn main() { + println!("Hello, world!"); +} diff --git a/tools/trace-parser/trace-tools/Cargo.toml b/tools/trace-parser/trace-tools/Cargo.toml index 729f4fa354d82d0b5a5000989ef1f73ec949915a..f10e2cc88588c09b4a00613d5aaede3ec7939ee0 100644 --- a/tools/trace-parser/trace-tools/Cargo.toml +++ b/tools/trace-parser/trace-tools/Cargo.toml @@ -25,10 +25,17 @@ serde = { version = "1.0", features = ["derive"] } nom = "8.0" bytemuck = "1.13" clap = { version = "4.4", features = ["derive"] } +prost = { version = "0.12", optional = true } +bytes = { version = "1.6", optional = true } +memmap2 = { version = "0.9", optional = true } + +[build-dependencies] +prost-build = { version = "0.12", optional = true } [features] -default = ["tracedat"] -tracedat = ["dep:traceevent"] +default = [ "tracedat", "perfetto" ] +tracedat = [ "dep:traceevent" ] +perfetto = [ "dep:prost", "dep:prost-build", "dep:bytes", "dep:memmap2" ] [target.'cfg(target_arch = "x86_64")'.dependencies] mimalloc = {version = "0.1", default-features = false } diff --git a/tools/trace-parser/trace-tools/build.rs b/tools/trace-parser/trace-tools/build.rs new file mode 100644 index 0000000000000000000000000000000000000000..974f2a84cf678f964617bf85ca7856362031548c --- /dev/null +++ b/tools/trace-parser/trace-tools/build.rs @@ -0,0 +1,36 @@ +// SPDX-License-Identifier: Apache-2.0 +// +// Copyright (C) 2024, ARM Limited and contributors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::io::Result; +use std::path::Path; +use std::env; + +fn main() -> Result<()> { + #[cfg(feature = "perfetto")] + protobuf()?; + + Ok(()) +} + +#[cfg(feature = "perfetto")] +fn protobuf() -> Result<()> { + let proto_root = env::var("PERFETTO_PROTO_ROOT").expect("PERFETTO_PROTO_ROOT env var must be set"); + let proto_root = Path::new(&proto_root); + + let proto = proto_root.join("perfetto_trace.proto"); + prost_build::compile_protos(&[proto], &[proto_root])?; + Ok(()) +} diff --git a/tools/trace-parser/trace-tools/src/bin/trace-dump.rs b/tools/trace-parser/trace-tools/src/bin/trace-dump.rs index ccf9b6bb594a56b9e5a37dba0ea16672f18d221d..ce43edb5680d97235735accd389d459f0dd3d64a 100644 --- a/tools/trace-parser/trace-tools/src/bin/trace-dump.rs +++ b/tools/trace-parser/trace-tools/src/bin/trace-dump.rs @@ -14,7 +14,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::{error::Error, fs::File, io::Write, path::PathBuf, process::ExitCode}; +use std::{ + error::Error, fs::File, io::Write, path::PathBuf, process::ExitCode, +}; #[cfg(target_arch = "x86_64")] use mimalloc::MiMalloc; @@ -23,7 +25,9 @@ use mimalloc::MiMalloc; static GLOBAL: MiMalloc = MiMalloc; use clap::{Parser, Subcommand, ValueEnum}; -use lib::error::DynMultiError; +use lib::{ + error::DynMultiError, +}; use parquet::basic::{Compression as ParquetCompression, ZstdLevel}; #[derive(Parser)] @@ -64,6 +68,7 @@ impl From for ParquetCompression { #[clap(rename_all = "lower")] enum TraceFormat { TraceDat, + Perfetto, } #[derive(Subcommand)] @@ -135,10 +140,10 @@ enum Command { impl Command { fn trace_format(&self) -> Option { match self { - Command::HumanReadable { trace_format, .. } => Some(trace_format.clone()), - Command::Parquet { trace_format, .. } => Some(trace_format.clone()), - Command::CheckHeader { trace_format, .. } => Some(trace_format.clone()), - Command::Metadata { trace_format, .. } => Some(trace_format.clone()), + Command::HumanReadable {trace_format, ..} => Some(trace_format.clone()), + Command::Parquet {trace_format, ..} => Some(trace_format.clone()), + Command::CheckHeader {trace_format, ..} => Some(trace_format.clone()), + Command::Metadata {trace_format, ..} => Some(trace_format.clone()), } } } @@ -153,7 +158,10 @@ fn _main() -> Result<(), Box> { #[cfg(feature = "tracedat")] Some(TraceFormat::TraceDat) => tracedat_main(&cli, &mut out), - _ => Err(DynMultiError::from_string("File format not handled".into())), + #[cfg(feature = "perfetto")] + Some(TraceFormat::Perfetto) => perfetto_main(&cli, &mut out), + + _ => Err(DynMultiError::from_string("File format not handled".into())) }; out.flush()?; @@ -195,28 +203,27 @@ fn main() -> ExitCode { #[cfg(feature = "tracedat")] fn tracedat_main(cli: &Cli, out: &mut W) -> Result<(), DynMultiError> -where - W: Write, + where W: Write { use std::ops::DerefMut as _; + use traceevent::{ + io::MmapFile, + header, + header::{Header, Timestamp}, + }; use lib::tracedat::{ check::check_header, parquet::dump::{dump_events, dump_metadata}, print::print_events, }; - use traceevent::{ - header, - header::{Header, Timestamp}, - io::MmapFile, - }; let open_trace = |path| -> Result<(Header, Box<_>), DynMultiError> { let file = std::fs::File::open(path)?; let reader = unsafe { MmapFile::new(file) }?; let mut reader = Box::new(reader); let header = header::header(reader.deref_mut())?; - Ok((header, reader)) + return Ok((header, reader)); }; match &cli.command { @@ -230,7 +237,6 @@ where unique_timestamps, compression, row_group_size, - max_errors, .. } => { let (header, reader) = open_trace(trace)?; @@ -250,15 +256,7 @@ where }; let compression = compression.clone().into(); - match dump_events( - &header, - reader, - make_ts, - event.clone(), - *row_group_size, - compression, - *max_errors, - ) { + match dump_events(&header, reader, make_ts, event.clone(), *row_group_size, compression) { Ok(metadata) => Ok(metadata.dump(File::create("meta.json")?)?), Err(err) => Err(err), } @@ -267,14 +265,90 @@ where let (header, _) = open_trace(trace)?; check_header(&header, out) } - Command::Metadata { - trace, - key, - max_errors, - .. - } => { + Command::Metadata { trace, key, .. } => { let (header, reader) = open_trace(trace)?; - dump_metadata(&header, reader, out, key.clone(), *max_errors) + dump_metadata(&header, reader, out, key.clone()) } } } + +#[cfg(feature = "perfetto")] +fn perfetto_main(cli: &Cli, out: &mut W) -> Result<(), DynMultiError> +where + W: Write +{ + let trace = match &cli.command { + Command::Parquet { + trace, + .. + } => Ok(trace), + _ => Err(DynMultiError::from_string("File format not handled".into())) + }?; + + let mut f = File::open(trace)?; + // let mut content: Vec = Vec::new(); + // use std::io::Read; + // f.read_to_end(&mut content)?; + + + //SAFETY: mmap is inherently unsafe as the memory content could change + // without notice if the backing file is modified. We have to rely on + // the user/OS being nice to us and not do that, or we might crash, + // there is no way around it unfortunately. + let mmap = unsafe { + memmap2::MmapOptions::new() + // .offset(0) + // .len(len) + .map(&f) + }?; + + // This MADV_WILLNEED is equivalent to MAP_POPULATE in terms of enabling read-ahead but + // will not trigger a complete read in memory upon creation of the mapping. This + // dramatically lowers the reported RES memory consumption at no performance cost. + let _ = mmap.advise(memmap2::Advice::WillNeed); + let _ = mmap.advise(memmap2::Advice::Sequential); + + let mut content = &mmap[..]; + + + pub mod perfettoproto { + include!(concat!(env!("OUT_DIR"), "/perfetto.protos.rs")); + } + + // Count 94232 + // let trace = perfettoproto::Trace::decode(&*content)?; + // for packet in trace.packet { + // writeln!(out, "{packet:#?}")?; + // } + // // writeln!(out, "{trace:#?}")?; + // return Ok(()); + + + use prost::Message; + let mut packet: perfettoproto::TracePacket = Default::default(); + + // FtraceEvent count: 94232 + use bytes::Buf; + let mut buf = &mut content; + while buf.len() != 0 { + use prost::encoding::decode_varint; + + // TracePacket are encoded as if they were a repeated field in a parent Trace message. + // That means the field tag in the parent Trace is encoded, followed by the varint + // specifying the length of the submessage, followed by the TracePacket messsage itself. + let tag = decode_varint(&mut buf)?; + let len = decode_varint(&mut buf)?; + let len: usize = len.try_into().unwrap(); + packet.merge(&buf[..len])?; + buf.advance(len); + + writeln!(out, "{packet:#?}")?; + + packet.clear(); + } + + // let trace = perfettoproto::Trace::decode(content)?; + // writeln!(out, "{trace:#?}")?; + + Ok(()) +} diff --git a/tools/trace-parser/trace-tools/src/lib/error.rs b/tools/trace-parser/trace-tools/src/lib/error.rs index ed38ea70f9da78131467e4dbe777f75792cb4a14..526db809b74b50b82fd01dd31104416df1cff927 100644 --- a/tools/trace-parser/trace-tools/src/lib/error.rs +++ b/tools/trace-parser/trace-tools/src/lib/error.rs @@ -80,3 +80,4 @@ impl From for DynMultiError { }) } } + diff --git a/tools/trace-parser/trace-tools/src/lib/perfetto/mod.rs b/tools/trace-parser/trace-tools/src/lib/perfetto/mod.rs new file mode 100644 index 0000000000000000000000000000000000000000..419e0671e8feed91e07a68f66b541019813a02ae --- /dev/null +++ b/tools/trace-parser/trace-tools/src/lib/perfetto/mod.rs @@ -0,0 +1,17 @@ +// SPDX-License-Identifier: Apache-2.0 +// +// Copyright (C) 2024, ARM Limited and contributors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +pub mod parquet; diff --git a/tools/trace-parser/trace-tools/src/lib/perfetto/parquet.rs b/tools/trace-parser/trace-tools/src/lib/perfetto/parquet.rs new file mode 100644 index 0000000000000000000000000000000000000000..20d250ea1b9dd92fa68109603b1e91b93dc82f1c --- /dev/null +++ b/tools/trace-parser/trace-tools/src/lib/perfetto/parquet.rs @@ -0,0 +1,16 @@ +// SPDX-License-Identifier: Apache-2.0 +// +// Copyright (C) 2024, ARM Limited and contributors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +