Merge remote-tracking branch 'remotes/stefanha/tags/tracing-pull-request' into staging

Pull request

User-visible changes:
 * The new qemu-trace-stap script makes it convenient to collect traces without
   writing SystemTap scripts.  See "man qemu-trace-stap" for details.

# gpg: Signature made Wed 30 Jan 2019 03:17:57 GMT
# gpg:                using RSA key 9CA4ABB381AB73C8
# gpg: Good signature from "Stefan Hajnoczi <stefanha@redhat.com>" [full]
# gpg:                 aka "Stefan Hajnoczi <stefanha@gmail.com>" [full]
# Primary key fingerprint: 8695 A8BF D3F9 7CDA AC35  775A 9CA4 ABB3 81AB 73C8

* remotes/stefanha/tags/tracing-pull-request:
  trace: rerun tracetool after ./configure changes
  trace: improve runstate tracing
  trace: add ability to do simple printf logging via systemtap
  trace: forbid use of %m in trace event format strings
  trace: enforce that every trace-events file has a final newline
  display: ensure qxl log_buf is a nul terminated string

Signed-off-by: Peter Maydell <peter.maydell@linaro.org>

4 files changed, 448 insertions, 0 deletions

diff --git a/scripts/qemu-trace-stap b/scripts/qemu-trace-stap
new file mode 100755
index 0000000000..91d1051cdc
--- /dev/null
+++ b/scripts/qemu-trace-stap
@@ -0,0 +1,175 @@
+#!/usr/bin/python
+# -*- python -*-
+#
+# Copyright (C) 2019 Red Hat, Inc
+#
+# QEMU SystemTap Trace Tool
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, see <http://www.gnu.org/licenses/>.
+
+from __future__ import print_function
+
+import argparse
+import copy
+import os.path
+import re
+import subprocess
+import sys
+
+
+def probe_prefix(binary):
+    dirname, filename = os.path.split(binary)
+    return re.sub("-", ".", filename) + ".log"
+
+
+def which(binary):
+    for path in os.environ["PATH"].split(os.pathsep):
+        if os.path.exists(os.path.join(path, binary)):
+                return os.path.join(path, binary)
+
+    print("Unable to find '%s' in $PATH" % binary)
+    sys.exit(1)
+
+
+def tapset_dir(binary):
+    dirname, filename = os.path.split(binary)
+    if dirname == '':
+        thisfile = which(binary)
+    else:
+        thisfile = os.path.realpath(binary)
+        if not os.path.exists(thisfile):
+            print("Unable to find '%s'" % thisfile)
+            sys.exit(1)
+
+    basedir = os.path.split(thisfile)[0]
+    tapset = os.path.join(basedir, "..", "share", "systemtap", "tapset")
+    return os.path.realpath(tapset)
+
+
+def tapset_env(tapset_dir):
+    tenv = copy.copy(os.environ)
+    tenv["SYSTEMTAP_TAPSET"] = tapset_dir
+    return tenv
+
+def cmd_run(args):
+    prefix = probe_prefix(args.binary)
+    tapsets = tapset_dir(args.binary)
+
+    if args.verbose:
+        print("Using tapset dir '%s' for binary '%s'" % (tapsets, args.binary))
+
+    probes = []
+    for probe in args.probes:
+        probes.append("probe %s.%s {}" % (prefix, probe))
+    if len(probes) == 0:
+        print("At least one probe pattern must be specified")
+        sys.exit(1)
+
+    script = " ".join(probes)
+    if args.verbose:
+        print("Compiling script '%s'" % script)
+        script = """probe begin { print("Running script, <Ctrl>-c to quit\\n") } """ + script
+
+    # We request an 8MB buffer, since the stap default 1MB buffer
+    # can be easily overflowed by frequently firing QEMU traces
+    stapargs = ["stap", "-s", "8"]
+    if args.pid is not None:
+        stapargs.extend(["-x", args.pid])
+    stapargs.extend(["-e", script])
+    subprocess.call(stapargs, env=tapset_env(tapsets))
+
+
+def cmd_list(args):
+    tapsets = tapset_dir(args.binary)
+
+    if args.verbose:
+        print("Using tapset dir '%s' for binary '%s'" % (tapsets, args.binary))
+
+    def print_probes(verbose, name):
+        prefix = probe_prefix(args.binary)
+        offset = len(prefix) + 1
+        script = prefix + "." + name
+
+        if verbose:
+            print("Listing probes with name '%s'" % script)
+        proc = subprocess.Popen(["stap", "-l", script],
+                                stdout=subprocess.PIPE, env=tapset_env(tapsets))
+        out, err = proc.communicate()
+        if proc.returncode != 0:
+            print("No probes found, are the tapsets installed in %s" % tapset_dir(args.binary))
+            sys.exit(1)
+
+        for line in out.splitlines():
+            if line.startswith(prefix):
+                print("%s" % line[offset:])
+
+    if len(args.probes) == 0:
+        print_probes(args.verbose, "*")
+    else:
+        for probe in args.probes:
+            print_probes(args.verbose, probe)
+
+
+def main():
+    parser = argparse.ArgumentParser(description="QEMU SystemTap trace tool")
+    parser.add_argument("-v", "--verbose", help="Print verbose progress info",
+                        action='store_true')
+
+    subparser = parser.add_subparsers(help="commands")
+    subparser.required = True
+    subparser.dest = "command"
+
+    runparser = subparser.add_parser("run", help="Run a trace session",
+                                     formatter_class=argparse.RawDescriptionHelpFormatter,
+                                     epilog="""
+
+To watch all trace points on the qemu-system-x86_64 binary:
+
+   %(argv0)s run qemu-system-x86_64
+
+To only watch the trace points matching the qio* and qcrypto* patterns
+
+   %(argv0)s run qemu-system-x86_64 'qio*' 'qcrypto*'
+""" % {"argv0": sys.argv[0]})
+    runparser.set_defaults(func=cmd_run)
+    runparser.add_argument("--pid", "-p", dest="pid",
+                           help="Restrict tracing to a specific process ID")
+    runparser.add_argument("binary", help="QEMU system or user emulator binary")
+    runparser.add_argument("probes", help="Probe names or wildcards",
+                           nargs=argparse.REMAINDER)
+
+    listparser = subparser.add_parser("list", help="List probe points",
+                                      formatter_class=argparse.RawDescriptionHelpFormatter,
+                                      epilog="""
+
+To list all trace points on the qemu-system-x86_64 binary:
+
+   %(argv0)s list qemu-system-x86_64
+
+To only list the trace points matching the qio* and qcrypto* patterns
+
+   %(argv0)s list qemu-system-x86_64 'qio*' 'qcrypto*'
+""" % {"argv0": sys.argv[0]})
+    listparser.set_defaults(func=cmd_list)
+    listparser.add_argument("binary", help="QEMU system or user emulator binary")
+    listparser.add_argument("probes", help="Probe names or wildcards",
+                            nargs=argparse.REMAINDER)
+
+    args = parser.parse_args()
+
+    args.func(args)
+    sys.exit(0)
+
+if __name__ == '__main__':
+    main()
diff --git a/scripts/qemu-trace-stap.texi b/scripts/qemu-trace-stap.texi
new file mode 100644
index 0000000000..07bb9eb94e
--- /dev/null
+++ b/scripts/qemu-trace-stap.texi
@@ -0,0 +1,140 @@
+@example
+@c man begin SYNOPSIS
+@command{qemu-trace-stap} @var{GLOBAL-OPTIONS} @var{COMMAND} @var{COMMAND-OPTIONS} @var{ARGS...}
+@c man end
+@end example
+
+@c man begin DESCRIPTION
+
+The @command{qemu-trace-stap} program facilitates tracing of the execution
+of QEMU emulators using SystemTap.
+
+It is required to have the SystemTap runtime environment installed to use
+this program, since it is a wrapper around execution of the @command{stap}
+program.
+
+@c man end
+
+@c man begin OPTIONS
+
+The following global options may be used regardless of which command
+is executed:
+
+@table @option
+@item @var{--verbose}, @var{-v}
+
+Display verbose information about command execution.
+
+@end table
+
+The following commands are valid:
+
+@table @option
+
+@item @var{list} @var{BINARY} @var{PATTERN...}
+
+List all the probe names provided by @var{BINARY} that match
+@var{PATTERN}.
+
+If @var{BINARY} is not an absolute path, it will be located by searching
+the directories listed in the @code{$PATH} environment variable.
+
+@var{PATTERN} is a plain string that is used to filter the results of
+this command. It may optionally contain a @code{*} wildcard to facilitate
+matching multiple probes without listing each one explicitly. Multiple
+@var{PATTERN} arguments may be given, causing listing of probes that match
+any of the listed names. If no @var{PATTERN} is given, the all possible
+probes will be listed.
+
+For example, to list all probes available in the @command{qemu-system-x86_64}
+binary:
+
+@example
+$ qemu-trace-stap list qemu-system-x86_64
+@end example
+
+To filter the list to only cover probes related to QEMU's cryptographic
+subsystem, in a binary outside @code{$PATH}
+
+@example
+$ qemu-trace-stap list /opt/qemu/4.0.0/bin/qemu-system-x86_64 'qcrypto*'
+@end example
+
+
+@item @var{run} @var{OPTIONS} @var{BINARY} @var{PATTERN...}
+
+Run a trace session, printing formatted output any time a process that is
+executing @var{BINARY} triggers a probe matching @var{PATTERN}.
+
+If @var{BINARY} is not an absolute path, it will be located by searching
+the directories listed in the @code{$PATH} environment variable.
+
+@var{PATTERN} is a plain string that matches a probe name shown by the
+@var{list} command. It may optionally contain a @code{*} wildcard to
+facilitate matching multiple probes without listing each one explicitly.
+Multiple @var{PATTERN} arguments may be given, causing all matching probes
+to be monitored. At least one @var{PATTERN} is required, since stap is not
+capable of tracing all known QEMU probes concurrently without overflowing
+its trace buffer.
+
+Invocation of this command does not need to be synchronized with
+invocation of the QEMU process(es). It will match probes on all
+existing running processes and all future launched processes,
+unless told to only monitor a specific process.
+
+Valid command specific options are:
+
+@table @option
+@item @var{--pid=PID}, @var{-p PID}
+
+Restrict the tracing session so that it only triggers for the process
+identified by @code{PID}.
+
+@end table
+
+For example, to monitor all processes executing @command{qemu-system-x86_64}
+as found on $PATH, displaying all I/O related probes:
+
+@example
+$ qemu-trace-stap run qemu-system-x86_64 'qio*'
+@end example
+
+To monitor only the QEMU process with PID 1732
+
+@example
+$ qemu-trace-stap run --pid=1732 qemu-system-x86_64 'qio*'
+@end example
+
+To monitor QEMU processes running an alternative binary outside of
+@code{$PATH}, displaying verbose information about setup of the
+tracing environment:
+
+@example
+$ qemu-trace-stap -v run /opt/qemu/4.0.0/qemu-system-x86_64 'qio*'
+@end example
+
+@end table
+
+@c man end
+
+@ignore
+
+@setfilename qemu-trace-stap
+@settitle QEMU SystemTap trace tool
+
+@c man begin LICENSE
+
+Copyright (C) 2019 Red Hat, Inc.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+
+@c man end
+
+@c man begin SEEALSO
+qemu(1), stap(1)
+@c man end
+
+@end ignore
diff --git a/scripts/tracetool/__init__.py b/scripts/tracetool/__init__.py
index 0e3c9e146c..6fca674936 100644
--- a/scripts/tracetool/__init__.py
+++ b/scripts/tracetool/__init__.py
@@ -274,6 +274,10 @@ class Event(object):
         props = groups["props"].split()
         fmt = groups["fmt"]
         fmt_trans = groups["fmt_trans"]
+        if fmt.find("%m") != -1 or fmt_trans.find("%m") != -1:
+            raise ValueError("Event format '%m' is forbidden, pass the error "
+                             "as an explicit trace argument")
+
         if len(fmt_trans) > 0:
             fmt = [fmt_trans, fmt]
         args = Arguments.build(groups["args"])
@@ -350,6 +354,8 @@ def read_events(fobj, fname):
 
     events = []
     for lineno, line in enumerate(fobj, 1):
+        if line[-1] != '\n':
+            raise ValueError("%s does not end with a new line" % fname)
         if not line.strip():
             continue
         if line.lstrip().startswith('#'):
diff --git a/scripts/tracetool/format/log_stap.py b/scripts/tracetool/format/log_stap.py
new file mode 100644
index 0000000000..3ccbc09d61
--- /dev/null
+++ b/scripts/tracetool/format/log_stap.py
@@ -0,0 +1,127 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+Generate .stp file that printfs log messages (DTrace with SystemTAP only).
+"""
+
+__author__     = "Daniel P. Berrange <berrange@redhat.com>"
+__copyright__  = "Copyright (C) 2014-2019, Red Hat, Inc."
+__license__    = "GPL version 2 or (at your option) any later version"
+
+__maintainer__ = "Daniel Berrange"
+__email__      = "berrange@redhat.com"
+
+import re
+
+from tracetool import out
+from tracetool.backend.dtrace import binary, probeprefix
+from tracetool.backend.simple import is_string
+from tracetool.format.stap import stap_escape
+
+def global_var_name(name):
+    return probeprefix().replace(".", "_") + "_" + name
+
+STATE_SKIP = 0
+STATE_LITERAL = 1
+STATE_MACRO = 2
+
+def c_macro_to_format(macro):
+    if macro.startswith("PRI"):
+        return macro[3]
+
+    if macro == "TARGET_FMT_plx":
+        return "%016x"
+
+    raise Exception("Unhandled macro '%s'" % macro)
+
+def c_fmt_to_stap(fmt):
+    state = 0
+    bits = []
+    literal = ""
+    macro = ""
+    escape = 0;
+    for i in range(len(fmt)):
+        if fmt[i] == '\\':
+            if escape:
+                escape = 0
+            else:
+                escape = 1
+            if state != STATE_LITERAL:
+                raise Exception("Unexpected escape outside string literal")
+            literal = literal + fmt[i]
+        elif fmt[i] == '"' and not escape:
+            if state == STATE_LITERAL:
+                state = STATE_SKIP
+                bits.append(literal)
+                literal = ""
+            else:
+                if state == STATE_MACRO:
+                    bits.append(c_macro_to_format(macro))
+                state = STATE_LITERAL
+        elif fmt[i] == ' ' or fmt[i] == '\t':
+            if state == STATE_MACRO:
+                bits.append(c_macro_to_format(macro))
+                macro = ""
+                state = STATE_SKIP
+            elif state == STATE_LITERAL:
+                literal = literal + fmt[i]
+        else:
+            escape = 0
+            if state == STATE_SKIP:
+                state = STATE_MACRO
+
+            if state == STATE_LITERAL:
+                literal = literal + fmt[i]
+            else:
+                macro = macro + fmt[i]
+
+    if state == STATE_MACRO:
+        bits.append(c_macro_to_format(macro))
+    elif state == STATE_LITERAL:
+        bits.append(literal)
+
+    fmt = re.sub("%(\d*)z(x|u|d)", "%\\1\\2", "".join(bits))
+    return fmt
+
+def generate(events, backend, group):
+    out('/* This file is autogenerated by tracetool, do not edit. */',
+        '')
+
+    for event_id, e in enumerate(events):
+        if 'disable' in e.properties:
+            continue
+
+        out('probe %(probeprefix)s.log.%(name)s = %(probeprefix)s.%(name)s ?',
+            '{',
+            probeprefix=probeprefix(),
+            name=e.name)
+
+        # Get references to userspace strings
+        for type_, name in e.args:
+            name = stap_escape(name)
+            if is_string(type_):
+                out('    try {',
+                    '        arg%(name)s_str = %(name)s ? ' +
+                    'user_string_n(%(name)s, 512) : "<null>"',
+                    '    } catch {}',
+                    name=name)
+
+        # Determine systemtap's view of variable names
+        fields = ["pid()", "gettimeofday_ns()"]
+        for type_, name in e.args:
+            name = stap_escape(name)
+            if is_string(type_):
+                fields.append("arg" + name + "_str")
+            else:
+                fields.append(name)
+
+        # Emit the entire record in a single SystemTap printf()
+        arg_str = ', '.join(arg for arg in fields)
+        fmt_str = "%d@%d " + e.name + " " + c_fmt_to_stap(e.fmt) + "\\n"
+        out('    printf("%(fmt_str)s", %(arg_str)s)',
+            fmt_str=fmt_str, arg_str=arg_str)
+
+        out('}')
+
+    out()