xref: /OK3568_Linux_fs/external/xserver/doc/dtrace/Xserver-DTrace.xml (revision 4882a59341e53eb6f0b4789bf948001014eff981)

<?xml version="1.0" encoding="ISO-8859-1"?>
<!DOCTYPE article PUBLIC "-//OASIS//DTD DocBook XML V4.3//EN"
 "http://www.oasis-open.org/docbook/xml/4.3/docbookx.dtd" [
 <!ENTITY % defs SYSTEM "/xserver/doc/xml/xserver.ent"> %defs;
]>

<article id="Xserver-DTrace">
  <articleinfo>
    <title>Xserver Provider for DTrace</title>
    <author>
      <firstname>Alan</firstname><surname>Coopersmith</surname>
      <affiliation>
	<orgname>Oracle Corporation</orgname>
	<orgdiv>Solaris Engineering</orgdiv>
      </affiliation>
    </author>
    <releaseinfo>X Server Version &xserver.version;</releaseinfo>
    <copyright><year>2005</year><year>2006</year><year>2007</year><year>2010</year><year>2020</year>
      <holder>Oracle and/or its affiliates.</holder>
    </copyright>
    <legalnotice id="copyright">
      <para>
Permission is hereby granted, free of charge, to any person obtaining a
copy of this software and associated documentation files (the "Software"),
to deal in the Software without restriction, including without limitation
the rights to use, copy, modify, merge, publish, distribute, sublicense,
and/or sell copies of the Software, and to permit persons to whom the
Software is furnished to do so, subject to the following conditions:
      </para><para>
The above copyright notice and this permission notice (including the next
paragraph) shall be included in all copies or substantial portions of the
Software.
      </para><para>
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
      </para>
    </legalnotice>
  </articleinfo>

  <sect1 id="introduction">
    <title>Introduction</title>
    <para>
      This page provides details on a
      <ulink url="http://dtrace.org/guide/chp-usdt.html">statically defined user application tracing provider</ulink>
      for the
      <ulink url="http://dtrace.org/blogs/about/">DTrace</ulink>
      facility in <productname>Solaris</productname> 10,
      <productname>MacOS X</productname> 10.5, and later releases.  This
      provider instruments various points in the X server, to allow
      tracing what client applications are up to. DTrace probes may be used
      with <ulink url="http://sourceware.org/systemtap/">SystemTap</ulink>
      on GNU/Linux systems.
    </para>

    <para>
      The provider was integrated into the X.Org git master repository
      with Solaris 10 &amp; OpenSolaris support for the Xserver 1.4 release,
      released in 2007 with X11R7.3.   Support for DTrace on MacOS X
      was added in Xserver 1.7.
    </para>

    <para>
      These probes expose the request and reply structure of the X protocol
      between clients and the X server, so an understanding of that basic
      nature will aid in learning how to use these probes.
    </para>
  </sect1>

  <sect1 id="probes">
    <title>Available probes</title>

    <para>
      Due to the way User-Defined DTrace probes work, arguments to
      these probes all bear undistinguished names of
      <parameter>arg0</parameter>, <parameter>arg1</parameter>,
      <parameter>arg2</parameter>, etc.  These tables should help you
      determine what the real data is for each of the probe arguments.

    <table id="Probes_and_their_arguments">
      <title>Probes and their arguments</title>
      <tgroup cols='9'>
	<colspec colname="probe" colwidth="2*"/>
	<colspec colname="desc" colwidth="3*"/>
	<colspec colname="arg0" colwidth="1*"/>
	<colspec colname="arg1" colwidth="1*"/>
	<colspec colname="arg2" colwidth="1*"/>
	<colspec colname="arg3" colwidth="1*"/>
	<colspec colname="arg4" colwidth="1*"/>
	<colspec colname="arg5" colwidth="1*"/>
	<colspec colname="arg6" colwidth="1*"/>
	<spanspec spanname="all" namest="probe" nameend="arg4"/>
	<thead>
	  <row>
	    <entry>Probe name</entry>
	    <entry>Description</entry>
	    <entry>arg0</entry>
	    <entry>arg1</entry>
	    <entry>arg2</entry>
	    <entry>arg3</entry>
	    <entry>arg4</entry>
	    <entry>arg5</entry>
	    <entry>arg6</entry>
	  </row>
	</thead>
	<tbody>
	  <row>
	    <entry spanname="all" class="grouphead">Request Probes</entry>
	  </row>
	  <row>
	    <entry>request-start</entry>
	    <entry>Called just before processing each client request.</entry>
	    <entry><parameter>requestName</parameter></entry>
	    <entry><parameter>requestCode</parameter></entry>
	    <entry><parameter>requestLength</parameter></entry>
	    <entry><parameter>clientId</parameter></entry>
	    <entry><parameter>requestBuffer</parameter></entry>
	    <entry nameend="arg5" class="unused"/>
	    <entry nameend="arg6" class="unused"/>
	  </row>
	  <row>
	    <entry>request-done</entry>
	    <entry>Called just after processing each client request.</entry>
	    <entry><parameter>requestName</parameter></entry>
	    <entry><parameter>requestCode</parameter></entry>
	    <entry><parameter>sequenceNumber</parameter></entry>
	    <entry><parameter>clientId</parameter></entry>
	    <entry><parameter>resultCode</parameter></entry>
	    <entry nameend="arg5" class="unused"/>
	    <entry nameend="arg6" class="unused"/>
	  </row>
	  <row>
	    <entry spanname="all" class="grouphead">Event Probes</entry>
	  </row>
	  <row>
	    <entry>send-event</entry>
	    <entry>Called just before send each event to a client.</entry>
	    <entry><parameter>clientId</parameter></entry>
	    <entry><parameter>eventCode</parameter></entry>
	    <entry><parameter>eventBuffer</parameter></entry>
	    <entry nameend="arg3" class="unused"/>
	    <entry nameend="arg4" class="unused"/>
	    <entry nameend="arg5" class="unused"/>
	    <entry nameend="arg6" class="unused"/>
	  </row>
	  <row>
	    <entry spanname="all" class="grouphead">Client Connection Probes</entry>
	  </row>
	  <row>
	    <entry>client-connect</entry>
	    <entry>Called when a new connection is opened from a client</entry>
	    <entry><parameter>clientId</parameter></entry>
	    <entry><parameter>clientFD</parameter></entry>
	    <entry nameend="arg2" class="unused"/>
	    <entry nameend="arg3" class="unused"/>
	    <entry nameend="arg4" class="unused"/>
	    <entry nameend="arg5" class="unused"/>
	    <entry nameend="arg6" class="unused"/>
	  </row>
	  <row>
	    <entry>client-auth</entry>
	    <entry>Called when client authenticates (normally just after connection opened)</entry>
	    <entry><parameter>clientId</parameter></entry>
	    <entry><parameter>clientAddr</parameter></entry>
	    <entry><parameter>clientPid</parameter></entry>
	    <entry><parameter>clientZoneId</parameter></entry>
	    <entry nameend="arg4" class="unused"/>
	    <entry nameend="arg5" class="unused"/>
	    <entry nameend="arg6" class="unused"/>
	  </row>
	  <row>
	    <entry>client-disconnect</entry>
	    <entry>Called when a client connection is closed</entry>
	    <entry><parameter>clientId</parameter></entry>
	    <entry nameend="arg1" class="unused"/>
	    <entry nameend="arg2" class="unused"/>
	    <entry nameend="arg3" class="unused"/>
	    <entry nameend="arg4" class="unused"/>
	    <entry nameend="arg5" class="unused"/>
	    <entry nameend="arg6" class="unused"/>
	  </row>
	  <row>
	    <entry spanname="all" class="grouphead">Resource Allocation Probes</entry>
	  </row>
	  <row>
	    <entry>resource-alloc</entry>
	    <entry>Called when a new resource (pixmap, gc, colormap, etc.) is allocated</entry>
	    <entry><parameter>resourceId</parameter></entry>
	    <entry><parameter>resourceTypeId</parameter></entry>
	    <entry><parameter>resourceValue</parameter></entry>
	    <entry><parameter>resourceTypeName</parameter></entry>
	    <entry nameend="arg4" class="unused"/>
	    <entry nameend="arg5" class="unused"/>
	    <entry nameend="arg6" class="unused"/>
	  </row>
	  <row>
	    <entry>resource-free</entry>
	    <entry>Called when a resource is freed</entry>
	    <entry><parameter>resourceId</parameter></entry>
	    <entry><parameter>resourceTypeId</parameter></entry>
	    <entry><parameter>resourceValue</parameter></entry>
	    <entry><parameter>resourceTypeName</parameter></entry>
	    <entry nameend="arg4" class="unused"/>
	    <entry nameend="arg5" class="unused"/>
	    <entry nameend="arg6" class="unused"/>
	  </row>
	  <row>
	    <entry spanname="all" class="grouphead">Input API probes</entry>
	  </row>
	  <row>
	    <entry>input-event</entry>
	    <entry>Called when an input event was submitted for processing</entry>
	    <entry><parameter>deviceid</parameter></entry>
	    <entry><parameter>eventtype</parameter></entry>
	    <entry><parameter>button</parameter> or
		   <parameter>keycode</parameter> or
		   <parameter>touchid</parameter></entry>
	    <entry><parameter>flags</parameter></entry>
	    <entry><parameter>nvalues</parameter></entry>
	    <entry><parameter>mask</parameter></entry>
	    <entry><parameter>values</parameter></entry>
	  </row>
	</tbody>
      </tgroup>
    </table>
    </para>
  </sect1>

  <sect1 id="arguments">
    <title>Data Available in Probe Arguments</title>

    <para>
      To access data in arguments of type <type>string</type>, you will need
      to use <ulink url="http://dtrace.org/guide/chp-actsub.html#chp-actsub-copyinstr"><function>copyinstr()</function></ulink>.
      To access data buffers referenced via <type>uintptr_t</type>'s, you will
      need to use <ulink url="http://dtrace.org/guide/chp-actsub.html#chp-actsub-copyin"><function>copyin()</function></ulink>.

    <table id="Probe_Arguments">
      <title>Probe Arguments</title>
      <tgroup cols='3'>
	<colspec colname="arg" colwidth="2*"/>
	<colspec colname="type" colwidth="1*"/>
	<colspec colname="desc" colwidth="7*"/>
	<thead>
	  <row>
	    <entry>Argument name</entry>
	    <entry>Type</entry>
	    <entry>Description</entry>
	  </row>
	</thead>
	<tbody>
	  <row>
	    <entry><parameter>clientAddr</parameter></entry>
	    <entry><type>string</type></entry>
	    <entry>String representing address client connected from</entry>
	  </row>
	  <row>
	    <entry><parameter>clientFD</parameter></entry>
	    <entry><type>int</type></entry>
	    <entry>X server's file descriptor for server side of each connection</entry>
	  </row>
	  <row>
	    <entry><parameter>clientId</parameter></entry>
	    <entry><type>int</type></entry>
	    <entry>Unique integer identifier for each connection to the
	      X server</entry>
	  </row>
	  <row>
	    <entry><parameter>clientPid</parameter></entry>
	    <entry><type>pid_t</type></entry>
	    <entry>Process id of client, if connection is local
	      (from <function>getpeerucred()</function>)</entry>
	  </row>
	  <row>
	    <entry><parameter>clientZoneId</parameter></entry>
	    <entry><type>zoneid_t</type></entry>
	    <entry>Solaris: Zone id of client, if connection is local
	      (from <function>getpeerucred()</function>)</entry>
	  </row>
	  <row>
	    <entry><parameter>eventBuffer</parameter></entry>
	    <entry><type>uintptr_t</type></entry>
	    <entry>Pointer to buffer containing X event - decode using
	      structures in
	      &lt;<ulink url="https://gitlab.freedesktop.org/xorg/proto/xorgproto/-/blob/master/include/X11/Xproto.h"><filename class="headerfile">X11/Xproto.h</filename></ulink>&gt;
	      and similar headers for each extension</entry>
	  </row>
	  <row>
	    <entry><parameter>eventCode</parameter></entry>
	    <entry><type>uint8_t</type></entry>
	    <entry>Event number of X event</entry>
	  </row>
	  <row>
	    <entry><parameter>resourceId</parameter></entry>
	    <entry><type>uint32_t</type></entry>
	    <entry>X resource id (XID)</entry>
	  </row>
	  <row>
	    <entry><parameter>resourceTypeId</parameter></entry>
	    <entry><type>uint32_t</type></entry>
	    <entry>Resource type id</entry>
	  </row>
	  <row>
	    <entry><parameter>resourceTypeName</parameter></entry>
	    <entry><type>string</type></entry>
	    <entry>String representing X resource type
		(<literal>"PIXMAP"</literal>, etc.)</entry>
	  </row>
	  <row>
	    <entry><parameter>resourceValue</parameter></entry>
	    <entry><type>uintptr_t</type></entry>
	    <entry>Pointer to data for X resource</entry>
	  </row>
	  <row>
	    <entry><parameter>resultCode</parameter></entry>
	    <entry><type>int</type></entry>
	    <entry>Integer code representing result status of request</entry>
	  </row>
	  <row>
	    <entry><parameter>requestBuffer</parameter></entry>
	    <entry><type>uintptr_t</type></entry>
	    <entry>Pointer to buffer containing X request - decode using
	      structures in
	      &lt;<ulink url="https://gitlab.freedesktop.org/xorg/proto/xorgproto/-/blob/master/include/X11/Xproto.h"><filename class="headerfile">X11/Xproto.h</filename></ulink>&gt;
	      and similar headers for each extension</entry>
	  </row>
	  <row>
	    <entry><parameter>requestCode</parameter></entry>
	    <entry><type>uint8_t</type></entry>
	    <entry>Request number of X request or Extension</entry>
	  </row>
	  <row>
	    <entry><parameter>requestName</parameter></entry>
	    <entry><type>string</type></entry>
	    <entry>Name of X request or Extension</entry>
	  </row>
	  <row>
	    <entry><parameter>requestLength</parameter></entry>
	    <entry><type>uint16_t</type></entry>
	    <entry>Length of X request</entry>
	  </row>
	  <row>
	    <entry><parameter>sequenceNumber</parameter></entry>
	    <entry><type>uint32_t</type></entry>
	    <entry>Number of X request in in this connection</entry>
	  </row>
	  <row>
	    <entry><parameter>deviceid</parameter></entry>
	    <entry><type>int</type></entry>
	    <entry>The device's numerical ID</entry>
	  </row>
	  <row>
	    <entry><parameter>eventtype</parameter></entry>
	    <entry><type>int</type></entry>
	    <entry>Protocol event type</entry>
	  </row>
	  <row>
	    <entry><parameter>button, keycode, touchid</parameter></entry>
	    <entry><type>uint32_t</type></entry>
	    <entry>The button number, keycode or touch ID</entry>
	  </row>
	  <row>
	    <entry><parameter>flags</parameter></entry>
	    <entry><type>uint32_t</type></entry>
	    <entry>Miscellaneous event-specific server flags</entry>
	  </row>
	  <row>
	    <entry><parameter>nvalues</parameter></entry>
	    <entry><type>int8_t</type></entry>
	    <entry>Number of bits in <parameter>mask</parameter> and number of elements
		    in <parameter>values</parameter></entry>
	  </row>
	  <row>
	    <entry><parameter>mask</parameter></entry>
	    <entry><type>uint8_t*</type></entry>
	    <entry>Binary mask indicating which indices in <parameter>values</parameter> contain
		  valid data</entry>
	  </row>
	  <row>
	    <entry><parameter>values</parameter></entry>
	    <entry><type>double*</type></entry>
	    <entry>Valuator values. Values for indices for which the
		  <parameter>mask</parameter> is not set are undefined</entry>
	  </row>
	</tbody>
      </tgroup>
    </table>
    </para>
  </sect1>

  <sect1 id="examples">
    <title>Examples</title>

    <example id="Counting_requests_by_request_name">
      <title>Counting requests by request name</title>

      <para>
	This script simply increments a counter for each different request
	made, and when you exit the script (such as by hitting
	<keycombo action='simul'><keycap>Control</keycap><keycap>C</keycap>
	</keycombo>) prints the counts.

	<programlisting>
#!/usr/sbin/dtrace -s

Xserver*:::request-start
{
    @counts[copyinstr(arg0)] = count();
}
	</programlisting>

	The output from a short run may appear as:
	<screen>
  QueryPointer                                                      1
  CreatePixmap                                                      2
  FreePixmap                                                        2
  PutImage                                                          2
  ChangeGC                                                         10
  CopyArea                                                         10
  CreateGC                                                         14
  FreeGC                                                           14
  RENDER                                                           28
  SetClipRectangles                                                40
	</screen>
      </para>

      <para>
	This can be rewritten slightly to cache the string containing the name
	of the request since it will be reused many times, instead of copying
	it over and over from the kernel:

	<programlisting>
#!/usr/sbin/dtrace -s

string Xrequest[uintptr_t];

Xserver*:::request-start
/Xrequest[arg0] == ""/
{
    Xrequest[arg0] = copyinstr(arg0);
}

Xserver*:::request-start
{
    @counts[Xrequest[arg0]] = count();
}
	</programlisting>
      </para>
    </example>

    <example id="Get_average_CPU_time_per_request">
      <title>Get average CPU time per request</title>

      <para>This script records the CPU time used between the probes at
	the start and end of each request and aggregates it per request type.

	<programlisting>
#!/usr/sbin/dtrace -s

Xserver*:::request-start
{
    reqstart = vtimestamp;
}

Xserver*:::request-done
{
    @times[copyinstr(arg0)] = avg(vtimestamp - reqstart);
}
	</programlisting>

	The output from a sample run might look like:

	<screen>
  ChangeGC                                                        889
  MapWindow                                                       907
  SetClipRectangles                                              1319
  PolyPoint                                                      1413
  PolySegment                                                    1434
  PolyRectangle                                                  1828
  FreeCursor                                                     1895
  FreeGC                                                         1950
  CreateGC                                                       2244
  FreePixmap                                                     2246
  GetInputFocus                                                  2249
  TranslateCoords                                                8508
  QueryTree                                                      8846
  GetGeometry                                                    9948
  CreatePixmap                                                  12111
  AllowEvents                                                   14090
  GrabServer                                                    14791
  MIT-SCREEN-SAVER                                              16747
  ConfigureWindow                                               22917
  SetInputFocus                                                 28521
  PutImage                                                     240841

	</screen>
      </para>
    </example>

    <example id="Monitoring_clients_that_connect_and_disconnect">
      <title>Monitoring clients that connect and disconnect</title>

      <para>
	This script simply prints information about each client that
	connects or disconnects from the server while it is running.
	Since the provider is specified as <code>Xserver$1</code> instead
	of <code>Xserver*</code> like previous examples, it won't monitor
	all Xserver processes running on the machine, but instead expects
	the process id of the X server to monitor to be specified as the
	argument to the script.

	<programlisting>
#!/usr/sbin/dtrace -s

Xserver$1:::client-connect
{
	printf("** Client Connect: id %d\n", arg0);
}

Xserver$1:::client-auth
{
	printf("** Client auth'ed: id %d =&gt; %s pid %d\n",
		arg0, copyinstr(arg1), arg2);
}

Xserver$1:::client-disconnect
{
	printf("** Client Disconnect: id %d\n", arg0);
}
	</programlisting>

	A sample run:

	<screen>
<prompt>#</prompt> <userinput>./foo.d 5790</userinput>
<computeroutput>dtrace: script './foo.d' matched 4 probes
CPU     ID                    FUNCTION:NAME
  0  15774 CloseDownClient:client-disconnect ** Client Disconnect: id 65

  2  15774 CloseDownClient:client-disconnect ** Client Disconnect: id 64

  0  15773 EstablishNewConnections:client-connect ** Client Connect: id 64

  0  15772            AuthAudit:client-auth ** Client auth'ed: id 64 =&gt; local host pid 2034

  0  15773 EstablishNewConnections:client-connect ** Client Connect: id 65

  0  15772            AuthAudit:client-auth ** Client auth'ed: id 65 =&gt; local host pid 2034

  0  15774 CloseDownClient:client-disconnect ** Client Disconnect: id 64
	  </computeroutput>
	</screen>

      </para>
    </example>

    <example id="Monitoring_clients_creating_Pixmaps">
      <title>Monitoring clients creating Pixmaps</title>

      <para>
	This script can be used to determine which clients are creating
	pixmaps in the X server, printing information about each client
	as it connects to help trace it back to the program on the other
	end of the X connection.

	<programlisting>
#!/usr/sbin/dtrace -qs

string Xrequest[uintptr_t];
string Xrestype[uintptr_t];

Xserver$1:::request-start
/Xrequest[arg0] == ""/
{
	Xrequest[arg0] = copyinstr(arg0);
}

Xserver$1:::resource-alloc
/arg3 != 0 &amp;&amp; Xrestype[arg3] == ""/
{
	Xrestype[arg3] = copyinstr(arg3);
}


Xserver$1:::request-start
/Xrequest[arg0] == "X_CreatePixmap"/
{
	printf("-&gt; %s: client %d\n", Xrequest[arg0], arg3);
}

Xserver$1:::request-done
/Xrequest[arg0] == "X_CreatePixmap"/
{
	printf("&lt;- %s: client %d\n", Xrequest[arg0], arg3);
}

Xserver$1:::resource-alloc
/Xrestype[arg3] == "PIXMAP"/
{
	printf("** Pixmap alloc: %08x\n", arg0);
}


Xserver$1:::resource-free
/Xrestype[arg3] == "PIXMAP"/
{
	printf("** Pixmap free:  %08x\n", arg0);
}

Xserver$1:::client-connect
{
	printf("** Client Connect: id %d\n", arg0);
}

Xserver$1:::client-auth
{
	printf("** Client auth'ed: id %d =&gt; %s pid %d\n",
		arg0, copyinstr(arg1), arg2);
}

Xserver$1:::client-disconnect
{
	printf("** Client Disconnect: id %d\n", arg0);
}
	</programlisting>

	Sample output from a run of this script:
	<screen><computeroutput>
** Client Connect: id 17
** Client auth'ed: id 17 =&gt; local host pid 20273
-&gt; X_CreatePixmap: client 17
** Pixmap alloc: 02200009
&lt;- X_CreatePixmap: client 17
-&gt; X_CreatePixmap: client 15
** Pixmap alloc: 01e00180
&lt;- X_CreatePixmap: client 15
-&gt; X_CreatePixmap: client 15
** Pixmap alloc: 01e00181
&lt;- X_CreatePixmap: client 15
-&gt; X_CreatePixmap: client 14
** Pixmap alloc: 01c004c8
&lt;- X_CreatePixmap: client 14
** Pixmap free:  02200009
** Client Disconnect: id 17
** Pixmap free:  01e00180
** Pixmap free:  01e00181
	  </computeroutput></screen>

      </para>

    </example>

    <example id="Input_API_monitoring_with_systemtap">
      <title>Input API monitoring with SystemTap</title>

      <para>
	This script can be used to monitor events submitted by drivers to
	the server for enqueuing. Due to the integration of the input API
	probes, some server-enqueued events will show up too.
	<programlisting>
  # Compile+run with
  #       stap -g xorg.stp /usr/bin/Xorg
  #


  function print_valuators:string(nvaluators:long, mask_in:long, valuators_in:long) %{
	  int i;
	  unsigned char *mask = (unsigned char*)THIS->mask_in;
	  double *valuators = (double*)THIS->valuators_in;
	  char str[128] = {0};
	  char *s = str;

  #define BitIsSet(ptr, bit) (((unsigned char*)(ptr))[(bit)>>3] &amp; (1 &lt;&lt; ((bit) &amp; 7)))

	  s += sprintf(s, "nval: %d ::", (int)THIS->nvaluators);
	  for (i = 0; i &lt; THIS->nvaluators; i++)
	  {
		  s += sprintf(s, "	%d: ", i);
		  if (BitIsSet(mask, i))
		      s += sprintf(s, "%d", (int)valuators[i]);
	  }

	  sprintf(THIS->__retvalue, "%s", str);
  %}

  probe process(@1).mark("input__event")
  {
      deviceid = $arg1
      type = $arg2
      detail = $arg3
      flags = $arg4
      nvaluators = $arg5

      str = print_valuators(nvaluators, $arg6, $arg7)
      printf("Event: device %d type %d detail %d flags %#x %s\n",
	      deviceid, type, detail, flags, str);
  }
	</programlisting>

	Sample output from a run of this script:
	<screen><computeroutput>
Event: device 13 type 4 detail 1 flags 0x0 nval: 0 ::
Event: device 13 type 6 detail 0 flags 0xa nval: 1 ::	0: 1
Event: device 13 type 6 detail 0 flags 0xa nval: 2 ::	0: 2	1: -1
Event: device 13 type 6 detail 0 flags 0xa nval: 2 ::	0: 2	1: -1
Event: device 13 type 6 detail 0 flags 0xa nval: 2 ::	0: 4	1: -3
Event: device 13 type 6 detail 0 flags 0xa nval: 2 ::	0: 3	1: -3
Event: device 13 type 6 detail 0 flags 0xa nval: 2 ::	0: 3	1: -2
Event: device 13 type 6 detail 0 flags 0xa nval: 2 ::	0: 2	1: -2
Event: device 13 type 6 detail 0 flags 0xa nval: 2 ::	0: 2	1: -2
Event: device 13 type 6 detail 0 flags 0xa nval: 2 ::	0: 2	1: -2
Event: device 13 type 6 detail 0 flags 0xa nval: 2 ::	0: 	1: -1
Event: device 13 type 6 detail 0 flags 0xa nval: 2 ::	0: 	1: -1
Event: device 13 type 5 detail 1 flags 0x0 nval: 0 ::
	</computeroutput></screen>

      </para>

    </example>

  </sect1>

</article>