1
0
Fork 0

tools/bpf: bpftool: add net support

Add "bpftool net" support. Networking devices are enumerated
to dump device index/name associated with xdp progs.

For each networking device, tc classes and qdiscs are enumerated
in order to check their bpf filters.
In addition, root handle and clsact ingress/egress are also checked for
bpf filters.  Not all filter information is printed out. Only ifindex,
kind, filter name, prog_id and tag are printed out, which are good
enough to show attachment information. If the filter action
is a bpf action, its bpf program id, bpf name and tag will be
printed out as well.

For example,
  $ ./bpftool net
  xdp [
  ifindex 2 devname eth0 prog_id 198
  ]
  tc_filters [
  ifindex 2 kind qdisc_htb name prefix_matcher.o:[cls_prefix_matcher_htb]
            prog_id 111727 tag d08fe3b4319bc2fd act []
  ifindex 2 kind qdisc_clsact_ingress name fbflow_icmp
            prog_id 130246 tag 3f265c7f26db62c9 act []
  ifindex 2 kind qdisc_clsact_egress name prefix_matcher.o:[cls_prefix_matcher_clsact]
            prog_id 111726 tag 99a197826974c876
  ifindex 2 kind qdisc_clsact_egress name cls_fg_dscp
            prog_id 108619 tag dc4630674fd72dcc act []
  ifindex 2 kind qdisc_clsact_egress name fbflow_egress
            prog_id 130245 tag 72d2d830d6888d2c
  ]
  $ ./bpftool -jp net
  [{
        "xdp": [{
                "ifindex": 2,
                "devname": "eth0",
                "prog_id": 198
            }
        ],
        "tc_filters": [{
                "ifindex": 2,
                "kind": "qdisc_htb",
                "name": "prefix_matcher.o:[cls_prefix_matcher_htb]",
                "prog_id": 111727,
                "tag": "d08fe3b4319bc2fd",
                "act": []
            },{
                "ifindex": 2,
                "kind": "qdisc_clsact_ingress",
                "name": "fbflow_icmp",
                "prog_id": 130246,
                "tag": "3f265c7f26db62c9",
                "act": []
            },{
                "ifindex": 2,
                "kind": "qdisc_clsact_egress",
                "name": "prefix_matcher.o:[cls_prefix_matcher_clsact]",
                "prog_id": 111726,
                "tag": "99a197826974c876"
            },{
                "ifindex": 2,
                "kind": "qdisc_clsact_egress",
                "name": "cls_fg_dscp",
                "prog_id": 108619,
                "tag": "dc4630674fd72dcc",
                "act": []
            },{
                "ifindex": 2,
                "kind": "qdisc_clsact_egress",
                "name": "fbflow_egress",
                "prog_id": 130245,
                "tag": "72d2d830d6888d2c"
            }
        ]
    }
  ]

Signed-off-by: Yonghong Song <yhs@fb.com>
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
hifive-unleashed-5.1
Yonghong Song 2018-09-05 16:58:06 -07:00 committed by Alexei Starovoitov
parent 36f1678d9e
commit f6f3bac08f
8 changed files with 676 additions and 7 deletions

View File

@ -0,0 +1,133 @@
================
bpftool-net
================
-------------------------------------------------------------------------------
tool for inspection of netdev/tc related bpf prog attachments
-------------------------------------------------------------------------------
:Manual section: 8
SYNOPSIS
========
**bpftool** [*OPTIONS*] **net** *COMMAND*
*OPTIONS* := { [{ **-j** | **--json** }] [{ **-p** | **--pretty** }] }
*COMMANDS* :=
{ **show** | **list** } [ **dev** name ] | **help**
NET COMMANDS
============
| **bpftool** **net { show | list } [ dev name ]**
| **bpftool** **net help**
DESCRIPTION
===========
**bpftool net { show | list } [ dev name ]**
List all networking device driver and tc attachment in the system.
Output will start with all xdp program attachment, followed by
all tc class/qdisc bpf program attachments. Both xdp programs and
tc programs are ordered based on ifindex number. If multiple bpf
programs attached to the same networking device through **tc filter**,
the order will be first all bpf programs attached to tc classes, then
all bpf programs attached to non clsact qdiscs, and finally all
bpf programs attached to root and clsact qdisc.
**bpftool net help**
Print short help message.
OPTIONS
=======
-h, --help
Print short generic help message (similar to **bpftool help**).
-v, --version
Print version number (similar to **bpftool version**).
-j, --json
Generate JSON output. For commands that cannot produce JSON, this
option has no effect.
-p, --pretty
Generate human-readable JSON output. Implies **-j**.
EXAMPLES
========
| **# bpftool net**
::
xdp [
ifindex 2 devname eth0 prog_id 198
]
tc_filters [
ifindex 2 kind qdisc_htb name prefix_matcher.o:[cls_prefix_matcher_htb]
prog_id 111727 tag d08fe3b4319bc2fd act []
ifindex 2 kind qdisc_clsact_ingress name fbflow_icmp
prog_id 130246 tag 3f265c7f26db62c9 act []
ifindex 2 kind qdisc_clsact_egress name prefix_matcher.o:[cls_prefix_matcher_clsact]
prog_id 111726 tag 99a197826974c876
ifindex 2 kind qdisc_clsact_egress name cls_fg_dscp
prog_id 108619 tag dc4630674fd72dcc act []
ifindex 2 kind qdisc_clsact_egress name fbflow_egress
prog_id 130245 tag 72d2d830d6888d2c
]
|
| **# bpftool -jp net**
::
[{
"xdp": [{
"ifindex": 2,
"devname": "eth0",
"prog_id": 198
}
],
"tc_filters": [{
"ifindex": 2,
"kind": "qdisc_htb",
"name": "prefix_matcher.o:[cls_prefix_matcher_htb]",
"prog_id": 111727,
"tag": "d08fe3b4319bc2fd",
"act": []
},{
"ifindex": 2,
"kind": "qdisc_clsact_ingress",
"name": "fbflow_icmp",
"prog_id": 130246,
"tag": "3f265c7f26db62c9",
"act": []
},{
"ifindex": 2,
"kind": "qdisc_clsact_egress",
"name": "prefix_matcher.o:[cls_prefix_matcher_clsact]",
"prog_id": 111726,
"tag": "99a197826974c876"
},{
"ifindex": 2,
"kind": "qdisc_clsact_egress",
"name": "cls_fg_dscp",
"prog_id": 108619,
"tag": "dc4630674fd72dcc",
"act": []
},{
"ifindex": 2,
"kind": "qdisc_clsact_egress",
"name": "fbflow_egress",
"prog_id": 130245,
"tag": "72d2d830d6888d2c"
}
]
}
]
SEE ALSO
========
**bpftool**\ (8), **bpftool-prog**\ (8), **bpftool-map**\ (8)

View File

@ -16,7 +16,7 @@ SYNOPSIS
**bpftool** **version**
*OBJECT* := { **map** | **program** | **cgroup** | **perf** }
*OBJECT* := { **map** | **program** | **cgroup** | **perf** | **net** }
*OPTIONS* := { { **-V** | **--version** } | { **-h** | **--help** }
| { **-j** | **--json** } [{ **-p** | **--pretty** }] }
@ -32,6 +32,8 @@ SYNOPSIS
*PERF-COMMANDS* := { **show** | **list** | **help** }
*NET-COMMANDS* := { **show** | **list** | **help** }
DESCRIPTION
===========
*bpftool* allows for inspection and simple modification of BPF objects
@ -58,4 +60,4 @@ OPTIONS
SEE ALSO
========
**bpftool-map**\ (8), **bpftool-prog**\ (8), **bpftool-cgroup**\ (8)
**bpftool-perf**\ (8)
**bpftool-perf**\ (8), **bpftool-net**\ (8)

View File

@ -494,10 +494,10 @@ _bpftool()
_filedir
return 0
;;
tree)
_filedir
return 0
;;
tree)
_filedir
return 0
;;
attach|detach)
local ATTACH_TYPES='ingress egress sock_create sock_ops \
device bind4 bind6 post_bind4 post_bind6 connect4 \
@ -552,6 +552,15 @@ _bpftool()
;;
esac
;;
net)
case $command in
*)
[[ $prev == $object ]] && \
COMPREPLY=( $( compgen -W 'help \
show list' -- "$cur" ) )
;;
esac
;;
esac
} &&
complete -F _bpftool bpftool

View File

@ -85,7 +85,7 @@ static int do_help(int argc, char **argv)
" %s batch file FILE\n"
" %s version\n"
"\n"
" OBJECT := { prog | map | cgroup | perf }\n"
" OBJECT := { prog | map | cgroup | perf | net }\n"
" " HELP_SPEC_OPTIONS "\n"
"",
bin_name, bin_name, bin_name);
@ -215,6 +215,7 @@ static const struct cmd cmds[] = {
{ "map", do_map },
{ "cgroup", do_cgroup },
{ "perf", do_perf },
{ "net", do_net },
{ "version", do_version },
{ 0 }
};

View File

@ -136,6 +136,7 @@ int do_map(int argc, char **arg);
int do_event_pipe(int argc, char **argv);
int do_cgroup(int argc, char **arg);
int do_perf(int argc, char **arg);
int do_net(int argc, char **arg);
int prog_parse_fd(int *argc, char ***argv);
int map_parse_fd(int *argc, char ***argv);
@ -165,4 +166,10 @@ struct btf_dumper {
*/
int btf_dumper_type(const struct btf_dumper *d, __u32 type_id,
const void *data);
struct nlattr;
struct ifinfomsg;
struct tcmsg;
int do_xdp_dump(struct ifinfomsg *ifinfo, struct nlattr **tb);
int do_filter_dump(struct tcmsg *ifinfo, struct nlattr **tb, const char *kind);
#endif

View File

@ -0,0 +1,233 @@
// SPDX-License-Identifier: GPL-2.0+
// Copyright (C) 2018 Facebook
#define _GNU_SOURCE
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <libbpf.h>
#include <net/if.h>
#include <linux/if.h>
#include <linux/rtnetlink.h>
#include <linux/tc_act/tc_bpf.h>
#include <sys/socket.h>
#include <bpf.h>
#include <nlattr.h>
#include "main.h"
#include "netlink_dumper.h"
struct bpf_netdev_t {
int *ifindex_array;
int used_len;
int array_len;
int filter_idx;
};
struct tc_kind_handle {
char kind[64];
int handle;
};
struct bpf_tcinfo_t {
struct tc_kind_handle *handle_array;
int used_len;
int array_len;
bool is_qdisc;
};
static int dump_link_nlmsg(void *cookie, void *msg, struct nlattr **tb)
{
struct bpf_netdev_t *netinfo = cookie;
struct ifinfomsg *ifinfo = msg;
if (netinfo->filter_idx > 0 && netinfo->filter_idx != ifinfo->ifi_index)
return 0;
if (netinfo->used_len == netinfo->array_len) {
netinfo->ifindex_array = realloc(netinfo->ifindex_array,
(netinfo->array_len + 16) * sizeof(int));
netinfo->array_len += 16;
}
netinfo->ifindex_array[netinfo->used_len++] = ifinfo->ifi_index;
return do_xdp_dump(ifinfo, tb);
}
static int dump_class_qdisc_nlmsg(void *cookie, void *msg, struct nlattr **tb)
{
struct bpf_tcinfo_t *tcinfo = cookie;
struct tcmsg *info = msg;
if (tcinfo->is_qdisc) {
/* skip clsact qdisc */
if (tb[TCA_KIND] &&
strcmp(nla_data(tb[TCA_KIND]), "clsact") == 0)
return 0;
if (info->tcm_handle == 0)
return 0;
}
if (tcinfo->used_len == tcinfo->array_len) {
tcinfo->handle_array = realloc(tcinfo->handle_array,
(tcinfo->array_len + 16) * sizeof(struct tc_kind_handle));
tcinfo->array_len += 16;
}
tcinfo->handle_array[tcinfo->used_len].handle = info->tcm_handle;
snprintf(tcinfo->handle_array[tcinfo->used_len].kind,
sizeof(tcinfo->handle_array[tcinfo->used_len].kind),
"%s_%s",
tcinfo->is_qdisc ? "qdisc" : "class",
tb[TCA_KIND] ? nla_getattr_str(tb[TCA_KIND]) : "unknown");
tcinfo->used_len++;
return 0;
}
static int dump_filter_nlmsg(void *cookie, void *msg, struct nlattr **tb)
{
const char *kind = cookie;
return do_filter_dump((struct tcmsg *)msg, tb, kind);
}
static int show_dev_tc_bpf(int sock, unsigned int nl_pid, int ifindex)
{
struct bpf_tcinfo_t tcinfo;
int i, handle, ret;
tcinfo.handle_array = NULL;
tcinfo.used_len = 0;
tcinfo.array_len = 0;
tcinfo.is_qdisc = false;
ret = nl_get_class(sock, nl_pid, ifindex, dump_class_qdisc_nlmsg,
&tcinfo);
if (ret)
return ret;
tcinfo.is_qdisc = true;
ret = nl_get_qdisc(sock, nl_pid, ifindex, dump_class_qdisc_nlmsg,
&tcinfo);
if (ret)
return ret;
for (i = 0; i < tcinfo.used_len; i++) {
ret = nl_get_filter(sock, nl_pid, ifindex,
tcinfo.handle_array[i].handle,
dump_filter_nlmsg,
tcinfo.handle_array[i].kind);
if (ret)
return ret;
}
/* root, ingress and egress handle */
handle = TC_H_ROOT;
ret = nl_get_filter(sock, nl_pid, ifindex, handle, dump_filter_nlmsg,
"root");
if (ret)
return ret;
handle = TC_H_MAKE(TC_H_CLSACT, TC_H_MIN_INGRESS);
ret = nl_get_filter(sock, nl_pid, ifindex, handle, dump_filter_nlmsg,
"qdisc_clsact_ingress");
if (ret)
return ret;
handle = TC_H_MAKE(TC_H_CLSACT, TC_H_MIN_EGRESS);
ret = nl_get_filter(sock, nl_pid, ifindex, handle, dump_filter_nlmsg,
"qdisc_clsact_egress");
if (ret)
return ret;
return 0;
}
static int do_show(int argc, char **argv)
{
int i, sock, ret, filter_idx = -1;
struct bpf_netdev_t dev_array;
unsigned int nl_pid;
char err_buf[256];
if (argc == 2) {
if (strcmp(argv[0], "dev") != 0)
usage();
filter_idx = if_nametoindex(argv[1]);
if (filter_idx == 0) {
fprintf(stderr, "invalid dev name %s\n", argv[1]);
return -1;
}
} else if (argc != 0) {
usage();
}
sock = bpf_netlink_open(&nl_pid);
if (sock < 0) {
fprintf(stderr, "failed to open netlink sock\n");
return -1;
}
dev_array.ifindex_array = NULL;
dev_array.used_len = 0;
dev_array.array_len = 0;
dev_array.filter_idx = filter_idx;
if (json_output)
jsonw_start_array(json_wtr);
NET_START_OBJECT;
NET_START_ARRAY("xdp", "\n");
ret = nl_get_link(sock, nl_pid, dump_link_nlmsg, &dev_array);
NET_END_ARRAY("\n");
if (!ret) {
NET_START_ARRAY("tc_filters", "\n");
for (i = 0; i < dev_array.used_len; i++) {
ret = show_dev_tc_bpf(sock, nl_pid,
dev_array.ifindex_array[i]);
if (ret)
break;
}
NET_END_ARRAY("\n");
}
NET_END_OBJECT;
if (json_output)
jsonw_end_array(json_wtr);
if (ret) {
if (json_output)
jsonw_null(json_wtr);
libbpf_strerror(ret, err_buf, sizeof(err_buf));
fprintf(stderr, "Error: %s\n", err_buf);
}
free(dev_array.ifindex_array);
close(sock);
return ret;
}
static int do_help(int argc, char **argv)
{
if (json_output) {
jsonw_null(json_wtr);
return 0;
}
fprintf(stderr,
"Usage: %s %s { show | list } [dev <devname>]\n"
" %s %s help\n",
bin_name, argv[-2], bin_name, argv[-2]);
return 0;
}
static const struct cmd cmds[] = {
{ "show", do_show },
{ "list", do_show },
{ "help", do_help },
{ 0 }
};
int do_net(int argc, char **argv)
{
return cmd_select(cmds, argc, argv, do_help);
}

View File

@ -0,0 +1,181 @@
// SPDX-License-Identifier: GPL-2.0+
// Copyright (C) 2018 Facebook
#include <stdlib.h>
#include <string.h>
#include <libbpf.h>
#include <linux/rtnetlink.h>
#include <linux/tc_act/tc_bpf.h>
#include <nlattr.h>
#include "main.h"
#include "netlink_dumper.h"
static void xdp_dump_prog_id(struct nlattr **tb, int attr,
const char *type)
{
if (!tb[attr])
return;
NET_DUMP_UINT(type, nla_getattr_u32(tb[attr]))
}
static int do_xdp_dump_one(struct nlattr *attr, unsigned int ifindex,
const char *name)
{
struct nlattr *tb[IFLA_XDP_MAX + 1];
unsigned char mode;
if (nla_parse_nested(tb, IFLA_XDP_MAX, attr, NULL) < 0)
return -1;
if (!tb[IFLA_XDP_ATTACHED])
return 0;
mode = nla_getattr_u8(tb[IFLA_XDP_ATTACHED]);
if (mode == XDP_ATTACHED_NONE)
return 0;
NET_START_OBJECT;
NET_DUMP_UINT("ifindex", ifindex);
if (name)
NET_DUMP_STR("devname", name);
if (tb[IFLA_XDP_PROG_ID])
NET_DUMP_UINT("prog_id", nla_getattr_u32(tb[IFLA_XDP_PROG_ID]));
if (mode == XDP_ATTACHED_MULTI) {
xdp_dump_prog_id(tb, IFLA_XDP_SKB_PROG_ID, "generic_prog_id");
xdp_dump_prog_id(tb, IFLA_XDP_DRV_PROG_ID, "drv_prog_id");
xdp_dump_prog_id(tb, IFLA_XDP_HW_PROG_ID, "offload_prog_id");
}
NET_END_OBJECT_FINAL;
return 0;
}
int do_xdp_dump(struct ifinfomsg *ifinfo, struct nlattr **tb)
{
if (!tb[IFLA_XDP])
return 0;
return do_xdp_dump_one(tb[IFLA_XDP], ifinfo->ifi_index,
nla_getattr_str(tb[IFLA_IFNAME]));
}
static char *hexstring_n2a(const unsigned char *str, int len,
char *buf, int blen)
{
char *ptr = buf;
int i;
for (i = 0; i < len; i++) {
if (blen < 3)
break;
sprintf(ptr, "%02x", str[i]);
ptr += 2;
blen -= 2;
}
return buf;
}
static int do_bpf_dump_one_act(struct nlattr *attr)
{
struct nlattr *tb[TCA_ACT_BPF_MAX + 1];
char buf[256];
if (nla_parse_nested(tb, TCA_ACT_BPF_MAX, attr, NULL) < 0)
return -LIBBPF_ERRNO__NLPARSE;
if (!tb[TCA_ACT_BPF_PARMS])
return -LIBBPF_ERRNO__NLPARSE;
NET_START_OBJECT_NESTED2;
if (tb[TCA_ACT_BPF_NAME])
NET_DUMP_STR("name", nla_getattr_str(tb[TCA_ACT_BPF_NAME]));
if (tb[TCA_ACT_BPF_ID])
NET_DUMP_UINT("bpf_id", nla_getattr_u32(tb[TCA_ACT_BPF_ID]));
if (tb[TCA_ACT_BPF_TAG])
NET_DUMP_STR("tag", hexstring_n2a(nla_data(tb[TCA_ACT_BPF_TAG]),
nla_len(tb[TCA_ACT_BPF_TAG]),
buf, sizeof(buf)));
NET_END_OBJECT_NESTED;
return 0;
}
static int do_dump_one_act(struct nlattr *attr)
{
struct nlattr *tb[TCA_ACT_MAX + 1];
if (!attr)
return 0;
if (nla_parse_nested(tb, TCA_ACT_MAX, attr, NULL) < 0)
return -LIBBPF_ERRNO__NLPARSE;
if (tb[TCA_ACT_KIND] && strcmp(nla_data(tb[TCA_ACT_KIND]), "bpf") == 0)
return do_bpf_dump_one_act(tb[TCA_ACT_OPTIONS]);
return 0;
}
static int do_bpf_act_dump(struct nlattr *attr)
{
struct nlattr *tb[TCA_ACT_MAX_PRIO + 1];
int act, ret;
if (nla_parse_nested(tb, TCA_ACT_MAX_PRIO, attr, NULL) < 0)
return -LIBBPF_ERRNO__NLPARSE;
NET_START_ARRAY("act", "");
for (act = 0; act <= TCA_ACT_MAX_PRIO; act++) {
ret = do_dump_one_act(tb[act]);
if (ret)
break;
}
NET_END_ARRAY(" ");
return ret;
}
static int do_bpf_filter_dump(struct nlattr *attr)
{
struct nlattr *tb[TCA_BPF_MAX + 1];
char buf[256];
int ret;
if (nla_parse_nested(tb, TCA_BPF_MAX, attr, NULL) < 0)
return -LIBBPF_ERRNO__NLPARSE;
if (tb[TCA_BPF_NAME])
NET_DUMP_STR("name", nla_getattr_str(tb[TCA_BPF_NAME]));
if (tb[TCA_BPF_ID])
NET_DUMP_UINT("prog_id", nla_getattr_u32(tb[TCA_BPF_ID]));
if (tb[TCA_BPF_TAG])
NET_DUMP_STR("tag", hexstring_n2a(nla_data(tb[TCA_BPF_TAG]),
nla_len(tb[TCA_BPF_TAG]),
buf, sizeof(buf)));
if (tb[TCA_BPF_ACT]) {
ret = do_bpf_act_dump(tb[TCA_BPF_ACT]);
if (ret)
return ret;
}
return 0;
}
int do_filter_dump(struct tcmsg *info, struct nlattr **tb, const char *kind)
{
int ret = 0;
if (tb[TCA_OPTIONS] && strcmp(nla_data(tb[TCA_KIND]), "bpf") == 0) {
NET_START_OBJECT;
NET_DUMP_UINT("ifindex", info->tcm_ifindex);
NET_DUMP_STR("kind", kind);
ret = do_bpf_filter_dump(tb[TCA_OPTIONS]);
NET_END_OBJECT_FINAL;
}
return ret;
}

View File

@ -0,0 +1,103 @@
// SPDX-License-Identifier: GPL-2.0+
// Copyright (C) 2018 Facebook
#ifndef _NETLINK_DUMPER_H_
#define _NETLINK_DUMPER_H_
#define NET_START_OBJECT \
{ \
if (json_output) \
jsonw_start_object(json_wtr); \
}
#define NET_START_OBJECT_NESTED(name) \
{ \
if (json_output) { \
jsonw_name(json_wtr, name); \
jsonw_start_object(json_wtr); \
} else { \
fprintf(stderr, "%s {", name); \
} \
}
#define NET_START_OBJECT_NESTED2 \
{ \
if (json_output) \
jsonw_start_object(json_wtr); \
else \
fprintf(stderr, "{"); \
}
#define NET_END_OBJECT_NESTED \
{ \
if (json_output) \
jsonw_end_object(json_wtr); \
else \
fprintf(stderr, "}"); \
}
#define NET_END_OBJECT \
{ \
if (json_output) \
jsonw_end_object(json_wtr); \
}
#define NET_END_OBJECT_FINAL \
{ \
if (json_output) \
jsonw_end_object(json_wtr); \
else \
fprintf(stderr, "\n"); \
}
#define NET_START_ARRAY(name, newline) \
{ \
if (json_output) { \
jsonw_name(json_wtr, name); \
jsonw_start_array(json_wtr); \
} else { \
fprintf(stderr, "%s [%s", name, newline);\
} \
}
#define NET_END_ARRAY(endstr) \
{ \
if (json_output) \
jsonw_end_array(json_wtr); \
else \
fprintf(stderr, "]%s", endstr); \
}
#define NET_DUMP_UINT(name, val) \
{ \
if (json_output) \
jsonw_uint_field(json_wtr, name, val); \
else \
fprintf(stderr, "%s %d ", name, val); \
}
#define NET_DUMP_LLUINT(name, val) \
{ \
if (json_output) \
jsonw_lluint_field(json_wtr, name, val);\
else \
fprintf(stderr, "%s %lld ", name, val); \
}
#define NET_DUMP_STR(name, str) \
{ \
if (json_output) \
jsonw_string_field(json_wtr, name, str);\
else \
fprintf(stderr, "%s %s ", name, str); \
}
#define NET_DUMP_STR_ONLY(str) \
{ \
if (json_output) \
jsonw_string(json_wtr, str); \
else \
fprintf(stderr, "%s ", str); \
}
#endif