Commit 0cf250d8 authored by Alexandre Duret-Lutz's avatar Alexandre Duret-Lutz

bin: introduce autcross

Fixes #252.

* NEWS: Mention it.
* bin/autcross.cc, bin/man/autcross.x, doc/org/autcross.org: New
files.
* bin/Makefile.am, bin/man/Makefile.am, doc/org/tools.org,
doc/Makefile.am: Add them.
* bin/autfilt.cc: Use is_universal() instead of is_deterministic().
* bin/common_hoaread.hh, bin/common_trans.cc, bin/common_trans.hh,
bin/ltlcross.cc, bin/ltldo.cc: Factor some bits common between
ltlcross, ltldo and autcross.
* tests/core/autcross.test, tests/core/autcross2.test: New files.
* tests/Makefile.am: Add them.
* tests/core/dra2dba.test, tests/core/sbacc.test,
tests/core/streett.test: Use autcross.
parent b9fff6a4
......@@ -2,6 +2,14 @@ New in spot 2.3.5.dev (not yet released)
Tools:
- genaut is a new binary that produces families of automata defined
in the literature (in the same way as we have genltl for LTL
formulas).
- autcross is a new binary that compares the output of tools
transforming automata (in the same way as we have ltlcross for
LTL translators).
- autfilt learned to build the union (--sum) or the intersection
(--sum-and) of two language by putting two automata side-by-side
and fiddling with the initial states. This complement the already
......@@ -11,9 +19,6 @@ New in spot 2.3.5.dev (not yet released)
- autfilt learned to complement any alternating automaton with
option --dualize.
- genaut is a binary to produce families of automata defined in the
literature (in the same way as we have genltl for LTL formulas).
- autfilt learned --split-edges to convert labels that are Boolean
formulas into labels that are min-terms. (See spot::split_edges()
below.)
......
autcross
autfilt
dstar2tgba
genaut
......
......@@ -59,6 +59,7 @@ libcommon_a_SOURCES = \
common_trans.hh
bin_PROGRAMS = \
autcross \
autfilt \
dstar2tgba \
genaut \
......@@ -77,6 +78,7 @@ bin_PROGRAMS = \
# version number that is automatically updated).
noinst_PROGRAMS = spot-x spot
autcross_SOURCES = autcross.cc
autfilt_SOURCES = autfilt.cc
ltlfilt_SOURCES = ltlfilt.cc
genaut_SOURCES = genaut.cc
......
This diff is collapsed.
......@@ -515,7 +515,7 @@ static bool opt_highlight_languages = false;
static spot::twa_graph_ptr
ensure_deterministic(const spot::twa_graph_ptr& aut, bool nonalt = false)
{
if ((!nonalt || aut->is_existential()) && spot::is_deterministic(aut))
if ((!nonalt || aut->is_existential()) && spot::is_universal(aut))
return aut;
spot::postprocessor p;
p.set_type(spot::postprocessor::Generic);
......
......@@ -43,6 +43,7 @@ read_automaton(const char* filename, spot::bdd_dict_ptr& dict);
class hoa_processor: public job_processor
{
protected:
spot::bdd_dict_ptr dict_;
public:
......
......@@ -33,14 +33,17 @@
#include <spot/tl/unabbrev.hh>
#include "common_conv.hh"
#include <spot/misc/escape.hh>
#include <spot/twaalgos/hoa.hh>
#include <spot/twaalgos/lbtt.hh>
#include <spot/twaalgos/neverclaim.hh>
// A set of tools for which we know the correct output
static struct shorthands_t
struct shorthands_t
{
const char* prefix;
const char* suffix;
}
shorthands[] = {
};
static shorthands_t shorthands_ltl[] = {
{ "lbt", " <%L>%O" },
{ "ltl2ba", " -f %s>%O" },
{ "ltl2da", " %f>%O" },
......@@ -55,25 +58,30 @@ static struct shorthands_t
{ "spin", " -f %s>%O" },
};
static void show_shorthands()
static shorthands_t shorthands_autproc[] = {
{ "autfilt", " %H>%O" },
{ "dstar2tgba", " %H>%O" },
{ "ltl2dstar", " -B %H %O" },
{ "nba2ldpa", " <%H>%O" },
{ "seminator", " %H>%O" },
};
static void show_shorthands(shorthands_t* begin, shorthands_t* end)
{
std::cout
<< ("If a COMMANDFMT does not use any %-sequence, and starts with one of\n"
"the following words, then the string on the right is appended.\n\n");
for (auto& s: shorthands)
std::cout << " "
<< std::left << std::setw(12) << s.prefix
<< s.suffix << '\n';
std::cout
<< ("\nAny {name} and directory component is skipped for the purpose of\n"
"matching those prefixes. So for instance\n"
" '{DRA} ~/mytools/ltl2dstar-0.5.2'\n"
"will changed into\n"
" '{DRA} ~/mytools/ltl2dstar-0.5.2 --output-format=hoa %[MR]L %O'\n");
while (begin != end)
{
std::cout << " "
<< std::left << std::setw(12) << begin->prefix
<< begin->suffix << '\n';
++begin;
}
}
tool_spec::tool_spec(const char* spec)
tool_spec::tool_spec(const char* spec, shorthands_t* begin, shorthands_t* end)
: spec(spec), cmd(spec), name(spec)
{
if (*cmd == '{')
......@@ -114,8 +122,9 @@ tool_spec::tool_spec(const char* spec)
++pos;
}
// Match a shorthand.
for (auto& p: shorthands)
while (begin != end)
{
auto& p = *begin++;
int n = strlen(p.prefix);
if (strncmp(basename, p.prefix, n) == 0)
{
......@@ -165,6 +174,20 @@ tool_spec::~tool_spec()
std::vector<tool_spec> tools;
void tools_push_trans(const char* trans)
{
tools.emplace_back(trans,
std::begin(shorthands_ltl),
std::end(shorthands_ltl));
}
void tools_push_autproc(const char* proc)
{
tools.emplace_back(proc,
std::begin(shorthands_autproc),
std::end(shorthands_autproc));
}
void
quoted_string::print(std::ostream& os, const char*) const
{
......@@ -254,16 +277,8 @@ printable_result_filename::print(std::ostream& os, const char*) const
spot::quote_shell_string(os, val()->name());
}
void
filed_formula::print(std::ostream& os, const char* pos) const
{
std::ostringstream ss;
f_.print(ss, pos);
os << '\'' << string_to_tmp(ss.str(), serial_) << '\'';
}
std::string
filed_formula::string_to_tmp(const std::string str, unsigned n) const
static std::string
string_to_tmp(const std::string str, unsigned n)
{
char prefix[30];
snprintf(prefix, sizeof prefix, "lcr-i%u-", n);
......@@ -278,6 +293,43 @@ filed_formula::string_to_tmp(const std::string str, unsigned n) const
return tmpname;
}
void
filed_formula::print(std::ostream& os, const char* pos) const
{
std::ostringstream ss;
f_.print(ss, pos);
os << '\'' << string_to_tmp(ss.str(), serial_) << '\'';
}
void
filed_automaton::print(std::ostream& os, const char* pos) const
{
std::ostringstream ss;
char* opt = nullptr;
if (*pos == '[')
{
++pos;
auto end = strchr(pos, ']');
opt = strndup(pos, end - pos);
pos = end + 1;
}
switch (*pos)
{
case 'H':
spot::print_hoa(ss, aut_, opt);
break;
case 'S':
spot::print_never_claim(ss, aut_, opt);
break;
case 'L':
spot::print_lbtt(ss, aut_, opt);
break;
}
if (opt)
free(opt);
os << '\'' << string_to_tmp(ss.str(), serial_) << '\'';
}
translator_runner::translator_runner(spot::bdd_dict_ptr dict,
bool no_output_allowed)
: dict(dict)
......@@ -345,6 +397,42 @@ translator_runner::round_formula(spot::formula f, unsigned serial)
filename_formula.new_round(serial);
}
autproc_runner::autproc_runner(bool no_output_allowed)
{
declare('H', &filename_automaton);
declare('S', &filename_automaton);
declare('L', &filename_automaton);
declare('O', &output);
size_t s = tools.size();
assert(s);
for (size_t n = 0; n < s; ++n)
{
// Check that each translator uses at least one input and
// one output.
std::vector<bool> has(256);
const tool_spec& t = tools[n];
scan(t.cmd, has);
if (!(has['H'] || has['S'] || has['L']))
error(2, 0, "no input %%-sequence in '%s'.\n Use "
"one of %%H,%%S,%%L to indicate the input automaton filename.",
t.spec);
if (!no_output_allowed && !has['O'])
error(2, 0, "no output %%-sequence in '%s'.\n Use "
"%%O to indicate where the automaton is output.",
t.spec);
// Remember the %-sequences used by all tools.
prime(t.cmd);
}
}
void
autproc_runner::round_automaton(spot::const_twa_graph_ptr aut, unsigned serial)
{
filename_automaton.new_round(aut, serial);
}
volatile bool timed_out = false;
unsigned timeout_count = 0;
......@@ -553,7 +641,7 @@ exec_command(const char* cmd)
int fd0 = open(stdin, O_RDONLY, 0644);
if (fd0 < 0)
error(2, errno, "failed to open '%s'", stdin);
if (dup2(fd0, 0) < 0)
if (dup2(fd0, STDIN_FILENO) < 0)
error(2, errno, "dup2() failed");
if (close(fd0) < 0)
error(2, errno, "close() failed");
......@@ -563,7 +651,7 @@ exec_command(const char* cmd)
int fd1 = creat(stdout, 0644);
if (fd1 < 0)
error(2, errno, "failed to open '%s'", stdout);
if (dup2(fd1, 1) < 0)
if (dup2(fd1, STDOUT_FILENO) < 0)
error(2, errno, "dup2() failed");
if (close(fd1) < 0)
error(2, errno, "close() failed");
......@@ -602,6 +690,11 @@ exec_with_timeout(const char* cmd)
if (child_pid == 0)
{
setpgid(0, 0);
// Close stdin so that children may not read our input. We had
// this nice surprise with Seminator, who greedily consumes its
// stdin (which was also ours) even if it does not use it
// because it was given a file.
close(STDIN_FILENO);
exec_command(cmd);
// never reached
return -1;
......@@ -675,7 +768,7 @@ static int parse_opt_trans(int key, char* arg, struct argp_state*)
switch (key)
{
case 't':
tools.push_back(arg);
tools_push_trans(arg);
break;
case 'T':
timeout = to_pos_int(arg);
......@@ -685,7 +778,14 @@ static int parse_opt_trans(int key, char* arg, struct argp_state*)
#endif
break;
case OPT_LIST:
show_shorthands();
show_shorthands(std::begin(shorthands_ltl), std::end(shorthands_ltl));
std::cout
<< ("\nAny {name} and directory component is skipped for the purpose "
"of\nmatching those prefixes. So for instance\n "
"'{DRA} ~/mytools/ltl2dstar-0.5.2'\n"
"will be changed into\n "
"'{DRA} ~/mytools/ltl2dstar-0.5.2 --output-format=hoa %[MW]L %O'"
"\n");
exit(0);
case OPT_RELABEL:
opt_relabel = true;
......@@ -698,3 +798,61 @@ static int parse_opt_trans(int key, char* arg, struct argp_state*)
const struct argp trans_argp = { options, parse_opt_trans, nullptr, nullptr,
nullptr, nullptr, nullptr };
static const argp_option options_aut[] =
{
/**************************************************/
{ nullptr, 0, nullptr, 0, "Specifying tools to call:", 2 },
{ "tool", 't', "COMMANDFMT", 0,
"register one tool to call", 0 },
{ "timeout", 'T', "NUMBER", 0, "kill tools after NUMBER seconds", 0 },
{ "list-shorthands", OPT_LIST, nullptr, 0,
"list availabled shorthands to use in COMMANDFMT", 0},
/**************************************************/
{ nullptr, 0, nullptr, 0,
"COMMANDFMT should specify input and output arguments using the "
"following character sequences:", 3 },
{ "%H,%S,%L", 0, nullptr, OPTION_DOC | OPTION_NO_USAGE,
"filename for the input automaton in (H) HOA, (S) Spin's neverclaim, "
"or (L) LBTT's format", 0 },
{ "%O", 0, nullptr, OPTION_DOC | OPTION_NO_USAGE,
"filename for the automaton output in HOA, never claim, LBTT, or "
"ltl2dstar's format", 0 },
{ "%%", 0, nullptr, OPTION_DOC | OPTION_NO_USAGE, "a single %", 0 },
{ nullptr, 0, nullptr, 0, nullptr, 0 }
};
static int parse_opt_autproc(int key, char* arg, struct argp_state*)
{
switch (key)
{
case 't':
tools_push_autproc(arg);
break;
case 'T':
timeout = to_pos_int(arg);
#if !ENABLE_TIMEOUT
std::cerr << "warning: setting a timeout is not supported "
<< "on your platform" << std::endl;
#endif
break;
case OPT_LIST:
show_shorthands(std::begin(shorthands_autproc),
std::end(shorthands_autproc));
std::cout
<< ("\nAny {name} and directory component is skipped for the purpose "
"of\nmatching those prefixes. So for instance\n "
"'{AF} ~/mytools/autfilt-2.4 --remove-fin'\n"
"will be changed into\n "
"'{AF} ~/mytools/autfilt-2.4 --remove-fin %H>%O'\n");
exit(0);
default:
return ARGP_ERR_UNKNOWN;
}
return 0;
}
const struct argp autproc_argp = { options_aut, parse_opt_autproc, nullptr,
nullptr, nullptr, nullptr, nullptr };
......@@ -28,9 +28,13 @@
#include <spot/twa/twagraph.hh>
extern const struct argp trans_argp;
extern const struct argp trans_argp; // ltlcross, ltldo
extern const struct argp autproc_argp; // autcross
extern bool opt_relabel;
struct shorthands_t;
struct tool_spec
{
// The translator command, as specified on the command-line.
......@@ -44,7 +48,7 @@ struct tool_spec
// name of the translator (or spec)
const char* name;
tool_spec(const char* spec);
tool_spec(const char* spec, shorthands_t* begin, shorthands_t* end);
tool_spec(const tool_spec& other);
tool_spec& operator=(const tool_spec& other);
~tool_spec();
......@@ -52,6 +56,9 @@ struct tool_spec
extern std::vector<tool_spec> tools;
void tools_push_trans(const char* trans);
void tools_push_autproc(const char* proc);
struct quoted_string final: public spot::printable_value<std::string>
{
using spot::printable_value<std::string>::operator=;
......@@ -80,7 +87,25 @@ struct filed_formula final: public spot::printable
private:
const quoted_formula& f_;
unsigned serial_;
std::string string_to_tmp(const std::string str, unsigned n) const;
};
struct filed_automaton final: public spot::printable
{
filed_automaton()
{
}
void print(std::ostream& os, const char* pos) const override;
void new_round(spot::const_twa_graph_ptr aut, unsigned serial)
{
aut_ = aut;
serial_ = serial;
}
private:
spot::const_twa_graph_ptr aut_;
unsigned serial_;
};
struct printable_result_filename final:
......@@ -118,6 +143,23 @@ public:
};
class autproc_runner: protected spot::formater
{
protected:
// Round-specific variables
filed_automaton filename_automaton;
// Run-specific variables
printable_result_filename output;
public:
using spot::formater::has;
autproc_runner(// whether we accept the absence of output
// specifier
bool no_output_allowed = false);
void round_automaton(spot::const_twa_graph_ptr aut, unsigned serial);
};
// Disable handling of timeout on systems that miss kill() or alarm().
// For instance MinGW.
#if HAVE_KILL && HAVE_ALARM
......
......@@ -219,7 +219,6 @@ example()
return std::cerr;
}
static void
end_error()
{
......@@ -423,7 +422,7 @@ parse_opt(int key, char* arg, struct argp_state*)
if (arg[0] == '-' && !arg[1])
jobs.emplace_back(arg, true);
else
tools.push_back(arg);
tools_push_trans(arg);
break;
case OPT_AUTOMATA:
opt_automata = true;
......@@ -1111,7 +1110,7 @@ namespace
std::vector<spot::twa_graph_ptr>& comp,
unsigned i)
{
if (!no_complement && x[i] && is_deterministic(x[i]))
if (!no_complement && x[i] && is_universal(x[i]))
comp[i] = dualize(x[i]);
};
......
......@@ -171,7 +171,7 @@ parse_opt(int key, char* arg, struct argp_state*)
if (arg[0] == '-' && !arg[1])
jobs.emplace_back(arg, true);
else
tools.push_back(arg);
tools_push_trans(arg);
break;
default:
return ARGP_ERR_UNKNOWN;
......
......@@ -25,6 +25,7 @@ convman7 = ARGP_HELP_FMT=header-col=0 $(SHELL) "$(x_to_1)" \
"$(PERL)" "$(top_srcdir)/tools/help2man -s7 -N"
dist_man1_MANS = \
autcross.1 \
autfilt.1 \
dstar2tgba.1 \
genaut.1 \
......@@ -44,6 +45,9 @@ dist_man7_MANS = \
MAINTAINERCLEANFILES = $(dist_man1_MANS) $(dist_man7_MANS)
EXTRA_DIST = $(dist_man1_MANS:.1=.x) $(dist_man7_MANS:.7=.x)
autcross.1: $(common_dep) $(srcdir)/autcross.x $(srcdir)/../autcross.cc
$(convman) ../autcross$(EXEEXT) $(srcdir)/autcross.x $@
autfilt.1: $(common_dep) $(srcdir)/autfilt.x $(srcdir)/../autfilt.cc
$(convman) ../autfilt$(EXEEXT) $(srcdir)/autfilt.x $@
......
.\" -*- coding: utf-8 -*-
[NAME]
autcross \- cross-compare tools that process automata
[ENVIRONMENT VARIABLES]
.TP
\fBSPOT_TMPDIR\fR, \fBTMPDIR\fR
These variables control in which directory temporary files (e.g.,
those who contain the input and output when interfacing with
translators) are created. \fBTMPDIR\fR is only read if
\fBSPOT_TMPDIR\fR does not exist. If none of these environment
variables exist, or if their value is empty, files are created in the
current directory.
.TP
\fBSPOT_TMPKEEP\fR
When this variable is defined, temporary files are not removed.
This is mostly useful for debugging.
[OUTPUT DATA]
The following columns are output in the CSV files.
.TP
\fBinput.source\fR
Location of the input automaton fed to the tool.
.TP
\fBinput.name\fR
Name of the input automaton, if any. This is supported
by the HOA format.
.TP
\fBinput.ap\fR,\fBoutput.ap\fR,
Number of atomic proposition in the input and output automata.
.TP
\fBinput.states\fR,\fBoutput.states\fR
Number of states in the input and output automata.
.TP
\fBinput.edges\fR,\fBoutput.edges\fR
Number of edges in the input and output automata.
.TP
\fBinput.transitions\fR,\fBoutput.transitions\fR
Number of transitions in the input and output automata.
.TP
\fBinput.acc_sets\fR,\fBoutput.acc_sets\fR
Number of acceptance sets in the input and output automata.
.TP
\fBinput.scc\fR,\fBoutput.scc\fR
Number of strongly connected components in the input and output automata.
.TP
\fBinput.nondetstates\fR,\fBoutput.nondetstates\fR
Number of nondeterministic states in the input and output automata.
.TP
\fBinput.nondeterministic\fR,\fBoutput.nondetstates\fR
1 if the automaton is nondeterministic, 0 if it is deterministic.
.TP
\fBinput.alternating\fR,\fBoutput.alternating\fR
1 if the automaton has some universal branching, 0 otherwise.
\fBexit_status\fR, \fBexit_code\fR
Information about how the execution of the tool went.
\fBexit_status\fR is a string that can take the following
values:
.RS
.TP
\f(CW"ok"\fR
The tool ran succesfully (this does not imply that the produced
automaton is correct) and autcross could parse the resulting
automaton. In this case \fBexit_code\fR is always 0.
.TP
\f(CW"timeout"\fR
The tool ran for more than the number of seconds
specified with the \fB\-\-timeout\fR option. In this
case \fBexit_code\fR is always -1.
.TP
\f(CW"exit code"\fR
The tool terminated with a non-zero exit code.
\fBexit_code\fR contains that value.
.TP
\f(CW"signal"\fR
The tool terminated with a signal.
\fBexit_code\fR contains that signal's number.
.TP
\f(CW"parse error"\fR
The tool terminated normally, but autcross could not
parse its output. In this case \fBexit_code\fR is always -1.
.TP
\f(CW"no output"\fR
The tool terminated normally, but without creating the specified
output file. In this case \fBexit_code\fR is always -1.
.RE
.TP
\fBtime\fR
A floating point number giving the run time of the tool in seconds.
This is reported for all executions, even failling ones.
[SEE ALSO]
.BR randaut (1),
.BR genaut (1),
.BR autfilt (1),
.BR ltlcross (1)
......@@ -78,6 +78,7 @@ ORG_FILES = \
org/spot.css \
org/arch.tex \
$(srcdir)/org/arch.png \
org/autcross.org \
org/autfilt.org \
org/csv.org \
org/citing.org \
......
# -*- coding: utf-8 -*-
#+TITLE: =autcross=
#+DESCRIPTION: Spot command-line tool for cross-comparing the output of automaton processors.
#+SETUPFILE: setup.org
#+HTML_LINK_UP: tools.html
=autcross= is a tool for cross-comparing the output of tools that
transform automata. It works similarly to [[file:ltlcross.org][=ltlcross=]] except that
inputs are automata.
The core of =autcross= is a loop that does the following steps:
- Input an automaton
- Process that input automaton with all the configured tools.
If there are 3 translators, the automata produced by those tools
will be named =A0=, =A1=, and =A2=.
- Ensure that all produced automata are equivalent.
Statistics about the results of each tools can optionally be saved in
a CSV file. And in case only those statistics matters, it is also
possible to disable the equivalence checks.
* Input automata
The input automata can be supplied either on standard input, or in
files specified with option =-F=.
If an input automaton is expressed in the HOA format and has a name,
that name will be displayed by =autcross= when processing the automaton,
and will appear in the CSV file if such a file is output.
* Specifying the tools to test
Each tool should be specified as a string that uses some of the