diff --git a/cmake/config_failbochs.sh.in b/cmake/config_failbochs.sh.in deleted file mode 100755 index 7cbf7693..00000000 --- a/cmake/config_failbochs.sh.in +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -SOURCE_DIR=@CMAKE_SOURCE_DIR@ -BINARY_DIR=@CMAKE_BINARY_DIR@ -PREFIX_DIR=@BOCHS_PREFIX_DIR@ - -if [ ! -d "$SOURCE_DIR" ]; then - echo Source directory does not exists! $SOURCE_DIR - exit 2 -fi - -if [ ! -d "$PREFIX_DIR" ]; then - echo Prefix directory does not exists! $BINARY_DIR - exit 2 -fi - - -./configure CXX="ag++ -p $SOURCE_DIR -I$SOURCE_DIR/src -I"$BINARY_DIR"/src --real-instances --Xcompiler" LIBTOOL="/bin/sh ./libtool --tag=CXX" --prefix=$PREFIX_DIR --enable-{a20-pin,x86-64,cpu-level=6,ne2000,acpi,pci,usb,repeat-speedups,trace-cache,fast-function-calls,host-specific-asms,disasm,all-optimizations,readline,clgd54xx,fpu,vmx=2,monitor-mwait,cdrom,sb16=linux,gdb-stub} --with-all-libs - diff --git a/doc/how-to-build.txt b/doc/how-to-build.txt index 1ab9fe33..ad8f8142 100644 --- a/doc/how-to-build.txt +++ b/doc/how-to-build.txt @@ -9,16 +9,18 @@ Required for Fail*: - libpcl1-dev - libboost-thread-dev - protobuf-compiler - - cmake + - cmake 2.8.2 (2.8.11 preferred) - cmake-curses-gui - binutils-dev - AspectC++ (ag++, ac++): AspectC++ 1.1 or newer is known to work and can be - obtained from http://www.aspectc.org; nightlies can be downloaded from - http://akut.aspectc.org + obtained from ; nightlies can be downloaded from + . Make sure you use the 64-bit version if running + in a 64-bit environment. - optional: * LLVM 3.3 (needed for several importers in tools/import-trace) (compiles/links with 3.1 or 3.2, but fails to properly import information from ELF binaries not compiled with -ffunction-sections) + * a MySQL 5.0+ or MariaDB 5.1+ (MariaDB 5.5 recommended) server Required for the Bochs simulator backend: @@ -66,11 +68,8 @@ For the first time: $ cd fail/ 2. (Optional) Cleanup previous CMake remnants: $ find -name CMakeCache.txt | xargs rm - 3. Create out of source build directory (${BUILD_DIR}, see also "fail-structure.txt"): + 3. Create out-of-source build directory (${BUILD_DIR}, see also "fail-structure.txt"): $ mkdir build - Note that currently this build directory must be located somewhere below - the fail/ directory, as generated .ah files in there will not be included - in the compile process otherwise. 4. Enter out-of-source build directory. All generated files end up there. $ cd build 5. Generate CMake environment. @@ -168,6 +167,7 @@ Configure Bochs to use debugging-related compiler flags (expects to be in ${BUIL $ CFLAGS="-g -O0" CXXFLAGS="-g -O0" ./configure --prefix=... ... (see above) You might additionally want to configure the rest of Fail* into debug mode by setting CMAKE_BUILD_TYPE to "Debug" (ccmake, see above). +FIXME: Does this still work? Profiling-based optimization build: @@ -233,3 +233,27 @@ After changes to Fail*/gem5 code (incl. aspect headers): $ make (or nice make -jN) (as in the last step of the previous section). +========================================================================================= +Database backend setup: MySQL / MariaDB +========================================================================================= + 1. Install MySQL or MariaDB via your favourite package manager. (MariaDB + offers repositories for all major distributions.) + 2. Increase network timeouts in /etc/mysql/my.cnf to a reasonably high value, + campaigns tend to timeout on a regular basis with the default settings: + net_write_timeout = 3600 + net_read_timeout = 3600 + You may also want to tune other settings, especially increase memory usage. + There's lots of tuning knowledge on the web, and also automatic tuning + scripts such as the "Tuning Primer" + or "MySQL Tuner" . + 3. Setup a non-root DB user. + Create a ~/.my.cnf (mode 0600) to avoid having to pass user name and + password to every program contacting the DB: + [client] + user=XXX + password=XXX + 4. Create a database for each FI campaign you want to conduct, and grant all + permissions to your non-root user. + + + diff --git a/doc/how-to-use.txt b/doc/how-to-use.txt index 980449f5..7611d771 100644 --- a/doc/how-to-use.txt +++ b/doc/how-to-use.txt @@ -112,6 +112,63 @@ An example for separate campaign/experiment implementations. 2. Run the FailBochs instance, in properly defined environment: $ fail-client -q +Experiment "dciao-kernelstructs": +********************************************************************** +This is an example for a database-driven FI campaign, with a campaign server +instantiating the generic DatabaseCampaign. The general workflow is as follows: + - Create a new database for this campaign, e.g., "dciao". (See + how-to-build.txt, "Database backend setup") + - Record a trace using the generic tracing tool: Build a fail-client with the + "generic-tracing" experiment, and parametrize it with -Wf,[option] (e.g., + "fail-client -Wf,--help"). Be aware that the generic tracing "experiment" + needs a complete simulator environment along with the ELF binary that + supplies symbol addresses; for FailBochs this means the usual bochsrc and + boot image files need to be present. + FIXME: dciao-kernelstructs has not been committed to + danceos/devel/experiment_targets/, use a "complete" example here + - Import the trace to the database using the import-trace tool (enable + BUILD_IMPORT_TRACE in the CMake configuration to get this built). The + --variant/--benchmark parameters are only significant if you intend to + evaluate multiple protection variants and/or benchmarks. Currently the + following importers (--importer X) are implemented: + MemoryImporter: Imports fault locations for single-location single-event + RAM faults (e.g., single-bit flips, or burst faults within the same + byte). Directly maps memory access events from the trace to the DB. + AdvancedMemoryImporter: A MemoryImporter that additionally imports + Relyzer-style conditional branch history, instruction opcodes, and a + virtual duration = time2 - time1 + 1 column. + RegisterImporter: Imports fault locations for single-location single-event + register-file faults (e.g., single-bit flips in general-purpose + registers). Considers only instruction addresses from the trace, + disassembles corresponding instructions in the supplied ELF binary (using + LLVM's disassembler library), and extracts used/defined registers. + Registers are mapped to/from "addresses" with fail::LLVMtoFailTranslator. + InstructionImporter: Interprets every instruction fetch as a memory read, + and handles it the same way the MemoryImporter handles "normal" memory + accesses. Implements a fault model with faults in CPU instructions. + RandomJumpImporter: Implements multi-bit faults in the instruction pointer. + As the IP is read before every instruction, the fault space explodes + rapidly. Should therefore be limited to small memory areas to jump + from/to. + Note that specifying an importer with --importer adds more parameters to the + --help output in some cases. + DB detail: This tool creates and fills the variant and trace tables. + - Prune the fault space with the prune-trace tool (enable BUILD_PRUNE_TRACE in + the CMake configuration). This prepares all information necessary for + running the FI campaign. Currently only the "basic" pruning method is + available, applying usual def/use pruning. + DB detail: This tool creates and fills the fsppilot, fspgroup and fspmethod + tables. + - Run the campaign server as usual. In this case it does not do much more than + to instantiate the generic DatabaseCampaign, and to parametrize it with the + experiment-specific protobuf message type (DCIAOKernelProtoMsg) which must + adhere to some structural constraints (a required DatabaseCampaignMessage + fsppilot member, and a repeated group Result that contains no further + subgroups or repeated fields). The DatabaseCampaign takes care of + distributing unfinished jobs, and collecting the results in an automatically + created result table with columns corresponding to the fields in the Result + group of the protobuf message. + ========================================================================================= Parallelization ========================================================================================= @@ -165,16 +222,6 @@ Some useful things to note: - The runcampaign.sh script starts the coolchecksum-server. Note that the server instance will terminate immediately (without notice), if there is still an existing coolcampaign.csv file. - - In order to make the performance gains (mentioned above) take effect, a "workload - balancing" between the server and the clients is mandatory. This means that - the communication overhead (client <-> server) and the time needed to execute - the experiment code on the client-side should be in due proportion. More - specifically, for each experiment there will be exactly 2 TCP connections - (send parameter set to client, send result to server) established. Therefore - you should ensure that the jobs you distribute take enough time not to - overflow the server with requests. You may need to bundle parameters for - more than one experiment if a single experiment only takes a few hundred - milliseconds. (See existing experiments for examples.) ========================================================================================= Steps to run an experiment with gem5: diff --git a/src/core/sal/Architecture.hpp b/src/core/sal/Architecture.hpp new file mode 100644 index 00000000..01419379 --- /dev/null +++ b/src/core/sal/Architecture.hpp @@ -0,0 +1,15 @@ +// Architecture.hpp: wraps architecture definition headers +#ifndef __ARCHITECTURE_HPP__ + #define __ARCHITECTURE_HPP__ + +#include "config/FailConfig.hpp" + +#ifdef BUILD_X86 +#include "x86/X86Architecture.hpp" +#endif + +#ifdef BUILD_ARM +#include "arm/ARMArchitecture.hpp" +#endif + +#endif diff --git a/src/core/sal/CPU.cc b/src/core/sal/CPU.cc index 2315fa92..815c9f54 100644 --- a/src/core/sal/CPU.cc +++ b/src/core/sal/CPU.cc @@ -22,6 +22,7 @@ void CPUArchitecture::m_addRegister(Register* reg, RegisterType type) Register* CPUArchitecture::getRegister(size_t i) const { assert(i < m_Registers.size() && "FATAL ERROR: Invalid index provided!"); + assert(m_Registers[i]->getId() == i && "FATAL ERROR: Register index mismatch"); return m_Registers[i]; } diff --git a/src/core/sal/CPU.hpp b/src/core/sal/CPU.hpp index 65ff9d3e..23a020f5 100644 --- a/src/core/sal/CPU.hpp +++ b/src/core/sal/CPU.hpp @@ -13,7 +13,7 @@ namespace fail { /** * \class CPUArchitecture * This is the base class for CPU architectures that can be used to merge information and - * functionallity that every backend with the same target architecture will share. The classes + * functionality that every backend with the same target architecture will share. The classes * directly derived from this are especially meant to be usable in campaigns, so they shouldn't * contain any backend specific code. */ diff --git a/src/core/sal/Register.hpp b/src/core/sal/Register.hpp index 66f5ec9b..379220aa 100644 --- a/src/core/sal/Register.hpp +++ b/src/core/sal/Register.hpp @@ -102,7 +102,7 @@ public: */ iterator begin() { return m_Regs.begin(); } /** - * Returns an iterator to the end of the interal data structure. + * Returns an iterator to the end of the internal data structure. * \code * [1|2| ... |n]X * ^ diff --git a/src/core/sal/arm/ArmArchitecture.hpp b/src/core/sal/arm/ArmArchitecture.hpp index 1184867e..e360c5b5 100644 --- a/src/core/sal/arm/ArmArchitecture.hpp +++ b/src/core/sal/arm/ArmArchitecture.hpp @@ -17,6 +17,10 @@ public: ~ArmArchitecture(); }; +#ifdef BUILD_ARM +typedef ArmArchitecture Architecture; +#endif + /** * \enum GPRegIndex * Defines the general purpose (GP) register identifier for the ARM diff --git a/src/core/sal/x86/X86Architecture.hpp b/src/core/sal/x86/X86Architecture.hpp index ac09a28c..fa1e64f3 100644 --- a/src/core/sal/x86/X86Architecture.hpp +++ b/src/core/sal/x86/X86Architecture.hpp @@ -18,6 +18,10 @@ public: ~X86Architecture(); }; +#ifdef BUILD_X86 +typedef X86Architecture Architecture; +#endif + /** * \enum GPRegisterId * Symbolic identifier to access the x86 general purpose register diff --git a/src/core/util/Database.cc b/src/core/util/Database.cc index 19094b48..dc0b8073 100644 --- a/src/core/util/Database.cc +++ b/src/core/util/Database.cc @@ -8,7 +8,15 @@ static fail::Logger LOG("Database", true); using namespace fail; +#ifndef __puma +boost::mutex Database::m_global_lock; +#endif + Database::Database(const std::string &username, const std::string &host, const std::string &database) { +#ifndef __puma + boost::lock_guard guard(m_global_lock); +#endif + handle = mysql_init(0); last_result = 0; mysql_options(handle, MYSQL_READ_DEFAULT_FILE, "~/.my.cnf"); @@ -26,6 +34,9 @@ Database::~Database() // flush cached INSERTs if available insert_multiple(); +#ifndef __puma + boost::lock_guard guard(m_global_lock); +#endif mysql_close(handle); } @@ -217,7 +228,7 @@ void Database::cmdline_setup() { HOSTNAME = cmd.addOption("H", "hostname", Arg::Required, "-h/--hostname \tMYSQL Hostname (default: taken from ~/.my.cnf)"); USERNAME = cmd.addOption("u", "username", Arg::Required, - "-u/--username \tMYSQL Username (default: taken from ~/.my.cnf, or your current user)"); + "-u/--username \tMYSQL Username (default: taken from ~/.my.cnf, or your current user)\n"); } Database * Database::cmdline_connect() { diff --git a/src/core/util/Database.hpp b/src/core/util/Database.hpp index ff234f7f..66c0710b 100644 --- a/src/core/util/Database.hpp +++ b/src/core/util/Database.hpp @@ -23,6 +23,7 @@ namespace fail { MYSQL_RES *last_result; // !< Used for mysql_result_free #ifndef __puma boost::mutex m_handle_lock; + static boost::mutex m_global_lock; #endif std::string m_insertquery; std::vector m_insertquery_values; diff --git a/src/core/util/llvmdisassembler/LLVMDisassembler.cpp b/src/core/util/llvmdisassembler/LLVMDisassembler.cpp index 590e3b11..aff689e2 100644 --- a/src/core/util/llvmdisassembler/LLVMDisassembler.cpp +++ b/src/core/util/llvmdisassembler/LLVMDisassembler.cpp @@ -105,6 +105,7 @@ void LLVMDisassembler::disassemble() instr_info.opcode = Inst.getOpcode(); instr_info.length = Size; instr_info.address = SectionAddr + Index; + instr_info.conditional_branch = desc.isConditionalBranch(); unsigned int pos = 0; for (MCInst::iterator it = Inst.begin(); it != Inst.end(); ++it) { diff --git a/src/core/util/llvmdisassembler/LLVMDisassembler.hpp b/src/core/util/llvmdisassembler/LLVMDisassembler.hpp index 75535053..48f442a4 100644 --- a/src/core/util/llvmdisassembler/LLVMDisassembler.hpp +++ b/src/core/util/llvmdisassembler/LLVMDisassembler.hpp @@ -42,6 +42,7 @@ public: unsigned int opcode; unsigned int address; unsigned char length; + bool conditional_branch; std::vector reg_uses; std::vector reg_defs; }; diff --git a/src/core/util/llvmdisassembler/testing/llvmDisTest.cc b/src/core/util/llvmdisassembler/testing/llvmDisTest.cc index 4ada92ee..f2a1fc59 100644 --- a/src/core/util/llvmdisassembler/testing/llvmDisTest.cc +++ b/src/core/util/llvmdisassembler/testing/llvmDisTest.cc @@ -62,6 +62,10 @@ int main(int argc, char* argv[]) { it != instr.reg_defs.end(); ++it) { std::cout << reg_info.getName(*it) << "(" << *it << ") "; } + + if (instr.conditional_branch) { + std::cout << "(conditional branch)"; + } std::cout << std::endl; } } diff --git a/src/experiments/generic-tracing/experiment.cc b/src/experiments/generic-tracing/experiment.cc index f7a114c9..0d57f049 100644 --- a/src/experiments/generic-tracing/experiment.cc +++ b/src/experiments/generic-tracing/experiment.cc @@ -41,7 +41,7 @@ void GenericTracing::parseOptions() { CommandLine::option_handle FULL_TRACE = cmd.addOption("", "full-trace", Arg::None, "--full-trace \tDo a full trace (more data, default: off)"); CommandLine::option_handle MEM_SYMBOL = cmd.addOption("m", "memory-symbol", Arg::Required, - "-m,--memory-symbol \tELF symbol(s) to trace accesses (without specifiying all mem read/writes are traced)"); + "-m,--memory-symbol \tELF symbol(s) to trace accesses (default: all mem read/writes are traced)"); CommandLine::option_handle MEM_REGION = cmd.addOption("M", "memory-region", Arg::Required, "-M,--memory-region \trestrict memory region which is traced" " Possible formats: 0x
, 0x
:0x
, 0x
:"); diff --git a/src/experiments/instantiate-experiment-indirect.ah.in b/src/experiments/instantiate-experiment-indirect.ah.in index b3a72752..e40f15dc 100644 --- a/src/experiments/instantiate-experiment-indirect.ah.in +++ b/src/experiments/instantiate-experiment-indirect.ah.in @@ -5,10 +5,6 @@ // includes generated headers (e.g., protobuf message definitions) that are not // guaranteed to exist when the aspect is woven. -// FIXME: cmake does not remove these .ah files when the user configures -// another experiment (or even makes "clean"). Currently, this needs to be -// worked around by manually removing $BUILDDIR/core/experiments/*/*.ah . - // You need to provide the implementation of this function in your experiment // directory: void instantiate@EXPERIMENT_TYPE@(); diff --git a/src/experiments/instantiate-experiment.ah.in b/src/experiments/instantiate-experiment.ah.in index 5bf1ed32..d6cb24b1 100644 --- a/src/experiments/instantiate-experiment.ah.in +++ b/src/experiments/instantiate-experiment.ah.in @@ -1,10 +1,6 @@ #ifndef __INSTANTIATE_@EXPERIMENT_TYPE@_AH__ #define __INSTANTIATE_@EXPERIMENT_TYPE@_AH__ -// FIXME: cmake does not remove these .ah files when the user configures -// another experiment (or even makes "clean"). Currently, this needs to be -// worked around by manually removing $BUILDDIR/core/experiments/*/*.ah . - // Make sure your experiment declaration is in experiment.hpp: #include "../experiments/@EXPERIMENT_NAME@/experiment.hpp" #include "sal/SALInst.hpp" diff --git a/src/experiments/nanojpeg/campaign.cc b/src/experiments/nanojpeg/campaign.cc index 04e4a08e..d7fc8e91 100644 --- a/src/experiments/nanojpeg/campaign.cc +++ b/src/experiments/nanojpeg/campaign.cc @@ -52,14 +52,14 @@ bool NanoJPEGCampaign::run() // list: latest accesses (instr offset | bit mask) map > > reg_cascade; // open up an equivalence class for all bits in all GPRs - reg_cascade[RID_EAX].push_front(std::pair(0, 0xffffffffffffffffULL)); - reg_cascade[RID_EBX].push_front(std::pair(0, 0xffffffffffffffffULL)); - reg_cascade[RID_ECX].push_front(std::pair(0, 0xffffffffffffffffULL)); - reg_cascade[RID_EDX].push_front(std::pair(0, 0xffffffffffffffffULL)); - reg_cascade[RID_ESP].push_front(std::pair(0, 0xffffffffffffffffULL)); - reg_cascade[RID_EBP].push_front(std::pair(0, 0xffffffffffffffffULL)); - reg_cascade[RID_ESI].push_front(std::pair(0, 0xffffffffffffffffULL)); - reg_cascade[RID_EDI].push_front(std::pair(0, 0xffffffffffffffffULL)); + reg_cascade[RID_CAX].push_front(std::pair(0, 0xffffffffffffffffULL)); + reg_cascade[RID_CBX].push_front(std::pair(0, 0xffffffffffffffffULL)); + reg_cascade[RID_CCX].push_front(std::pair(0, 0xffffffffffffffffULL)); + reg_cascade[RID_CDX].push_front(std::pair(0, 0xffffffffffffffffULL)); + reg_cascade[RID_CSP].push_front(std::pair(0, 0xffffffffffffffffULL)); + reg_cascade[RID_CBP].push_front(std::pair(0, 0xffffffffffffffffULL)); + reg_cascade[RID_CSI].push_front(std::pair(0, 0xffffffffffffffffULL)); + reg_cascade[RID_CDI].push_front(std::pair(0, 0xffffffffffffffffULL)); // load trace ifstream tracef(NANOJPEG_TRACE); @@ -145,7 +145,7 @@ bool NanoJPEGCampaign::run() acc->second &= ~common_mask; // new EC with experiments: acc->first -- instr, common_mask -// if (reg != RID_EBP && reg != RID_ESI && reg != RID_EDI) { +// if (reg != RID_CBP && reg != RID_CSI && reg != RID_CDI) { count_exp += add_experiment_ec(acc->first, instr, absolute_instr, reg, common_mask); // } @@ -189,7 +189,7 @@ bool NanoJPEGCampaign::run() // skip empty EC (because register was read within the same instruction)? if (acc->first <= instr) { // new EC with known result: acc->first -- instr, common_mask -// if (reg != RID_EBP && reg != RID_ESI && reg != RID_EDI) { +// if (reg != RID_CBP && reg != RID_CSI && reg != RID_CDI) { count_known += add_known_ec(acc->first, instr, absolute_instr, reg, common_mask); // } } @@ -234,7 +234,7 @@ bool NanoJPEGCampaign::run() // skip empty EC (because register was read within the same instruction)? if (acc->first <= instr) { // new EC with known result: acc->first -- instr, common_mask -// if (reg != RID_EBP && reg != RID_ESI && reg != RID_EDI) { +// if (reg != RID_CBP && reg != RID_CSI && reg != RID_CDI) { count_exp += add_experiment_ec(acc->first, instr, absolute_instr, reg, common_mask); // } } diff --git a/tools/import-trace/AdvancedMemoryImporter.cc b/tools/import-trace/AdvancedMemoryImporter.cc new file mode 100644 index 00000000..6cf96962 --- /dev/null +++ b/tools/import-trace/AdvancedMemoryImporter.cc @@ -0,0 +1,171 @@ +#include +#include "AdvancedMemoryImporter.hpp" + +using namespace llvm; +using namespace llvm::object; +using namespace fail; + +static fail::Logger LOG("AdvancedMemoryImporter"); + +std::string AdvancedMemoryImporter::database_additional_columns() +{ + return MemoryImporter::database_additional_columns() + + "opcode INT UNSIGNED NULL, " + "duration BIGINT UNSIGNED AS (time2 - time1 + 1) PERSISTENT, " + "jumphistory INT UNSIGNED NULL, "; +} + +void AdvancedMemoryImporter::database_insert_columns(std::string& sql, unsigned& num_columns) +{ + // FIXME upcall? + sql = ", opcode, jumphistory"; + num_columns = 2; +} + +//#include + +bool AdvancedMemoryImporter::database_insert_data(Trace_Event &ev, MYSQL_BIND *bind, unsigned num_columns, bool is_fake) +{ + static my_bool null = true; + // FIXME upcall? + assert(num_columns == 2); +#if 0 + // sanity check + if (!is_fake && delayed_entries.size() > 0 && ev.ip() != delayed_entries.front().ev.ip()) { + std::string out; + google::protobuf::TextFormat::PrintToString(ev, &out); + std::cout << "ev: " << out << std::endl; + google::protobuf::TextFormat::PrintToString(delayed_entries.front().ev, &out); + std::cout << "delayed_entries.front.ev: " << out << std::endl; + } +#endif + assert(is_fake || delayed_entries.size() == 0 || ev.ip() == delayed_entries.front().ev.ip()); + bind[0].buffer_type = MYSQL_TYPE_LONG; + bind[0].is_unsigned = 1; + bind[0].buffer = &delayed_entries.front().opcode; + bind[1].buffer_type = MYSQL_TYPE_LONG; + bind[1].is_unsigned = 1; + bind[1].buffer = &m_cur_branchmask; + if (is_fake) { + bind[0].is_null = &null; + bind[1].is_null = &null; + } + return true; +} + +void AdvancedMemoryImporter::insert_delayed_entries(bool finalizing) +{ + unsigned branchmask; + unsigned last_branches_before = UINT_MAX; + // If we don't know enough future, and there's a chance we'll learn more, + // delay further. + for (std::deque::iterator it = delayed_entries.begin(); + it != delayed_entries.end() && + (it->branches_before + BRANCH_WINDOW_SIZE <= branches_taken.size() || + finalizing); + it = delayed_entries.erase(it)) { + // determine branche decisions before / after this mem event + if (it->branches_before != last_branches_before) { + branchmask = 0; + int pos = std::max(-(signed)BRANCH_WINDOW_SIZE, - (signed) it->branches_before); + int maxpos = std::min(BRANCH_WINDOW_SIZE, branches_taken.size() - it->branches_before); + for (; pos < maxpos; ++pos) { + branchmask |= + ((unsigned) branches_taken[it->branches_before + pos]) + << (16 - pos - 1); + } + m_cur_branchmask = branchmask; + } + + //LOG << "AdvancedMemoryImporter::insert_delayed_entries instr = " << it->instr << " data_address = " << it->ev.memaddr() << std::endl; + + // trigger INSERT + // (will call back via database_insert_data() and ask for additional data) + MemoryImporter::handle_mem_event(it->curtime, it->instr, it->ev); + } + + // FIXME branches_taken could be shrunk here to stay within a bounded + // memory footprint +} + +bool AdvancedMemoryImporter::handle_ip_event(fail::simtime_t curtime, instruction_count_t instr, + Trace_Event &ev) +{ + // Previous instruction was a branch, check and remember whether it was taken + if (m_last_was_conditional_branch) { + m_last_was_conditional_branch = false; + branches_taken.push_back(ev.ip() != m_ip_jump_not_taken); + } + + // Check whether we know enough branch-taken future to INSERT a few more + // (delayed) trace entries + insert_delayed_entries(false); + + if (!binary) { + /* Disassemble the binary if necessary */ + llvm::InitializeAllTargetInfos(); + llvm::InitializeAllTargetMCs(); + llvm::InitializeAllDisassemblers(); + + if (error_code ec = createBinary(m_elf->getFilename(), binary)) { + LOG << m_elf->getFilename() << "': " << ec.message() << ".\n"; + return false; + } + +// necessary due to an AspectC++ bug triggered by LLVM 3.3's dyn_cast() +#ifndef __puma + ObjectFile *obj = dyn_cast(binary.get()); + disas.reset(new LLVMDisassembler(obj)); +#endif + disas->disassemble(); + LLVMDisassembler::InstrMap &instr_map = disas->getInstrMap(); + LOG << "instructions disassembled: " << std::dec << instr_map.size() << " Triple: " << disas->GetTriple() << std::endl; +#if 0 + for (LLVMDisassembler::InstrMap::const_iterator it = instr_map.begin(); + it != instr_map.end(); ++it) { + LOG << "DIS " << std::hex << it->second.address << " " << (int) it->second.length << std::endl; + } +#endif + } + + const LLVMDisassembler::InstrMap &instr_map = disas->getInstrMap(); + const LLVMDisassembler::InstrMap::const_iterator it = instr_map.find(ev.ip()); + if (it == instr_map.end()) { + LOG << "WARNING: LLVMDisassembler hasn't disassembled instruction at 0x" + << ev.ip() << " -- are you using LLVM < 3.3?" << std::endl; + return true; // probably weird things will happen now + } + const LLVMDisassembler::Instr &opcode = it->second; + + /* Now we've got the opcode and know whether it's a conditional branch. If + * it is, the next IP event will tell us whether it was taken or not. */ + if (opcode.conditional_branch) { + m_last_was_conditional_branch = true; + m_ip_jump_not_taken = opcode.address + opcode.length; + } + + // IP events may need to be delayed, too, if the parent Importer draws any + // information from them. MemoryImporter does not, though. + //return MemoryImporter::handle_ip_event(curtime, instr, ev); + return true; +} + +bool AdvancedMemoryImporter::handle_mem_event(fail::simtime_t curtime, instruction_count_t instr, + Trace_Event &ev) +{ + const LLVMDisassembler::InstrMap &instr_map = disas->getInstrMap(); + const LLVMDisassembler::Instr &opcode = instr_map.at(ev.ip()); + DelayedTraceEntry entry = { curtime, instr, ev, opcode.opcode, branches_taken.size() }; + delayed_entries.push_back(entry); + + // delay upcall to handle_mem_event until we know enough future branch decisions + return true; +} + +bool AdvancedMemoryImporter::trace_end_reached() +{ + LOG << "inserting remaining trace events ..." << std::endl; + // INSERT the remaining entries (with incomplete branch future) + insert_delayed_entries(true); + return true; +} diff --git a/tools/import-trace/AdvancedMemoryImporter.hpp b/tools/import-trace/AdvancedMemoryImporter.hpp new file mode 100644 index 00000000..6491c8d2 --- /dev/null +++ b/tools/import-trace/AdvancedMemoryImporter.hpp @@ -0,0 +1,60 @@ +#ifndef __ADVANCED_MEMORY_IMPORTER_H__ +#define __ADVANCED_MEMORY_IMPORTER_H__ + +#include +#include +#include "MemoryImporter.hpp" + +#include "util/llvmdisassembler/LLVMDisassembler.hpp" + +/** + * A MemoryImporter that additionally imports Relyzer-style conditional branch + * history, instruction opcodes, and a virtual duration = time2 - time1 + 1 + * column (MariaDB 5.2+ only!) for fault-space pruning purposes. + * + * Initially this was implemented by directly passing through trace events to + * the MemoryImporter, keeping a record of conditional jumps and opcodes, and + * UPDATEing all inserted rows in a second pass when the MemoryImporter is + * finished. + * + * Unfortunately, UPDATE is very slow, and keeping all information in memory + * till the end doesn't scale indefinitely. Therefore the implementation now + * delays passing memory access events upwards to the MemoryImporter only until + * enough branch history is aggregated, and taps into Importer's database + * operations with a set of new virtual functions that are called downwards. + */ +class AdvancedMemoryImporter : public MemoryImporter { + llvm::OwningPtr binary; + llvm::OwningPtr disas; + bool m_last_was_conditional_branch; + fail::guest_address_t m_ip_jump_not_taken; + std::vector branches_taken; + struct DelayedTraceEntry { + fail::simtime_t curtime; + instruction_count_t instr; + Trace_Event ev; + unsigned opcode; + unsigned branches_before; + }; + std::deque delayed_entries; + static const unsigned BRANCH_WINDOW_SIZE = 16; //!< increasing this requires changing the underlying data types + + unsigned m_cur_branchmask; + + void insert_delayed_entries(bool finalizing); + +public: + AdvancedMemoryImporter() : m_last_was_conditional_branch(false) {} + +protected: + virtual std::string database_additional_columns(); + virtual void database_insert_columns(std::string& sql, unsigned& num_columns); + virtual bool database_insert_data(Trace_Event &ev, MYSQL_BIND *bind, unsigned num_columns, bool is_fake); + virtual bool handle_ip_event(fail::simtime_t curtime, instruction_count_t instr, + Trace_Event &ev); + virtual bool handle_mem_event(fail::simtime_t curtime, instruction_count_t instr, + Trace_Event &ev); + virtual bool trace_end_reached(); +}; + +#endif diff --git a/tools/import-trace/CMakeLists.txt b/tools/import-trace/CMakeLists.txt index 561a9c9c..56afe370 100644 --- a/tools/import-trace/CMakeLists.txt +++ b/tools/import-trace/CMakeLists.txt @@ -8,6 +8,7 @@ if (BUILD_LLVM_DISASSEMBLER) InstructionImporter.cc RegisterImporter.cc RandomJumpImporter.cc + AdvancedMemoryImporter.cc ) include(FindLLVM) diff --git a/tools/import-trace/Importer.cc b/tools/import-trace/Importer.cc index 71c900e8..accc562f 100644 --- a/tools/import-trace/Importer.cc +++ b/tools/import-trace/Importer.cc @@ -13,13 +13,16 @@ bool Importer::init(const std::string &variant, const std::string &benchmark, Da if (!m_variant_id) { return false; } + m_extended_trace_regs = + m_arch.getRegisterSetOfType(RT_TRACE); LOG << "Importing to variant " << variant << "/" << benchmark << " (ID: " << m_variant_id << ")" << std::endl; return true; } bool Importer::create_database() { - std::string create_statement = "CREATE TABLE IF NOT EXISTS trace (" + std::stringstream create_statement; + create_statement << "CREATE TABLE IF NOT EXISTS trace (" " variant_id int(11) NOT NULL," " instr1 int(10) unsigned NOT NULL," " instr1_absolute int(10) unsigned DEFAULT NULL," @@ -29,10 +32,20 @@ bool Importer::create_database() { " time2 bigint(10) unsigned NOT NULL," " data_address int(10) unsigned NOT NULL," " width tinyint(3) unsigned NOT NULL," - " accesstype enum('R','W') NOT NULL," + " accesstype enum('R','W') NOT NULL,"; + if (m_extended_trace) { + create_statement << " data_value int(10) unsigned NULL,"; + for (UniformRegisterSet::iterator it = m_extended_trace_regs->begin(); + it != m_extended_trace_regs->end(); ++it) { + create_statement << " r" << (*it)->getId() << " int(10) unsigned NULL,"; + create_statement << " r" << (*it)->getId() << "_deref int(10) unsigned NULL,"; + } + } + create_statement << database_additional_columns(); + create_statement << " PRIMARY KEY (variant_id,data_address,instr2)" ") engine=MyISAM "; - return db->query(create_statement.c_str()); + return db->query(create_statement.str().c_str()); } @@ -82,17 +95,24 @@ bool Importer::copy_to_database(fail::ProtoIStream &ps) { /* Another instruction was executed, handle it in the subclass */ if (!handle_ip_event(curtime, instr, ev)) { + LOG << "error: handle_ip_event() failed at instr=" << instr << std::endl; return false; } instr++; } else { if (!handle_mem_event(curtime, instr, ev)) { + LOG << "error: handle_mem_event() failed at instr=" << instr << std::endl; return false; } } } + if (!trace_end_reached()) { + LOG << "trace_end_reached() failed" << std::endl; + return false; + } + // Why -1? In most cases it does not make sense to inject before the // very last instruction, as we won't execute it anymore. This *only* // makes sense if we also inject into parts of the result vector. This @@ -103,7 +123,7 @@ bool Importer::copy_to_database(fail::ProtoIStream &ps) { m_last_ip = ev.ip(); // The last event in the log /* Signal that the trace was completely imported */ - LOG << "trace duration: " << (curtime - m_time_trace_start) << " ticks" << std::endl; + LOG << "trace duration: " << std::dec << (curtime - m_time_trace_start) << " ticks" << std::endl; LOG << "Inserted " << m_row_count << " real trace events into the database" << std::endl; @@ -219,60 +239,169 @@ bool Importer::copy_to_database(fail::ProtoIStream &ps) { bool Importer::add_trace_event(margin_info_t &begin, margin_info_t &end, access_info_t &access, bool is_fake) { - static MYSQL_STMT *stmt = 0; - if (!stmt) { - std::string sql("INSERT INTO trace (variant_id, instr1, instr1_absolute, instr2, instr2_absolute, time1, time2, data_address, width," - " accesstype)" - "VALUES (?,?,?,?,?,?,?,?,?,?)"); - stmt = mysql_stmt_init(db->getHandle()); - if (mysql_stmt_prepare(stmt, sql.c_str(), sql.length())) { - LOG << "query '" << sql << "' failed: " << mysql_error(db->getHandle()) << std::endl; + Trace_Event e; + e.set_ip(end.ip); + e.set_memaddr(access.data_address); + e.set_width(access.data_width); + e.set_accesstype(access.access_type == 'R' ? e.READ : e.WRITE); + return add_trace_event(begin, end, e, is_fake); +} + +bool Importer::add_trace_event(margin_info_t &begin, margin_info_t &end, + Trace_Event &event, bool is_fake) { + if (!m_import_write_ecs && event.accesstype() == event.WRITE) { + return true; + } + + // insert extended trace info if configured and available + bool extended = m_extended_trace && event.has_trace_ext(); + + MYSQL_STMT **stmt; + unsigned *columns; + static MYSQL_STMT *stmt_basic = 0; + static unsigned columns_basic = 0; + static MYSQL_STMT *stmt_extended = 0; + static unsigned columns_extended = 0; + stmt = extended ? &stmt_extended : &stmt_basic; + columns = extended ? &columns_extended : &columns_basic; + + static unsigned num_additional_columns = 0; + + if (!*stmt) { + std::stringstream sql; + sql << "INSERT INTO trace (variant_id, instr1, instr1_absolute, instr2, instr2_absolute, time1, time2, " + "data_address, width, accesstype"; + *columns = 10; + if (extended) { + sql << ", data_value"; + (*columns)++; + for (UniformRegisterSet::iterator it = m_extended_trace_regs->begin(); + it != m_extended_trace_regs->end(); ++it) { + sql << ", r" << (*it)->getId() << ", r" << (*it)->getId() << "_deref"; + } + } + + // Ask specialized importers whether they want to INSERT additional + // columns. + std::string additional_columns; + database_insert_columns(additional_columns, num_additional_columns); + sql << additional_columns; + + sql << ") VALUES (?"; + for (unsigned i = 1; + i < *columns + + (extended ? m_extended_trace_regs->count() * 2 : 0) + + num_additional_columns; + ++i) { + sql << ",?"; + } + sql << ")"; + + *stmt = mysql_stmt_init(db->getHandle()); + if (mysql_stmt_prepare(*stmt, sql.str().c_str(), sql.str().length())) { + LOG << "query '" << sql.str() << "' failed: " << mysql_error(db->getHandle()) << std::endl; return false; } } - MYSQL_BIND bind[10]; - my_bool is_null = is_fake; - my_bool null = true; + // extended trace: + // retrieve register / register-dereferenced values + std::map register_values; + std::map register_values_deref; + unsigned data_value = 0; + const Trace_Event_Extended& ev_ext = event.trace_ext(); + if (extended) { + data_value = ev_ext.data(); + for (int i = 0; i < ev_ext.registers_size(); i++) { + const Trace_Event_Extended_Registers& reg = ev_ext.registers(i); + register_values[reg.id()] = reg.value(); + register_values_deref[reg.id()] = reg.value_deref(); + } + } + + // C99 / g++ extension VLA to the rescue: + MYSQL_BIND bind[*columns + m_extended_trace_regs->count() * 2 + num_additional_columns]; + my_bool fake_null = is_fake; + my_bool null = true, not_null = false; long unsigned accesstype_len = 1; + unsigned data_address = event.memaddr(); + unsigned width = event.width(); + char accesstype = event.accesstype() == event.READ ? 'R' : 'W'; + // LOG << m_variant_id << "-" << ":" << begin.dyninstr << ":" << end.dyninstr << "-" << data_address << " " << accesstype << std::endl; - + // sizeof works fine for VLAs memset(bind, 0, sizeof(bind)); - for (unsigned i = 0; i < sizeof(bind)/sizeof(*bind); ++i) { + for (unsigned i = 0; i < *columns; ++i) { bind[i].buffer_type = MYSQL_TYPE_LONG; bind[i].is_unsigned = 1; switch (i) { case 0: bind[i].buffer = &m_variant_id; break; case 1: bind[i].buffer = &begin.dyninstr; break; case 2: bind[i].buffer = &begin.ip; - bind[i].is_null = begin.ip == 0 ? &null : &is_null; + bind[i].is_null = begin.ip == 0 ? &null : &fake_null; break; case 3: bind[i].buffer = &end.dyninstr; break; case 4: bind[i].buffer = &end.ip; - bind[i].is_null = &is_null; break; + bind[i].is_null = &fake_null; break; case 5: bind[i].buffer = &begin.time; bind[i].buffer_type = MYSQL_TYPE_LONGLONG; break; case 6: bind[i].buffer = &end.time; bind[i].buffer_type = MYSQL_TYPE_LONGLONG; break; - case 7: bind[i].buffer = &access.data_address; break; - case 8: bind[i].buffer = &access.data_width; break; - case 9: bind[i].buffer = &access.access_type; + case 7: bind[i].buffer = &data_address; break; + case 8: bind[i].buffer = &width; break; + case 9: bind[i].buffer = &accesstype; bind[i].buffer_type = MYSQL_TYPE_STRING; bind[i].buffer_length = accesstype_len; bind[i].length = &accesstype_len; break; + // only visited in extended mode: + case 10: bind[i].buffer = &data_value; + bind[i].is_null = ev_ext.has_data() ? ¬_null : &null; break; } } - if (mysql_stmt_bind_param(stmt, bind)) { - LOG << "mysql_stmt_bind_param() failed: " << mysql_stmt_error(stmt) << std::endl; + if (extended) { + unsigned i = 0; + for (UniformRegisterSet::iterator it = m_extended_trace_regs->begin(); + it != m_extended_trace_regs->end(); ++it, ++i) { + assert(*columns + i*2 + 1 < sizeof(bind)/sizeof(*bind)); + bind[*columns + i*2 ].buffer_type = MYSQL_TYPE_LONG; + bind[*columns + i*2 ].is_unsigned = 1; + if (register_values.count((*it)->getId())) { + bind[*columns + i*2 ].buffer = ®ister_values[(*it)->getId()]; + } else { + bind[*columns + i*2 ].buffer = &width; // arbitrary + bind[*columns + i*2 ].is_null = &null; + } + + bind[*columns + i*2 + 1].buffer_type = MYSQL_TYPE_LONG; + bind[*columns + i*2 + 1].is_unsigned = 1; + if (register_values_deref.count((*it)->getId())) { + bind[*columns + i*2 + 1].buffer = ®ister_values_deref[(*it)->getId()]; + } else { + bind[*columns + i*2 + 1].buffer = &width; // arbitrary + bind[*columns + i*2 + 1].is_null = &null; + } + } + } + + // Ask specialized importers what concrete data they want to INSERT. + if (num_additional_columns) { + unsigned pos = *columns + (extended ? m_extended_trace_regs->count() * 2 : 0); + if (!database_insert_data(event, bind + pos, num_additional_columns, is_fake)) { + return false; + } + } + + if (mysql_stmt_bind_param(*stmt, bind)) { + LOG << "mysql_stmt_bind_param() failed: " << mysql_stmt_error(*stmt) << std::endl; return false; } - if (mysql_stmt_execute(stmt)) { - LOG << "mysql_stmt_execute() failed: " << mysql_stmt_error(stmt) << std::endl; + if (mysql_stmt_execute(*stmt)) { + LOG << "mysql_stmt_execute() failed: " << mysql_stmt_error(*stmt) << std::endl; LOG << "IP: " << std::hex << end.ip << std::endl; return false; } diff --git a/tools/import-trace/Importer.hpp b/tools/import-trace/Importer.hpp index e7492860..f467d607 100644 --- a/tools/import-trace/Importer.hpp +++ b/tools/import-trace/Importer.hpp @@ -6,6 +6,7 @@ #include "util/ProtoStream.hpp" #include "util/ElfReader.hpp" #include "sal/SALConfig.hpp" +#include "sal/Architecture.hpp" #include "util/Database.hpp" #include "util/MemoryMap.hpp" #include "comm/TracePlugin.pb.h" @@ -22,7 +23,11 @@ protected: fail::MemoryMap *m_mm; char m_faultspace_rightmargin; bool m_sanitychecks; + bool m_import_write_ecs; + bool m_extended_trace; fail::Database *db; + fail::Architecture m_arch; + fail::UniformRegisterSet *m_extended_trace_regs; /* How many rows were inserted into the database */ unsigned m_row_count; @@ -60,8 +65,54 @@ protected: instruction_count_t m_last_instr; fail::simtime_t m_last_time; +protected: + /** + * Allows specialized importers to add more table columns instead of + * completely overriding create_database(). The returned SQL CREATE TABLE + * snippet should be terminated with a comma if non-empty. Should call and + * pass through their parent's implementation. + */ + virtual std::string database_additional_columns() { return ""; } + /** + * Similar to database_additional_columns(), this allows specialized + * importers to define which additional columns it wants to INSERT + * alongside what add_trace_event() adds by itself. This may be identical + * to or a subset of what database_additional_columns() specifies. The SQL + * snippet should *begin* with a comma if non-empty. + */ + virtual void database_insert_columns(std::string& sql, unsigned& num_columns) { num_columns = 0; } + /** + * Will be called back from add_trace_event() to fill in data for the + * columns specified by database_insert_columns(). + */ + virtual bool database_insert_data(Trace_Event &ev, MYSQL_BIND *bind, unsigned num_columns, bool is_fake) { return true; } + /** + * Use this variant if passing through the IP/MEM event does not make any + * sense for your Importer implementation. + */ + virtual bool add_trace_event(margin_info_t &begin, margin_info_t &end, + access_info_t &event, bool is_fake = false); + /** + * Use this variant for passing through the IP/MEM event your Importer + * received. + */ + virtual bool add_trace_event(margin_info_t &begin, margin_info_t &end, + Trace_Event &event, bool is_fake = false); + virtual void open_unused_ec_intervals(); + virtual bool close_ec_intervals(); + + virtual bool handle_ip_event(fail::simtime_t curtime, instruction_count_t instr, + Trace_Event &ev) = 0; + virtual bool handle_mem_event(fail::simtime_t curtime, instruction_count_t instr, + Trace_Event &ev) = 0; + /** + * May be overridden by importers that need to do stuff after the last + * event was consumed. + */ + virtual bool trace_end_reached() { return true; } + public: - Importer() : m_sanitychecks(false), m_row_count(0), m_time_trace_start(0) {} + Importer() : m_sanitychecks(false), m_import_write_ecs(true), m_extended_trace(false), m_row_count(0), m_time_trace_start(0) {} bool init(const std::string &variant, const std::string &benchmark, fail::Database *db); /** @@ -73,25 +124,14 @@ public: virtual bool create_database(); virtual bool copy_to_database(fail::ProtoIStream &ps); virtual bool clear_database(); - virtual bool add_trace_event(margin_info_t &begin, margin_info_t &end, - access_info_t &event, bool is_fake = false); - virtual void open_unused_ec_intervals(); - virtual bool close_ec_intervals(); - - virtual bool handle_ip_event(fail::simtime_t curtime, instruction_count_t instr, - const Trace_Event &ev) = 0; - virtual bool handle_mem_event(fail::simtime_t curtime, instruction_count_t instr, - const Trace_Event &ev) = 0; - void set_elf(fail::ElfReader *elf) { m_elf = elf; } void set_memorymap(fail::MemoryMap *mm) { m_mm = mm; } void set_faultspace_rightmargin(char accesstype) { m_faultspace_rightmargin = accesstype; } void set_sanitychecks(bool enabled) { m_sanitychecks = enabled; } - - - + void set_import_write_ecs(bool enabled) { m_import_write_ecs = enabled; } + void set_extended_trace(bool enabled) { m_extended_trace = enabled; } }; #endif diff --git a/tools/import-trace/InstructionImporter.cc b/tools/import-trace/InstructionImporter.cc index a1e72f79..a5322898 100644 --- a/tools/import-trace/InstructionImporter.cc +++ b/tools/import-trace/InstructionImporter.cc @@ -11,7 +11,7 @@ using namespace fail; static Logger LOG("InstructionImporter"); bool InstructionImporter::handle_ip_event(fail::simtime_t curtime, instruction_count_t instr, - const Trace_Event &ev) { + Trace_Event &ev) { if (!binary) { /* Disassemble the binary if necessary */ llvm::InitializeAllTargetInfos(); @@ -60,11 +60,12 @@ bool InstructionImporter::handle_ip_event(fail::simtime_t curtime, instruction_c // we're currently looking at; the EC is defined by // data_address, dynamic instruction start/end, the absolute PC at // the end, and time start/end - access_info_t access; - access.access_type = 'R'; // instruction fetch is always a read - access.data_address = data_address; - access.data_width = 1; // exactly one byte - if (!add_trace_event(left_margin, right_margin, access)) { + + // pass through potentially available extended trace information + ev.set_accesstype(ev.READ); // instruction fetch is always a read + ev.set_memaddr(data_address); + ev.set_width(1); // exactly one byte + if (!add_trace_event(left_margin, right_margin, ev)) { LOG << "add_trace_event failed" << std::endl; return false; } diff --git a/tools/import-trace/InstructionImporter.hpp b/tools/import-trace/InstructionImporter.hpp index 81b58436..53fc9af7 100644 --- a/tools/import-trace/InstructionImporter.hpp +++ b/tools/import-trace/InstructionImporter.hpp @@ -9,11 +9,11 @@ class InstructionImporter : public Importer { llvm::OwningPtr binary; llvm::OwningPtr disas; -public: +protected: virtual bool handle_ip_event(fail::simtime_t curtime, instruction_count_t instr, - const Trace_Event &ev); + Trace_Event &ev); virtual bool handle_mem_event(fail::simtime_t curtime, instruction_count_t instr, - const Trace_Event &ev) { + Trace_Event &ev) { /* ignore on purpose */ return true; } diff --git a/tools/import-trace/MemoryImporter.cc b/tools/import-trace/MemoryImporter.cc index 0c7f59b4..64571529 100644 --- a/tools/import-trace/MemoryImporter.cc +++ b/tools/import-trace/MemoryImporter.cc @@ -5,12 +5,12 @@ using namespace fail; static fail::Logger LOG("MemoryImporter"); -bool MemoryImporter::handle_ip_event(simtime_t curtime, instruction_count_t instr, const Trace_Event &ev) { +bool MemoryImporter::handle_ip_event(simtime_t curtime, instruction_count_t instr, Trace_Event &ev) { return true; } bool MemoryImporter::handle_mem_event(simtime_t curtime, instruction_count_t instr, - const Trace_Event &ev) { + Trace_Event &ev) { address_t from = ev.memaddr(), to = ev.memaddr() + ev.width(); // Iterate over all accessed bytes // FIXME Keep complete trace information (access width)? @@ -39,11 +39,11 @@ bool MemoryImporter::handle_mem_event(simtime_t curtime, instruction_count_t ins // we're currently looking at; the EC is defined by // data_address, dynamic instruction start/end, the absolute PC at // the end, and time start/end - access_info_t access; - access.access_type = ev.accesstype() == ev.READ ? 'R' : 'W'; - access.data_address = data_address; - access.data_width = 1; // exactly one byte - if (!add_trace_event(left_margin, right_margin, access)) { + + // pass through potentially available extended trace information + ev.set_memaddr(data_address); + ev.set_width(1); // exactly one byte + if (!add_trace_event(left_margin, right_margin, ev)) { LOG << "add_trace_event failed" << std::endl; return false; } @@ -54,4 +54,3 @@ bool MemoryImporter::handle_mem_event(simtime_t curtime, instruction_count_t ins } return true; } - diff --git a/tools/import-trace/MemoryImporter.hpp b/tools/import-trace/MemoryImporter.hpp index 8edbe475..33b24436 100644 --- a/tools/import-trace/MemoryImporter.hpp +++ b/tools/import-trace/MemoryImporter.hpp @@ -6,12 +6,11 @@ class MemoryImporter : public Importer { -public: - +protected: virtual bool handle_ip_event(fail::simtime_t curtime, instruction_count_t instr, - const Trace_Event &ev); + Trace_Event &ev); virtual bool handle_mem_event(fail::simtime_t curtime, instruction_count_t instr, - const Trace_Event &ev); + Trace_Event &ev); }; #endif diff --git a/tools/import-trace/RandomJumpImporter.cc b/tools/import-trace/RandomJumpImporter.cc index 857c0b28..0a7656ef 100644 --- a/tools/import-trace/RandomJumpImporter.cc +++ b/tools/import-trace/RandomJumpImporter.cc @@ -18,14 +18,14 @@ bool RandomJumpImporter::cb_commandline_init() { CommandLine &cmd = CommandLine::Inst(); FROM = cmd.addOption("", "jump-from", Arg::Required, - "--jump-from\t RandomJump: Which addresses should be jumped from\n"); + "--jump-from \tRandomJump: Which addresses should be jumped from (a memory map; may be used more than once)"); TO = cmd.addOption("", "jump-to", Arg::Required, - "--jump-to\t RandomJump: Where to jump (a memory map>\n"); + "--jump-to \tRandomJump: Where to jump (a memory map; may be used more than once)"); return true; } bool RandomJumpImporter::handle_ip_event(fail::simtime_t curtime, instruction_count_t instr, - const Trace_Event &ev) { + Trace_Event &ev) { if (!binary) { // Parse command line again, for jump-from and jump-to // operations @@ -35,8 +35,8 @@ bool RandomJumpImporter::handle_ip_event(fail::simtime_t curtime, instruction_co return false; } - // Read FROM memory file - if (cmd[FROM].count() > 0) { + // Read FROM memory map(s) + if (cmd[FROM]) { m_mm_from = new MemoryMap(); for (option::Option *o = cmd[FROM]; o; o = o->next()) { if (!m_mm_from->readFromFile(o->arg)) { @@ -46,7 +46,8 @@ bool RandomJumpImporter::handle_ip_event(fail::simtime_t curtime, instruction_co } } - if (cmd[TO].count() > 0) { + // Read TO memory map(s) + if (cmd[TO]) { m_mm_to = new MemoryMap(); for (option::Option *o = cmd[TO]; o; o = o->next()) { if (!m_mm_to->readFromFile(o->arg)) { @@ -81,7 +82,7 @@ bool RandomJumpImporter::handle_ip_event(fail::simtime_t curtime, instruction_co /* Collect all addresses we want to jump to */ for (LLVMDisassembler::InstrMap::const_iterator instr = instr_map.begin(); instr != instr_map.end(); ++instr) { - if (m_mm_to->isMatching(instr->first)) { + if (m_mm_to && m_mm_to->isMatching(instr->first)) { m_jump_to_addresses.push_back(instr->first); } } @@ -91,12 +92,12 @@ bool RandomJumpImporter::handle_ip_event(fail::simtime_t curtime, instruction_co // skip events that are outside the memory map. -m instruction map if (m_mm && !m_mm->isMatching(ev.ip())) { - return true; + return true; } // skip events that are outside the --jump-from memory map. - if (!m_mm_from->isMatching(ev.ip())) { - return true; + if (m_mm_from && !m_mm_from->isMatching(ev.ip())) { + return true; } @@ -116,11 +117,12 @@ bool RandomJumpImporter::handle_ip_event(fail::simtime_t curtime, instruction_co // we're currently looking at; the EC is defined by // data_address, dynamic instruction start/end, the absolute PC at // the end, and time start/end - access_info_t access; - access.access_type = 'R'; // instruction fetch is always a read - access.data_address = to_addr; - access.data_width = 4; // exactly one byte - if (!add_trace_event(margin, margin, access)) { + + // pass through potentially available extended trace information + ev.set_accesstype(ev.READ); // instruction fetch is always a read + ev.set_memaddr(to_addr); + ev.set_width(4); // FIXME arbitrary, use Instr.length instead? + if (!add_trace_event(margin, margin, ev)) { LOG << "add_trace_event failed" << std::endl; return false; } diff --git a/tools/import-trace/RandomJumpImporter.hpp b/tools/import-trace/RandomJumpImporter.hpp index e9e38575..b18105a0 100644 --- a/tools/import-trace/RandomJumpImporter.hpp +++ b/tools/import-trace/RandomJumpImporter.hpp @@ -16,16 +16,18 @@ class RandomJumpImporter : public Importer { fail::MemoryMap *m_mm_from, *m_mm_to; std::vector m_jump_to_addresses; public: + RandomJumpImporter() : m_mm_from(0), m_mm_to(0) {} /** * Callback function that can be used to add command line options * to the campaign */ virtual bool cb_commandline_init(); +protected: virtual bool handle_ip_event(fail::simtime_t curtime, instruction_count_t instr, - const Trace_Event &ev); + Trace_Event &ev); virtual bool handle_mem_event(fail::simtime_t curtime, instruction_count_t instr, - const Trace_Event &ev) { + Trace_Event &ev) { /* ignore on purpose */ return true; } diff --git a/tools/import-trace/RegisterImporter.cc b/tools/import-trace/RegisterImporter.cc index 3df4dc58..381b3d42 100644 --- a/tools/import-trace/RegisterImporter.cc +++ b/tools/import-trace/RegisterImporter.cc @@ -17,22 +17,21 @@ static Logger LOG("RegisterImporter"); bool RegisterImporter::cb_commandline_init() { CommandLine &cmd = CommandLine::Inst(); - NO_GP = cmd.addOption("", "no-gp", Arg::None, - "--no-gp\t RegisterImporter: do not inject general purpose registers\n"); + NO_GP = cmd.addOption("", "no-gp", Arg::None, + "--no-gp \tRegisterImporter: do not inject general purpose registers"); FLAGS = cmd.addOption("", "flags", Arg::None, - "--flags: RegisterImporter: trace flags register\n"); - IP = cmd.addOption("", "ip", Arg::None, - "--ip: RegisterImporter: trace instruction pointer\n"); - NO_SPLIT = cmd.addOption("", "do-not-split", Arg::None, - "--do-not-split: RegisterImporter: Do not split the registers into one byte chunks\n"); - + "--flags \tRegisterImporter: inject flags register"); + IP = cmd.addOption("", "ip", Arg::None, + "--ip \tRegisterImporter: inject instruction pointer"); + NO_SPLIT = cmd.addOption("", "do-not-split", Arg::None, + "--do-not-split \tRegisterImporter: Do not split the registers into one byte chunks"); return true; } bool RegisterImporter::addRegisterTrace(simtime_t curtime, instruction_count_t instr, - const Trace_Event &ev, + Trace_Event &ev, const LLVMtoFailTranslator::reginfo_t &info, char access_type) { address_t from, to; @@ -75,11 +74,20 @@ bool RegisterImporter::addRegisterTrace(simtime_t curtime, instruction_count_t i // we're currently looking at; the EC is defined by // data_address, dynamic instruction start/end, the absolute PC at // the end, and time start/end +<<<<<<< HEAD access_info_t access; access.access_type = access_type; // instruction fetch is always a read access.data_address = data_address; access.data_width = chunk_width; if (!add_trace_event(left_margin, right_margin, access)) { +======= + + // pass through potentially available extended trace information + ev.set_width(1); // exactly one byte + ev.set_memaddr(data_address); + ev.set_accesstype(access_type == 'R' ? ev.READ : ev.WRITE); + if (!add_trace_event(left_margin, right_margin, ev)) { +>>>>>>> dcd2c021a5ac91d38187d397914e5f51e2fc8819 LOG << "add_trace_event failed" << std::endl; return false; } @@ -93,7 +101,7 @@ bool RegisterImporter::addRegisterTrace(simtime_t curtime, instruction_count_t i bool RegisterImporter::handle_ip_event(fail::simtime_t curtime, instruction_count_t instr, - const Trace_Event &ev) { + Trace_Event &ev) { if (!binary) { // Parse command line again, for jump-from and jump-to // operations @@ -102,6 +110,7 @@ bool RegisterImporter::handle_ip_event(fail::simtime_t curtime, instruction_coun std::cerr << "Error parsing arguments." << std::endl; return false; } +<<<<<<< HEAD // Read FROM memory file if (cmd[NO_GP].count() > 0) { @@ -117,6 +126,11 @@ bool RegisterImporter::handle_ip_event(fail::simtime_t curtime, instruction_coun do_split_registers = false; } +======= + do_gp = !cmd[NO_GP]; + do_flags = cmd[FLAGS]; + do_ip = cmd[IP]; +>>>>>>> dcd2c021a5ac91d38187d397914e5f51e2fc8819 /* Disassemble the binary if necessary */ llvm::InitializeAllTargetInfos(); @@ -146,7 +160,7 @@ bool RegisterImporter::handle_ip_event(fail::simtime_t curtime, instruction_coun const LLVMDisassembler::Instr &opcode = instr_map.at(ev.ip()); //const MCRegisterInfo ®_info = disas->getRegisterInfo(); - fail::LLVMtoFailTranslator & ltof = disas->getTranslator() ; + fail::LLVMtoFailTranslator & ltof = disas->getTranslator() ; for (std::vector::const_iterator it = opcode.reg_uses.begin(); it != opcode.reg_uses.end(); ++it) { diff --git a/tools/import-trace/RegisterImporter.hpp b/tools/import-trace/RegisterImporter.hpp index c3425d6b..a6cefc1d 100644 --- a/tools/import-trace/RegisterImporter.hpp +++ b/tools/import-trace/RegisterImporter.hpp @@ -13,7 +13,7 @@ class RegisterImporter : public Importer { llvm::OwningPtr disas; bool addRegisterTrace(fail::simtime_t curtime, instruction_count_t instr, - const Trace_Event &ev, + Trace_Event &ev, const fail::LLVMtoFailTranslator::reginfo_t &info, char access_type); @@ -29,10 +29,11 @@ public: */ virtual bool cb_commandline_init(); +protected: virtual bool handle_ip_event(fail::simtime_t curtime, instruction_count_t instr, - const Trace_Event &ev); + Trace_Event &ev); virtual bool handle_mem_event(fail::simtime_t curtime, instruction_count_t instr, - const Trace_Event &ev) { + Trace_Event &ev) { /* ignore on purpose */ return true; } diff --git a/tools/import-trace/main.cc b/tools/import-trace/main.cc index a3946915..299a81ae 100644 --- a/tools/import-trace/main.cc +++ b/tools/import-trace/main.cc @@ -12,6 +12,7 @@ #include "InstructionImporter.hpp" #include "RegisterImporter.hpp" #include "RandomJumpImporter.hpp" +#include "AdvancedMemoryImporter.hpp" #endif @@ -49,8 +50,7 @@ ProtoIStream openProtoStream(std::string input_file) { } int main(int argc, char *argv[]) { - std::string trace_file, username, hostname, database, benchmark; - std::string variant; + std::string trace_file, variant, benchmark; ElfReader *elf_file = 0; MemoryMap *memorymap = 0; @@ -87,7 +87,13 @@ int main(int argc, char *argv[]) { CommandLine::option_handle NO_DELETE = cmd.addOption("", "no-delete", Arg::None, "--no-delete \tAssume there are no DB entries for this variant/benchmark, don't issue a DELETE"); - + CommandLine::option_handle NO_WRITE_ECS = + cmd.addOption("", "no-write-ecs", Arg::None, + "--no-write-ecs \tDo not import any write ECs into the database; " + "results in a perforated fault space and is OK if you only use absolute failure numbers"); + CommandLine::option_handle EXTENDED_TRACE = + cmd.addOption("", "extended-trace", Arg::None, + "--extended-trace \tImport extended trace information if available"); // variant 1: care (synthetic Rs) // variant 2: don't care (synthetic Ws) @@ -106,7 +112,7 @@ int main(int argc, char *argv[]) { CommandLine::option_handle ENABLE_SANITYCHECKS = cmd.addOption("", "enable-sanitychecks", Arg::None, "--enable-sanitychecks \tEnable sanity checks " - "(in case something looks fishy)" + "(in case something looks fishy) " "(default: disabled)"); if (!cmd.parse()) { @@ -116,28 +122,30 @@ int main(int argc, char *argv[]) { Importer *importer; - if (cmd[IMPORTER].count() > 0) { + if (cmd[IMPORTER]) { std::string imp(cmd[IMPORTER].first()->arg); if (imp == "BasicImporter" || imp == "MemoryImporter" || imp == "memory" || imp == "mem") { - LOG << "Using MemoryImporter" << endl; + imp = "MemoryImporter"; importer = new MemoryImporter(); #ifdef BUILD_LLVM_DISASSEMBLER } else if (imp == "InstructionImporter" || imp == "code") { - LOG << "Using InstructionImporter" << endl; + imp = "InstructionImporter"; importer = new InstructionImporter(); } else if (imp == "RegisterImporter" || imp == "regs") { - LOG << "Using RegisterImporter" << endl; + imp = "RegisterImporter"; importer = new RegisterImporter(); } else if (imp == "RandomJumpImporter") { - LOG << "Using RandomJumpImporter" << endl; importer = new RandomJumpImporter(); + } else if (imp == "AdvancedMemoryImporter") { + importer = new AdvancedMemoryImporter(); #endif } else { LOG << "Unkown import method: " << imp << endl; exit(-1); } + LOG << "Using " << imp << endl; } else { LOG << "Using MemoryImporter" << endl; @@ -158,30 +166,33 @@ int main(int argc, char *argv[]) { exit(0); } - if (cmd[TRACE_FILE].count() > 0) + if (cmd[TRACE_FILE]) { trace_file = std::string(cmd[TRACE_FILE].first()->arg); - else + } else { trace_file = "trace.pb"; + } ProtoIStream ps = openProtoStream(trace_file); Database *db = Database::cmdline_connect(); - if (cmd[VARIANT].count() > 0) + if (cmd[VARIANT]) { variant = std::string(cmd[VARIANT].first()->arg); - else + } else { variant = "none"; + } - if (cmd[BENCHMARK].count() > 0) + if (cmd[BENCHMARK]) { benchmark = std::string(cmd[BENCHMARK].first()->arg); - else + } else { benchmark = "none"; + } - if (cmd[ELF_FILE].count() > 0) { + if (cmd[ELF_FILE]) { elf_file = new ElfReader(cmd[ELF_FILE].first()->arg); } importer->set_elf(elf_file); - if (cmd[MEMORYMAP].count() > 0) { + if (cmd[MEMORYMAP]) { memorymap = new MemoryMap(); for (option::Option *o = cmd[MEMORYMAP]; o; o = o->next()) { if (!memorymap->readFromFile(o->arg)) { @@ -191,7 +202,7 @@ int main(int argc, char *argv[]) { } importer->set_memorymap(memorymap); - if (cmd[FAULTSPACE_RIGHTMARGIN].count() > 0) { + if (cmd[FAULTSPACE_RIGHTMARGIN]) { std::string rightmargin(cmd[FAULTSPACE_RIGHTMARGIN].first()->arg); if (rightmargin == "W") { importer->set_faultspace_rightmargin('W'); @@ -205,9 +216,9 @@ int main(int argc, char *argv[]) { importer->set_faultspace_rightmargin('W'); } - if (cmd[ENABLE_SANITYCHECKS].count() > 0) { - importer->set_sanitychecks(true); - } + importer->set_sanitychecks(cmd[ENABLE_SANITYCHECKS]); + importer->set_extended_trace(cmd[EXTENDED_TRACE]); + importer->set_import_write_ecs(!cmd[NO_WRITE_ECS]); if (!importer->init(variant, benchmark, db)) { LOG << "importer->init() failed" << endl; @@ -223,7 +234,7 @@ int main(int argc, char *argv[]) { exit(-1); } - if (cmd[NO_DELETE].count() == 0 && !importer->clear_database()) { + if (!cmd[NO_DELETE] && !importer->clear_database()) { LOG << "clear_database() failed" << endl; exit(-1); } diff --git a/tools/prune-trace/Pruner.cc b/tools/prune-trace/Pruner.cc index 759a4bbe..a3d339c5 100644 --- a/tools/prune-trace/Pruner.cc +++ b/tools/prune-trace/Pruner.cc @@ -71,7 +71,7 @@ bool Pruner::create_database() { " data_address int(10) unsigned NOT NULL," " fspmethod_id int(11) NOT NULL," " pilot_id int(11) NOT NULL," - " PRIMARY KEY (variant_id, data_address, instr2, fspmethod_id, pilot_id)," + " PRIMARY KEY (variant_id, data_address, instr2, fspmethod_id)," " KEY joinresults (pilot_id,fspmethod_id)) engine=MyISAM"; return db->query(create_statement.c_str());