From 13175c259bac9afed6f208fee0f24b25b63d742b Mon Sep 17 00:00:00 2001 From: Richard Hellwig Date: Fri, 24 Jan 2014 16:44:56 +0100 Subject: [PATCH] import-trace: import debug info If the --debug option is set, the line number table of the elf binary will be imported into the database. The information will be stored in the "dbg_mapping" table. If the --sources option is set, the source files will be imported into the database. Only the files that were actually used in the elf binary will be imported. Change-Id: I0e9de6b456bc42b329c1700c25e5839d9552cdbb --- doc/how-to-build.txt | 2 + src/core/util/CMakeLists.txt | 4 +- src/core/util/DwarfReader.cc | 258 +++++++++++++++++++++++++++++ src/core/util/DwarfReader.hpp | 40 +++++ tools/import-trace/CMakeLists.txt | 3 +- tools/import-trace/ElfImporter.cc | 227 ++++++++++++++++++++++++- tools/import-trace/ElfImporter.hpp | 23 +++ 7 files changed, 551 insertions(+), 6 deletions(-) create mode 100644 src/core/util/DwarfReader.cc create mode 100644 src/core/util/DwarfReader.hpp diff --git a/doc/how-to-build.txt b/doc/how-to-build.txt index ad8f8142..381cedc2 100644 --- a/doc/how-to-build.txt +++ b/doc/how-to-build.txt @@ -8,6 +8,8 @@ Required for Fail*: - libprotobuf-dev - libpcl1-dev - libboost-thread-dev + - libdwarf + - libelf - protobuf-compiler - cmake 2.8.2 (2.8.11 preferred) - cmake-curses-gui diff --git a/src/core/util/CMakeLists.txt b/src/core/util/CMakeLists.txt index 1b506a14..d3f4d81b 100644 --- a/src/core/util/CMakeLists.txt +++ b/src/core/util/CMakeLists.txt @@ -11,6 +11,8 @@ set(SRCS Demangler.cc Disassembler.hpp Disassembler.cc + DwarfReader.cc + DwarfReader.hpp elfinfo/elfinfo.cc elfinfo/elfinfo.h gzstream/gzstream.C @@ -65,7 +67,7 @@ mark_as_advanced(FAIL_OBJDUMP) add_library(fail-util ${SRCS}) add_dependencies(fail-util fail-comm) -target_link_libraries(fail-util ${PROTOBUF_LIBRARY} ${Boost_LIBRARIES} ${LIB_IBERTY} ${ZLIB_LIBRARIES}) +target_link_libraries(fail-util ${PROTOBUF_LIBRARY} ${Boost_LIBRARIES} ${LIB_IBERTY} ${ZLIB_LIBRARIES} dwarf elf) option(BUILD_LLVM_DISASSEMBLER "Build the LLVM-based disassembler (LLVM 3.3 preferred, for 3.1 and 3.2 read doc/how-to-build.txt)" OFF) if (BUILD_LLVM_DISASSEMBLER) diff --git a/src/core/util/DwarfReader.cc b/src/core/util/DwarfReader.cc new file mode 100644 index 00000000..aa7f612f --- /dev/null +++ b/src/core/util/DwarfReader.cc @@ -0,0 +1,258 @@ +#include +#include +#include + +#include "DwarfReader.hpp" +#include "libdwarf.h" +#include "libelf.h" +#include "Logger.hpp" + +/** + * This source code is based on bcov 0.2. + * Sourcefile: src/coverage.cpp + * http://bcov.sourceforge.net/ + * GNU GENERAL PUBLIC LICENSE +*/ + +using namespace fail; +using namespace std; + +static Logger LOG("DwarfReader"); + +static void dwarfErrorHandler(Dwarf_Error error, Dwarf_Ptr /*userData*/) +{ + char* msg=dwarf_errmsg(error); + cerr << "dwarf error: " << msg << endl; +} + +static string normalize(const string& filePath) +{ + // A quick scan first... + bool hadSep=false,needsFix=false; + string::size_type len=filePath.length(); + if (!needsFix) + for (string::size_type index=0;index=len) { + if (hadSep) + result.resize(result.length()-1); + } else { + result+=c; + } + continue; + } + char n=filePath[index+1]; + if (n=='/') { + index++; continue; + } + if (n=='.') { + if (index+2>=len) { + index++; + string::size_type split=result.rfind('/',result.length()-2); + if (split!=string::npos) { + if (result.substr(split)!="/../") { + result.resize(split); + } + } else if (result.length()>0) { + if ((result!="../")&&(result!="/")) { + result.clear(); + } + } else result=".."; + continue; + } else { + n=filePath[index+2]; + if (n=='/') { + index+=2; + string::size_type split=result.rfind('/',result.length()-2); + if (split!=string::npos) { + if (result.substr(split)!="/../") { + result.resize(split+1); + } + } else if (result.length()>0) { + if ((result!="../")&&(result!="/")) { + result.clear(); + } + } else result="../"; + continue; + } + } + } + } + result+=c; hadSep=false; + } + return result; +} + +bool DwarfReader::read_source_files(const std::string& fileName,std::list& lines) { + + // Open The file + int fd=open(fileName.c_str(),O_RDONLY); + if (fd<0) return false; + + // Initialize libdwarf + Dwarf_Debug dbg; + int status = dwarf_init(fd, DW_DLC_READ,dwarfErrorHandler,0,&dbg,0); + if (status==DW_DLV_ERROR) { close(fd); return false; } + if (status==DW_DLV_NO_ENTRY) { close(fd); return true; } + + // Iterator over the headers + Dwarf_Unsigned header; + while (dwarf_next_cu_header(dbg,0,0,0,0,&header,0)==DW_DLV_OK) { + // Access the die + Dwarf_Die die; + if (dwarf_siblingof(dbg,0,&die,0)!=DW_DLV_OK) + return false; + + // Get the source lines + Dwarf_Line* lineBuffer; + Dwarf_Signed lineCount; + if (dwarf_srclines(die,&lineBuffer,&lineCount,0)!=DW_DLV_OK) + continue; //return false; + + // Store them + for (int index=0;index(addr) << " line source:" << normalize(lineSource) << endl; + lines.push_back(normalize(lineSource)); + } + + dwarf_dealloc(dbg,lineSource,DW_DLA_STRING); + } + + // Release the memory + for (int index=0;index& addrToLineList) { + + // Open The file + int fd=open(fileName.c_str(),O_RDONLY); + if (fd<0) { + return false; + } + + // Initialize libdwarf + Dwarf_Debug dbg; + int status = dwarf_init(fd, DW_DLC_READ,dwarfErrorHandler,0,&dbg,0); + if (status==DW_DLV_ERROR) { + close(fd); + return false; + } + if (status==DW_DLV_NO_ENTRY) { + close(fd); + return true; + } + + // Iterator over the headers + Dwarf_Unsigned header; + while (dwarf_next_cu_header(dbg,0,0,0,0,&header,0)==DW_DLV_OK) { + // Access the die + Dwarf_Die die; + if (dwarf_siblingof(dbg,0,&die,0)!=DW_DLV_OK) { + return false; + } + + // Get the source lines + Dwarf_Line* lineBuffer; + Dwarf_Signed lineCount; + if (dwarf_srclines(die,&lineBuffer,&lineCount,0)!=DW_DLV_OK) { + continue; //return false; + } + + // Store them + for (int index=0;index +#include +#include + +namespace fail { + + +/** + * This source code is based on bcov 0.2. + * Sourcefile: src/coverage.cpp + * http://bcov.sourceforge.net/ + * GNU GENERAL PUBLIC LICENSE +*/ + + struct addrToLine { + int absoluteAddr; + int lineNumber; + std::string lineSource; + }; + + /** + * \class DwarfReader + * ToDO + */ + + class DwarfReader { + + public: + + bool read_source_files(const std::string& fileName, std::list& lines); + bool read_mapping(std::string fileName, std::list& addrToLineList); +}; + +} // end-of-namespace fail + +#endif //__DwarfREADER_HPP__ + diff --git a/tools/import-trace/CMakeLists.txt b/tools/import-trace/CMakeLists.txt index d0e5f659..583e397d 100644 --- a/tools/import-trace/CMakeLists.txt +++ b/tools/import-trace/CMakeLists.txt @@ -33,7 +33,8 @@ target_link_libraries(import-trace ${MYSQL_LIBRARIES} fail-util fail-comm - fail-sal) + fail-sal +) if (BUILD_LLVM_DISASSEMBLER) target_link_libraries(import-trace fail-llvmdisassembler fail-sal ${LLVM_LIBS} ${LLVM_LDFLAGS} ${Boost_LIBRARIES}) diff --git a/tools/import-trace/ElfImporter.cc b/tools/import-trace/ElfImporter.cc index 6656fa7a..ead360ac 100644 --- a/tools/import-trace/ElfImporter.cc +++ b/tools/import-trace/ElfImporter.cc @@ -1,5 +1,6 @@ #include #include +#include #include "util/Logger.hpp" #include "ElfImporter.hpp" #include "util/pstream.h" @@ -25,11 +26,18 @@ bool ElfImporter::cb_commandline_init() { OBJDUMP = cmd.addOption("", "objdump", Arg::Required, "--objdump \tObjdump: location of objdump binary, otherwise LLVM Disassembler is used"); + SOURCECODE = cmd.addOption("", "sources", Arg::None, + "--sources \timport all source files into the database"); + DEBUGINFO = cmd.addOption("", "debug", Arg::None, + "--debug \timport some debug informations into the database"); return true; } bool ElfImporter::create_database() { + CommandLine &cmd = CommandLine::Inst(); std::stringstream create_statement; + + create_statement.str(""); create_statement << "CREATE TABLE IF NOT EXISTS objdump (" " variant_id int(11) NOT NULL," " instr_address int(11) NOT NULL," @@ -38,7 +46,50 @@ bool ElfImporter::create_database() { " comment VARCHAR(128)," " PRIMARY KEY (variant_id,instr_address)" ") engine=MyISAM "; - return db->query(create_statement.str().c_str()); + if (!db->query(create_statement.str().c_str())) { + return false; + } + + if (cmd[SOURCECODE]) { + create_statement.str(""); + create_statement << "CREATE TABLE IF NOT EXISTS dbg_filename (" + " file_id int(11) NOT NULL AUTO_INCREMENT," + " variant_id int(11) NOT NULL," + " path VARCHAR(1024)," + " PRIMARY KEY (file_id)" + ") engine=MyISAM "; + if (!db->query(create_statement.str().c_str())) { + return false; + } + + create_statement.str(""); + create_statement << "CREATE TABLE IF NOT EXISTS dbg_source (" + " variant_id int(11) NOT NULL," + " linenumber int(11) NOT NULL," + " file_id int(11) NOT NULL," + " line VARCHAR(1024)," + " PRIMARY KEY (variant_id, linenumber, file_id)" + ") engine=MyISAM "; + if (!db->query(create_statement.str().c_str())) { + return false; + } + } + + if (cmd[DEBUGINFO]) { + create_statement.str(""); + create_statement << "CREATE TABLE IF NOT EXISTS dbg_mapping (" + " variant_id int(11) NOT NULL," + " instr_absolute int(11) NOT NULL," + " linenumber int(11) NOT NULL," + " file_id int(11) NOT NULL," + " PRIMARY KEY (variant_id, instr_absolute ,linenumber)" + ") engine=MyISAM "; + if (!db->query(create_statement.str().c_str())) { + return false; + } + } + + return true; } bool ElfImporter::copy_to_database(ProtoIStream &ps) { @@ -54,12 +105,33 @@ bool ElfImporter::copy_to_database(ProtoIStream &ps) { } if (cmd[OBJDUMP]) { // using objdump - return import_with_objdump(std::string(cmd[OBJDUMP].first()->arg)); + if(!import_with_objdump(std::string(cmd[OBJDUMP].first()->arg))) { + return false; + } } else { LOG << "importing an objdump with internal llvm dissassembler is not yet implemented" << std::endl; - return false; } + if (cmd[SOURCECODE]) { // import sources + std::list sourceFiles; + + if (!import_source_files(m_elf->getFilename(),sourceFiles)) { + cerr << "unable to read dwarf2 debug info" << endl; + return false; + } + + for (std::list::iterator it = sourceFiles.begin(); it != sourceFiles.end(); ++it) { + if(!import_source_code(*it)) { + return false; + } + } + } + + if (cmd[DEBUGINFO]) { // import debug informations + if(!import_mapping(m_elf->getFilename())) { + return false; + } + } return true; } @@ -196,14 +268,161 @@ bool ElfImporter::import_instruction(fail::address_t addr, char *opcode, int opc return true; } +bool ElfImporter::import_source_code(std::string fileName) { + LOG << "Importing Sourcefile: " << fileName << endl; + + ifstream in(fileName.c_str()); + if (!in.is_open()) { + LOG << "Sourcefile not found: " << fileName << endl; + } else { + unsigned lineNo=0; + while (!in.eof()) { + ++lineNo; + + // Read and strip the current line + string currentLine; + getline(in,currentLine); + if ((!currentLine.length())&&(in.eof())) break; + while (true) { + unsigned l=currentLine.length(); + if (!l) break; + char c=currentLine[l-1]; + if ((c!='\n')&&(c!='\r')&&(c!=' ')&&(c!='\t')) break; + currentLine=currentLine.substr(0,l-1); + } + + //Sonderzeichen + currentLine = db->escape_string(currentLine); + + std::stringstream ss; + ss << "SELECT file_id FROM dbg_filename WHERE path = " << "\"" << fileName.c_str() << "\""; + + MYSQL_RES *res = db->query(ss.str().c_str(), true); + MYSQL_ROW row; + row = mysql_fetch_row(res); + + // INSERT group entry + std::stringstream sql; + + sql << "(" << m_variant_id << "," << lineNo << ","; + if (row != NULL) { + sql << row[0]; + } else { + sql << -1; + } + sql << "," << "\"" << currentLine << "\"" << ")"; + + if (!db->insert_multiple("INSERT INTO dbg_source (variant_id, linenumber, file_id, line) VALUES ", sql.str().c_str())){ + LOG << "Can not import sourcelines!" << endl;; + } + } + db->insert_multiple(); + } + return true; +} + +bool ElfImporter::import_source_files(const std::string& fileName,std::list& lines) { + + //std::list lines; + if (!dwReader.read_source_files(fileName, lines)) { + return false; + } + for (std::list::iterator it = lines.begin(); it != lines.end(); ++it) { + // INSERT group entry + std::stringstream sql; + sql << "(" << m_variant_id << "," << "\"" << (*it).c_str() << "\"" << ")"; + + if(!db->insert_multiple("INSERT INTO dbg_filename (variant_id, path) VALUES ", sql.str().c_str())){ + LOG << "Can not import filename!"<< endl; + } + } + return true; +} + +bool ElfImporter::import_mapping(std::string fileName) { + + std::list mapping; + if (!dwReader.read_mapping(fileName, mapping)) { + return false; + } + + while (!mapping.empty()) + { + struct addrToLine temp_addrToLine; + temp_addrToLine = mapping.front(); + + std::stringstream ss; + ss << "SELECT file_id FROM dbg_filename WHERE path = " << "\"" << temp_addrToLine.lineSource << "\""; + + MYSQL_RES *res = db->query(ss.str().c_str(), true); + MYSQL_ROW row; + row = mysql_fetch_row(res); + + // INSERT group entry + std::stringstream sql; + sql << "(" << m_variant_id << "," << temp_addrToLine.absoluteAddr << "," << temp_addrToLine.lineNumber << ","; + if(row != NULL) { + sql << row[0] << ")"; + } else { + sql << -1 << ")"; + } + + //ToDo: Skip duplicated entrys with ON DUPLICATE KEY UPDATE + if(!db->insert_multiple("INSERT IGNORE INTO dbg_mapping (variant_id, instr_absolute, linenumber, file_id) VALUES ", sql.str().c_str())){ + LOG << "Can not import line number information!" << endl;; + } + + mapping.pop_front(); + } + + return true; +} bool ElfImporter::clear_database() { + CommandLine &cmd = CommandLine::Inst(); std::stringstream ss; + bool ret = true; + + ss.str(""); ss << "DELETE FROM objdump WHERE variant_id = " << m_variant_id; - bool ret = db->query(ss.str().c_str()) == 0 ? false : true; + if (ret) { + ret = db->query(ss.str().c_str()) == 0 ? false : true; + } LOG << "deleted " << db->affected_rows() << " rows from objdump table" << std::endl; + + + //ToDo: Reset auto increment value to 1 + if (cmd[SOURCECODE]) { + ss.str(""); + ss << "DELETE FROM dbg_source WHERE variant_id = " << m_variant_id; + + if (ret) { + ret = db->query(ss.str().c_str()) == 0 ? false : true; + } + LOG << "deleted " << db->affected_rows() << " rows from dbg_source table" << std::endl; + + ss.str(""); + ss << "DELETE FROM dbg_filename WHERE variant_id = " << m_variant_id; + + if (ret) { + ret = db->query(ss.str().c_str()) == 0 ? false : true; + } + LOG << "deleted " << db->affected_rows() << " rows from dbg_source table" << std::endl; + } + + //ToDo: Reset auto increment value to 1 + if (cmd[DEBUGINFO]) { + ss.str(""); + ss << "DELETE FROM dbg_mapping WHERE variant_id = " << m_variant_id; + + if (ret) { + ret = db->query(ss.str().c_str()) == 0 ? false : true; + } + LOG << "deleted " << db->affected_rows() << " rows from source table" << std::endl; + } + return ret; } diff --git a/tools/import-trace/ElfImporter.hpp b/tools/import-trace/ElfImporter.hpp index 20f4a656..f1a77189 100644 --- a/tools/import-trace/ElfImporter.hpp +++ b/tools/import-trace/ElfImporter.hpp @@ -1,9 +1,15 @@ #ifndef __OBJDUMP_IMPORTER_H__ #define __OBJDUMP_IMPORTER_H__ +#include +#include +#include "libdwarf.h" +#include "libelf.h" + #include "Importer.hpp" #include "util/llvmdisassembler/LLVMDisassembler.hpp" #include "util/CommandLine.hpp" +#include "util/DwarfReader.hpp" /** The ElfImporter is not a real trace importer, but we locate it @@ -13,12 +19,25 @@ The ElfImporter calls objdump and dissassembles an elf binary and imports the results into the database + + In addition, debugging information can be imported: + + If the --sources option is set, the source files will be imported + into the database. Only the files that were actually used in the + elf binary will be imported. + + If the --debug option is set, the line number table of the elf binary will + be imported into the database. The information will be stored in the + "mapping" table. */ class ElfImporter : public Importer { llvm::OwningPtr binary; llvm::OwningPtr disas; fail::CommandLine::option_handle OBJDUMP; + fail::CommandLine::option_handle SOURCECODE; + fail::CommandLine::option_handle DEBUGINFO; + fail::DwarfReader dwReader; bool import_with_objdump(const std::string &objdump_binary); bool evaluate_objdump_line(const std::string &line); @@ -27,6 +46,10 @@ class ElfImporter : public Importer { bool import_instruction(fail::address_t addr, char opcode[16], int opcode_length, const std::string &instruction, const std::string &comment); + bool import_source_files(const std::string& fileName,std::list& lines); + bool import_source_code(std::string fileName); + bool import_mapping(std::string fileName); + protected: virtual bool handle_ip_event(fail::simtime_t curtime, instruction_count_t instr, Trace_Event &ev) { return true; }