import-trace: import debug info

If the --debug option is set, the line number table of the elf binary will
be imported into the database. The information will be stored in the
"dbg_mapping" table.

If the --sources option is set, the source files will be imported
into the database. Only the files that were actually used in the
elf binary will be imported.

Change-Id: I0e9de6b456bc42b329c1700c25e5839d9552cdbb

doc/how-to-build.txt

@@ -8,6 +8,8 @@ Required for Fail*:
  - libprotobuf-dev
  - libpcl1-dev
  - libboost-thread-dev
+ - libdwarf
+ - libelf
  - protobuf-compiler
  - cmake 2.8.2 (2.8.11 preferred)
  - cmake-curses-gui

src/core/util/CMakeLists.txt

@@ -11,6 +11,8 @@ set(SRCS
  Demangler.cc
  Disassembler.hpp
  Disassembler.cc
+ DwarfReader.cc
+ DwarfReader.hpp
  elfinfo/elfinfo.cc
  elfinfo/elfinfo.h
  gzstream/gzstream.C
@@ -65,7 +67,7 @@ mark_as_advanced(FAIL_OBJDUMP)
 
 add_library(fail-util ${SRCS})
 add_dependencies(fail-util fail-comm)
-target_link_libraries(fail-util ${PROTOBUF_LIBRARY} ${Boost_LIBRARIES} ${LIB_IBERTY} ${ZLIB_LIBRARIES})
+target_link_libraries(fail-util ${PROTOBUF_LIBRARY} ${Boost_LIBRARIES} ${LIB_IBERTY} ${ZLIB_LIBRARIES} dwarf elf)
 
 option(BUILD_LLVM_DISASSEMBLER "Build the LLVM-based disassembler (LLVM 3.3 preferred, for 3.1 and 3.2 read doc/how-to-build.txt)" OFF)
 if (BUILD_LLVM_DISASSEMBLER)

src/core/util/DwarfReader.cc

@@ -0,0 +1,258 @@
+#include <fstream>
+#include <fcntl.h>
+#include <unistd.h>
+
+#include "DwarfReader.hpp"
+#include "libdwarf.h"
+#include "libelf.h"
+#include "Logger.hpp"
+
+/**
+	* This source code is based on bcov 0.2.
+	* Sourcefile: src/coverage.cpp
+	* http://bcov.sourceforge.net/
+	* GNU GENERAL PUBLIC LICENSE
+*/
+
+using namespace fail;
+using namespace std;
+
+static Logger LOG("DwarfReader");
+
+static void dwarfErrorHandler(Dwarf_Error error, Dwarf_Ptr /*userData*/)
+{
+	char* msg=dwarf_errmsg(error);
+	cerr << "dwarf error: " << msg << endl;
+}
+
+static string normalize(const string& filePath)
+{
+	// A quick scan first...
+	bool hadSep=false,needsFix=false;
+	string::size_type len=filePath.length();
+	if (!needsFix)
+	for (string::size_type index=0;index<len;index++) {
+		char c=filePath[index];
+		if (c=='/') {
+			if (hadSep) {
+				needsFix=true;
+			}
+			hadSep=true;
+		} else {
+			if (c=='.') {
+				if (hadSep||(index==0)) {
+					needsFix=true;
+				}
+			}
+			hadSep=false;
+		}
+	}
+	if (!needsFix) {
+		return filePath;
+	}
+	hadSep=false;
+	// Construct the fixed result
+	string result;
+	for (string::size_type index=0;index<len;index++) {
+		char c=filePath[index];
+		if (c=='/') {
+			if (hadSep) {
+			} else result+=c;
+			hadSep=true;
+		} else {
+			if ((c=='.')&&(hadSep||(index==0))) {
+				if (index+1>=len) {
+					if (hadSep)
+						result.resize(result.length()-1);
+				} else {
+					result+=c;
+				}
+				continue;
+			}
+			char n=filePath[index+1];
+			if (n=='/') {
+				index++; continue;
+			}
+			if (n=='.') {
+				if (index+2>=len) {
+					index++;
+					string::size_type split=result.rfind('/',result.length()-2);
+					if (split!=string::npos) {
+						if (result.substr(split)!="/../") {
+							result.resize(split);
+						}
+					} else if (result.length()>0) {
+						if ((result!="../")&&(result!="/")) {
+							result.clear();
+						}
+					} else result="..";
+					continue;
+				} else {
+					n=filePath[index+2];
+					if (n=='/') {
+						index+=2;
+						string::size_type split=result.rfind('/',result.length()-2);
+						if (split!=string::npos) {
+							if (result.substr(split)!="/../") {
+								result.resize(split+1);
+							}
+						} else if (result.length()>0) {
+							if ((result!="../")&&(result!="/")) {
+								result.clear();
+							}
+						} else result="../";
+						continue;
+					}
+				}
+			}
+		}
+		result+=c; hadSep=false;
+	}
+	return result;
+}
+
+bool DwarfReader::read_source_files(const std::string& fileName,std::list<std::string>& lines) {
+
+	// Open The file
+	int fd=open(fileName.c_str(),O_RDONLY);
+	if (fd<0) return false;
+
+	// Initialize libdwarf
+	Dwarf_Debug dbg;
+	int status = dwarf_init(fd, DW_DLC_READ,dwarfErrorHandler,0,&dbg,0);
+	if (status==DW_DLV_ERROR) { close(fd); return false; }
+	if (status==DW_DLV_NO_ENTRY) { close(fd); return true; }
+
+	// Iterator over the headers
+	Dwarf_Unsigned header;
+	while (dwarf_next_cu_header(dbg,0,0,0,0,&header,0)==DW_DLV_OK) {
+		// Access the die
+		Dwarf_Die die;
+		if (dwarf_siblingof(dbg,0,&die,0)!=DW_DLV_OK)
+			return false;
+
+		// Get the source lines
+		Dwarf_Line* lineBuffer;
+		Dwarf_Signed lineCount;
+		if (dwarf_srclines(die,&lineBuffer,&lineCount,0)!=DW_DLV_OK)
+			continue; //return false;
+
+		// Store them
+		for (int index=0;index<lineCount;index++) {
+			Dwarf_Unsigned lineNo;
+			if (dwarf_lineno(lineBuffer[index],&lineNo,0)!=DW_DLV_OK)
+				return false;
+			char* lineSource;
+			if (dwarf_linesrc(lineBuffer[index],&lineSource,0)!=DW_DLV_OK)
+				return false;
+			Dwarf_Bool isCode;
+			if (dwarf_linebeginstatement(lineBuffer[index],&isCode,0)!=DW_DLV_OK)
+				return false;
+			Dwarf_Addr addr;
+			if (dwarf_lineaddr(lineBuffer[index],&addr,0)!=DW_DLV_OK)
+				return false;
+
+			if (lineNo&&isCode) {
+				//LOG << "lineNo: " << lineNo   << " addr: " << reinterpret_cast<void*>(addr) << " line source:" << normalize(lineSource) << endl;
+				lines.push_back(normalize(lineSource));
+			}
+
+			dwarf_dealloc(dbg,lineSource,DW_DLA_STRING);
+		}
+
+		// Release the memory
+		for (int index=0;index<lineCount;index++)
+			dwarf_dealloc(dbg,lineBuffer[index],DW_DLA_LINE);
+		dwarf_dealloc(dbg,lineBuffer,DW_DLA_LIST);
+	}
+
+	//Remove duplicated entrys
+	lines.sort();
+	lines.unique();
+
+	// Shut down libdwarf
+	if (dwarf_finish(dbg,0)!=DW_DLV_OK)
+		return false;
+
+	close(fd);
+	return true;
+}
+
+bool DwarfReader::read_mapping(std::string fileName, std::list<addrToLine>& addrToLineList) {
+
+	// Open The file
+	int fd=open(fileName.c_str(),O_RDONLY);
+	if (fd<0) {
+		return false;
+	}
+
+	// Initialize libdwarf
+	Dwarf_Debug dbg;
+	int status = dwarf_init(fd, DW_DLC_READ,dwarfErrorHandler,0,&dbg,0);
+	if (status==DW_DLV_ERROR) {
+		close(fd);
+		return false;
+	}
+	if (status==DW_DLV_NO_ENTRY) {
+		close(fd);
+		return true;
+	}
+
+	// Iterator over the headers
+	Dwarf_Unsigned header;
+	while (dwarf_next_cu_header(dbg,0,0,0,0,&header,0)==DW_DLV_OK) {
+		// Access the die
+		Dwarf_Die die;
+		if (dwarf_siblingof(dbg,0,&die,0)!=DW_DLV_OK) {
+			return false;
+		}
+
+		// Get the source lines
+		Dwarf_Line* lineBuffer;
+		Dwarf_Signed lineCount;
+		if (dwarf_srclines(die,&lineBuffer,&lineCount,0)!=DW_DLV_OK) {
+			continue; //return false;
+		}
+
+		// Store them
+		for (int index=0;index<lineCount;index++) {
+			Dwarf_Unsigned lineNo;
+			if (dwarf_lineno(lineBuffer[index],&lineNo,0)!=DW_DLV_OK) {
+				return false;
+			}
+			char* lineSource;
+			if (dwarf_linesrc(lineBuffer[index],&lineSource,0)!=DW_DLV_OK) {
+				return false;
+			}
+			Dwarf_Bool isCode;
+			if (dwarf_linebeginstatement(lineBuffer[index],&isCode,0)!=DW_DLV_OK) {
+				return false;
+			}
+			Dwarf_Addr addr;
+			if (dwarf_lineaddr(lineBuffer[index],&addr,0)!=DW_DLV_OK) {
+				return false;
+			}
+
+			if (lineNo&&isCode) {
+				struct addrToLine newLine = { (int) addr, (int) lineNo, normalize(lineSource) };
+				addrToLineList.push_back(newLine);
+			}
+
+			dwarf_dealloc(dbg,lineSource,DW_DLA_STRING);
+		}
+
+		// Release the memory
+		for (int index=0;index<lineCount;index++) {
+			dwarf_dealloc(dbg,lineBuffer[index],DW_DLA_LINE);
+		}
+		dwarf_dealloc(dbg,lineBuffer,DW_DLA_LIST);
+	}
+
+	// Shut down libdwarf
+	if (dwarf_finish(dbg,0)!=DW_DLV_OK) {
+		return false;
+	}
+
+	close(fd);
+	return true;
+}

src/core/util/DwarfReader.hpp

@@ -0,0 +1,40 @@
+#ifndef __DWARFREADER_HPP__
+#define __DWARFREADER_HPP__
+
+#include <string>
+#include <ostream>
+#include <list>
+
+namespace fail {
+
+
+/**
+	* This source code is based on bcov 0.2.
+	* Sourcefile: src/coverage.cpp
+	* http://bcov.sourceforge.net/
+	* GNU GENERAL PUBLIC LICENSE
+*/
+
+	struct addrToLine {
+		int absoluteAddr;
+		int lineNumber;
+		std::string lineSource;
+	};
+
+	/**
+	* \class DwarfReader
+	* ToDO
+	*/
+
+	class DwarfReader {
+
+		public:
+
+		bool read_source_files(const std::string& fileName, std::list<std::string>& lines);
+		bool read_mapping(std::string fileName, std::list<addrToLine>& addrToLineList);
+};
+
+} // end-of-namespace fail
+
+#endif //__DwarfREADER_HPP__
+

tools/import-trace/CMakeLists.txt

@@ -33,7 +33,8 @@ target_link_libraries(import-trace
   ${MYSQL_LIBRARIES}
   fail-util 
   fail-comm
-  fail-sal)
+  fail-sal
+)
 
 if (BUILD_LLVM_DISASSEMBLER)
   target_link_libraries(import-trace fail-llvmdisassembler fail-sal ${LLVM_LIBS} ${LLVM_LDFLAGS} ${Boost_LIBRARIES})

tools/import-trace/ElfImporter.cc

@@ -1,5 +1,6 @@
 #include <sstream>
 #include <iostream>
+#include <fstream>
 #include "util/Logger.hpp"
 #include "ElfImporter.hpp"
 #include "util/pstream.h"
@@ -25,11 +26,18 @@ bool ElfImporter::cb_commandline_init() {
 
 	OBJDUMP = cmd.addOption("", "objdump", Arg::Required,
 							"--objdump \tObjdump: location of objdump binary, otherwise LLVM Disassembler is used");
+	SOURCECODE = cmd.addOption("", "sources", Arg::None,
+							"--sources \timport all source files into the database");
+	DEBUGINFO = cmd.addOption("", "debug", Arg::None,
+							"--debug \timport some debug informations into the database");
 	return true;
 }
 
 bool ElfImporter::create_database() {
+	CommandLine &cmd = CommandLine::Inst();
 	std::stringstream create_statement;
+
+	create_statement.str("");
 	create_statement << "CREATE TABLE IF NOT EXISTS objdump ("
 		"	variant_id int(11) NOT NULL,"
 		"	instr_address int(11) NOT NULL,"
@@ -38,7 +46,50 @@ bool ElfImporter::create_database() {
 		"	comment VARCHAR(128),"
 		"	PRIMARY KEY (variant_id,instr_address)"
 		") engine=MyISAM ";
-	return db->query(create_statement.str().c_str());
+	if (!db->query(create_statement.str().c_str())) {
+		return false;
+	}
+
+	if (cmd[SOURCECODE]) {
+		create_statement.str("");
+		create_statement << "CREATE TABLE IF NOT EXISTS dbg_filename ("
+			"	file_id int(11) NOT NULL AUTO_INCREMENT,"
+			"	variant_id int(11) NOT NULL,"
+			"	path VARCHAR(1024),"
+			"	PRIMARY KEY (file_id)"
+			") engine=MyISAM ";
+		if (!db->query(create_statement.str().c_str())) {
+			return false;
+		}
+
+		create_statement.str("");
+		create_statement << "CREATE TABLE IF NOT EXISTS dbg_source ("
+			"	variant_id int(11) NOT NULL,"
+			"	linenumber int(11) NOT NULL,"
+			"	file_id int(11) NOT NULL,"
+			"	line VARCHAR(1024),"
+			"	PRIMARY KEY (variant_id, linenumber, file_id)"
+			") engine=MyISAM ";
+		if (!db->query(create_statement.str().c_str())) {
+			return false;
+		}
+	}
+
+	if (cmd[DEBUGINFO]) {
+		create_statement.str("");
+		create_statement << "CREATE TABLE IF NOT EXISTS dbg_mapping ("
+			"	variant_id int(11) NOT NULL,"
+			"	instr_absolute int(11) NOT NULL,"
+			"	linenumber int(11) NOT NULL,"
+			"	file_id int(11) NOT NULL,"
+			"	PRIMARY KEY (variant_id, instr_absolute ,linenumber)"
+			") engine=MyISAM ";
+		if (!db->query(create_statement.str().c_str())) {
+			return false;
+		}
+	}
+
+	return true;
 }
 
 bool ElfImporter::copy_to_database(ProtoIStream &ps) {
@@ -54,12 +105,33 @@ bool ElfImporter::copy_to_database(ProtoIStream &ps) {
 	}
 
 	if (cmd[OBJDUMP]) { // using objdump
-		return import_with_objdump(std::string(cmd[OBJDUMP].first()->arg));
+		if(!import_with_objdump(std::string(cmd[OBJDUMP].first()->arg))) {
+			return false;
+		}
 	} else {
 		LOG << "importing an objdump with internal llvm dissassembler is not yet implemented" << std::endl;
-		return false;
 	}
 
+	if (cmd[SOURCECODE]) { // import sources
+		std::list<std::string> sourceFiles;
+
+		if (!import_source_files(m_elf->getFilename(),sourceFiles)) {
+			cerr << "unable to read dwarf2 debug info" << endl;
+			return false;
+		}
+
+		for (std::list<std::string>::iterator it = sourceFiles.begin(); it != sourceFiles.end(); ++it) {
+			if(!import_source_code(*it)) {
+				return false;
+			}
+		}
+	}
+
+	if (cmd[DEBUGINFO]) { // import debug informations
+		if(!import_mapping(m_elf->getFilename())) {
+			return false;
+		}
+	}
 
 	return true;
 }
@@ -196,14 +268,161 @@ bool ElfImporter::import_instruction(fail::address_t addr, char *opcode, int opc
 	return true;
 }
 
+bool ElfImporter::import_source_code(std::string fileName) {
 
+	LOG << "Importing Sourcefile: " << fileName << endl;
+
+	ifstream in(fileName.c_str());
+	if (!in.is_open()) {
+		LOG << "Sourcefile not found: " << fileName << endl;
+	} else {
+		unsigned lineNo=0;
+		while (!in.eof()) {
+			++lineNo;
+
+			// Read and strip the current line
+			string currentLine;
+			getline(in,currentLine);
+			if ((!currentLine.length())&&(in.eof())) break;
+			while (true) {
+				unsigned l=currentLine.length();
+				if (!l) break;
+				char c=currentLine[l-1];
+				if ((c!='\n')&&(c!='\r')&&(c!=' ')&&(c!='\t')) break;
+				currentLine=currentLine.substr(0,l-1);
+			}
+
+			//Sonderzeichen
+			currentLine = db->escape_string(currentLine);
+
+			std::stringstream ss;
+			ss << "SELECT file_id FROM dbg_filename WHERE path = " << "\"" << fileName.c_str() << "\"";
+
+			MYSQL_RES *res = db->query(ss.str().c_str(), true);
+			MYSQL_ROW row;
+			row = mysql_fetch_row(res);
+
+			// INSERT group entry
+			std::stringstream sql;
+
+			sql << "(" << m_variant_id << "," << lineNo << ",";
+			if (row != NULL) {
+				sql << row[0];
+			} else {
+				sql << -1;
+			}
+			sql << "," << "\"" << currentLine << "\"" << ")";
+
+			if (!db->insert_multiple("INSERT INTO dbg_source (variant_id, linenumber, file_id, line) VALUES ", sql.str().c_str())){
+				LOG << "Can not import sourcelines!" << endl;;
+			}
+		}
+		db->insert_multiple();
+	}
+	return true;
+}
+
+bool ElfImporter::import_source_files(const std::string& fileName,std::list<std::string>& lines) {
+
+	//std::list<addrToLine> lines;
+	if (!dwReader.read_source_files(fileName, lines)) {
+		return false;
+	}
+	for (std::list<std::string>::iterator it = lines.begin(); it != lines.end(); ++it) {
+		// INSERT group entry
+		std::stringstream sql;
+		sql << "(" << m_variant_id << "," << "\"" << (*it).c_str() << "\"" << ")";
+
+		if(!db->insert_multiple("INSERT INTO dbg_filename (variant_id, path) VALUES ", sql.str().c_str())){
+			LOG << "Can not import filename!"<< endl;
+		}
+	}
+	return true;
+}
+
+bool ElfImporter::import_mapping(std::string fileName) {
+
+	std::list<addrToLine> mapping;
+	if (!dwReader.read_mapping(fileName, mapping)) {
+		return false;
+	}
+
+	while (!mapping.empty())
+	{
+		struct addrToLine temp_addrToLine;
+		temp_addrToLine = mapping.front();
+
+		std::stringstream ss;
+		ss << "SELECT file_id FROM dbg_filename WHERE path = " << "\"" << temp_addrToLine.lineSource << "\"";
+
+		MYSQL_RES *res = db->query(ss.str().c_str(), true);
+		MYSQL_ROW row;
+		row = mysql_fetch_row(res);
+
+		// INSERT group entry
+		std::stringstream sql;
+		sql << "(" << m_variant_id << "," << temp_addrToLine.absoluteAddr << "," << temp_addrToLine.lineNumber << ",";
+		if(row != NULL) {
+			sql << row[0] << ")";
+		} else {
+			sql << -1 << ")";
+		}
+
+		//ToDo: Skip duplicated entrys with ON DUPLICATE KEY UPDATE
+		if(!db->insert_multiple("INSERT IGNORE INTO dbg_mapping (variant_id, instr_absolute, linenumber, file_id) VALUES ", sql.str().c_str())){
+			LOG << "Can not import line number information!" << endl;;
+		}
+
+		mapping.pop_front();
+	}
+
+	return true;
+}
 
 bool ElfImporter::clear_database() {
+	CommandLine &cmd = CommandLine::Inst();
 	std::stringstream ss;
+	bool ret = true;
+
+	ss.str("");
 	ss << "DELETE FROM objdump WHERE variant_id = " << m_variant_id;
 
-	bool ret = db->query(ss.str().c_str()) == 0 ? false : true;
+	if (ret) {
+		ret = db->query(ss.str().c_str()) == 0 ? false : true;
+	}
 	LOG << "deleted " << db->affected_rows() << " rows from objdump table" << std::endl;
+
+
+	//ToDo: Reset auto increment value to 1
+	if (cmd[SOURCECODE]) {
+		ss.str("");
+		ss << "DELETE FROM dbg_source WHERE variant_id = " << m_variant_id;
+
+		if (ret) {
+			ret = db->query(ss.str().c_str()) == 0 ? false : true;
+		}
+		LOG << "deleted " << db->affected_rows() << " rows from dbg_source table" << std::endl;
+
+		ss.str("");
+		ss << "DELETE FROM dbg_filename WHERE variant_id = " << m_variant_id;
+
+		if (ret) {
+			ret = db->query(ss.str().c_str()) == 0 ? false : true;
+		}
+		LOG << "deleted " << db->affected_rows() << " rows from dbg_source table" << std::endl;
+	}
+
+	//ToDo: Reset auto increment value to 1
+	if (cmd[DEBUGINFO]) {
+		ss.str("");
+		ss << "DELETE FROM dbg_mapping WHERE variant_id = " << m_variant_id;
+
+		if (ret) {
+			ret = db->query(ss.str().c_str()) == 0 ? false : true;
+		}
+		LOG << "deleted " << db->affected_rows() << " rows from source table" << std::endl;
+	}
+
 	return ret;
 }
 

tools/import-trace/ElfImporter.hpp

@@ -1,9 +1,15 @@
 #ifndef __OBJDUMP_IMPORTER_H__
 #define __OBJDUMP_IMPORTER_H__
 
+#include <string>
+#include <list>
+#include "libdwarf.h"
+#include "libelf.h"
+
 #include "Importer.hpp"
 #include "util/llvmdisassembler/LLVMDisassembler.hpp"
 #include "util/CommandLine.hpp"
+#include "util/DwarfReader.hpp"
 
 /**
 	The ElfImporter is not a real trace importer, but we locate it
@@ -13,12 +19,25 @@
 
 	The ElfImporter calls objdump and dissassembles an elf binary
 	and imports the results into the database
+
+	In addition, debugging information can be imported:
+
+	If the --sources option is set, the source files will be imported
+	into the database. Only the files that were actually used in the
+	elf binary will be imported.
+
+	If the --debug option is set, the line number table of the elf binary will
+	be imported into the database. The information will be stored in the
+	"mapping" table.
 */
 class ElfImporter : public Importer {
 	llvm::OwningPtr<llvm::object::Binary> binary;
 	llvm::OwningPtr<fail::LLVMDisassembler> disas;
 
 	fail::CommandLine::option_handle OBJDUMP;
+	fail::CommandLine::option_handle SOURCECODE;
+	fail::CommandLine::option_handle DEBUGINFO;
+	fail::DwarfReader dwReader;
 
 	bool import_with_objdump(const std::string &objdump_binary);
 	bool evaluate_objdump_line(const std::string &line);
@@ -27,6 +46,10 @@ class ElfImporter : public Importer {
 	bool import_instruction(fail::address_t addr, char opcode[16], int opcode_length,
 								const std::string &instruction, const std::string &comment);
 
+	bool import_source_files(const std::string& fileName,std::list<std::string>& lines);
+	bool import_source_code(std::string fileName);
+	bool import_mapping(std::string fileName);
+
 protected:
 	virtual bool handle_ip_event(fail::simtime_t curtime, instruction_count_t instr,
 									 Trace_Event &ev) { return true; }