shutdown cleanups revisited

This change became necessary as we observed weird fail-client SIGSEGV
crashes with both Bochs and Gem5 backends and different experiments.

Some Fail* components are instantiated statically: the
SimulatorController instance "simulator", containing the
ListenerManager and the CoroutineManager, and the active
ExperimentFlow subclass(es)
(experiments/instantiate-experiment*.ah.in).  The experiment(s) is
registered as an active flow in the CoroutineManager at startup.

As plugins (which are ExperimentFlows themselves) are often created on
an experiment's stack, ExperimentFlows deregister themselves on
destruction (e.g., when leaving the plugin variable's scope).  The
core problem is, that the creation and destruction order of statically
instantiated objects depends on the link order; if the experiment is
destroyed after the CoroutineManager, its automatic self-deregistering
feature talks to the smoking ruins of the latter.

This change removes all static instantiations of ExperimentFlow and
replaces them with constructions on the heap.  Additionally it makes
sure that the CoroutineManager recognizes that a shutdown is in
progress, and refrains from touching potentially already destroyed
data structures when a (mistakenly globally instantiated)
ExperimentFlow deregisters in this case.

Change-Id: I8a7d42fb141222cd2cce6040ab1a01f9de61be24

src/core/efw/CoroutineManager.cc

@@ -8,9 +8,9 @@ namespace fail {
 
 void CoroutineManager::m_invoke(void* pData)
 {
-	//std::cerr << "CORO m_invoke " << co_current() << std::endl;
-	// TODO: Log-Level?
-	reinterpret_cast<ExperimentFlow*>(pData)->coroutine_entry();
+	ExperimentFlow *flow = reinterpret_cast<ExperimentFlow*>(pData);
+	flow->coroutine_entry();
+	simulator.removeFlow(flow);
 	//m_togglerstack.pop();
 	// FIXME: need to pop our caller
 	co_exit(); // deletes the associated coroutine memory as well
@@ -20,7 +20,12 @@ void CoroutineManager::m_invoke(void* pData)
 	while (1); // freeze.
 }
 
-CoroutineManager::~CoroutineManager() { }
+CoroutineManager::~CoroutineManager()
+{
+	// Note that we do not destroy the associated coroutines; this causes
+	// problems when shutting down.
+	m_Flows.clear();
+}
 
 void CoroutineManager::toggle(ExperimentFlow* flow)
 {
@@ -53,7 +58,9 @@ void CoroutineManager::remove(ExperimentFlow* flow)
 	// find coroutine handle for this flow
 	flowmap_t::iterator it = m_Flows.find(flow);
 	if (it == m_Flows.end()) {
-		assert(false && "FATAL ERROR: Cannot remove flow");
+		// Not finding the flow to remove is not an error; especially when
+		// shutting down this is the common case, as ~CoroutineManager probably
+		// clears the flow list before the ExperimentFlow destructors run.
 		return;
 	}
 	corohandle_t coro = it->second;
@@ -67,7 +74,9 @@ void CoroutineManager::remove(ExperimentFlow* flow)
 	// delete coroutine (and handle the special case we're removing
 	// ourselves)
 	if (coro == co_current()) {
-		co_exit();
+		if (!m_Terminated) {
+			co_exit();
+		}
 	} else {
 		co_delete(coro);
 	}

src/core/efw/CoroutineManager.hpp

@@ -29,10 +29,12 @@ private:
 	std::stack<corohandle_t> m_togglerstack;
 	//! manages the run-calls for each ExperimentFlow-object
 	static void m_invoke(void* pData);
+	//! \c true if terminated explicitly using simulator.terminate()
+	bool m_Terminated;
 public:
 	static const ExperimentFlow* SIM_FLOW; //!< the simulator coroutine flow
 
-	CoroutineManager() : m_simCoro(co_current()) { }
+	CoroutineManager() : m_simCoro(co_current()), m_Terminated(false) { }
 	~CoroutineManager();
 	/**
 	 * Creates a new coroutine for the specified experiment flow.
@@ -63,6 +65,12 @@ public:
 	 * @return the current experiment flow.
 	 */
 	ExperimentFlow* getCurrent();
+	/**
+	 * Sets the termination flag. This should be called when Fail
+	 * exists due to a call to \c ::exit() (used, e.g., in
+	 * \c SimulatorController::terminate()). This cannot be undone.
+	 */
+	 void setTerminated() { m_Terminated = true; }
 };
 
 } // end-of-namespace: fail

src/core/efw/ExperimentFlow.hpp

@@ -13,6 +13,11 @@ namespace fail {
 class ExperimentFlow {
 public:
 	ExperimentFlow() { }
+	virtual ~ExperimentFlow()
+	{
+		simulator.clearListeners(this); // remove residual events
+		simulator.removeFlow(this);
+	}
 	/**
 	 * Defines the experiment flow.
 	 * @return \c true if the experiment was successful, \c false otherwise

src/core/sal/SimulatorController.cc

@@ -217,6 +217,8 @@ void SimulatorController::terminate(int exCode)
 	// Attention: This could cause problems, e.g., because of non-closed sockets
 	std::cout << "[FAIL] Exit called by experiment with exit code: " << exCode << std::endl;
 	// TODO: (Non-)Verbose-Mode? Log-Level?
+
+	m_Flows.setTerminated(); // we are about to terminate
 	exit(exCode);
 }
 

src/experiments/ecos_kernel_test/instantiateExperiment.cc

@@ -1,8 +1,7 @@
 #include "experiment.hpp"
 #include "sal/SALInst.hpp"
 
-static EcosKernelTestExperiment experiment;
 void instantiateEcosKernelTestExperiment()
 {
-	fail::simulator.addFlow(&experiment);
+	fail::simulator.addFlow(new EcosKernelTestExperiment);
 }

src/experiments/instantiate-experiment-indirect.ah.in

@@ -12,6 +12,10 @@
 // You need to provide the implementation of this function in your experiment
 // directory:
 void instantiate@EXPERIMENT_TYPE@();
+// The experiment needs to be instantiated dynamically (on the stack, or the
+// heap), as the ExperimentFlow destructor deregisters from the
+// CoroutineManager which may not exist anymore if the global
+// construction/destruction order is inappropriate.
 
 aspect @EXPERIMENT_TYPE@ExperimentHook {
 	advice execution ("void fail::SimulatorController::initExperiments()") : after () {

src/experiments/instantiate-experiment.ah.in

@@ -9,10 +9,14 @@
 #include "../experiments/@EXPERIMENT_NAME@/experiment.hpp"
 #include "sal/SALInst.hpp"
 
+// The experiment needs to be instantiated dynamically (on the stack, or the
+// heap), as the ExperimentFlow destructor deregisters from the
+// CoroutineManager which may not exist anymore if the global
+// construction/destruction order is inappropriate.
+
 aspect @EXPERIMENT_TYPE@ExperimentHook {
-	@EXPERIMENT_TYPE@ experiment;
 	advice execution ("void fail::SimulatorController::initExperiments()") : after () {
-		fail::simulator.addFlow(&experiment);
+		fail::simulator.addFlow(new @EXPERIMENT_TYPE@);
 	}
 };
 

src/experiments/nanojpeg/instantiateExperiment.cc

@@ -1,8 +1,7 @@
 #include "experiment.hpp"
 #include "sal/SALInst.hpp"
 
-static NanoJPEGExperiment experiment;
 void instantiateNanoJPEGExperiment()
 {
-	fail::simulator.addFlow(&experiment);
+	fail::simulator.addFlow(new NanoJPEGExperiment);
 }

src/experiments/rampage/instantiateExperiment.cc

@@ -1,8 +1,7 @@
 #include "experiment.hpp"
 #include "sal/SALInst.hpp"
 
-static RAMpageExperiment experiment;
 void instantiateRAMpageExperiment()
 {
-	fail::simulator.addFlow(&experiment);
+	fail::simulator.addFlow(new RAMpageExperiment);
 }