Rewrite Mnemonics + ResultsDataWriteGroups queries to be faster

scripts/Queries/Mnemonics.pm

@@ -16,17 +16,25 @@ sub query {
     my $filters =
       Filters::build_filter_clause( $experiment_dir, @filter_config_names );
 
+    # The fspgroup/fsppilot joins are only needed for the filters
+    my $filter_joins = "";
+    if ( $filters =~ /\b(?:g|p)\./ ) {
+        $filter_joins .=
+"\nLEFT JOIN fspgroup g ON g.variant_id = t.variant_id AND g.instr2 = t.instr2 AND g.data_physical_address = t.data_physical_address";
+    }
+    if ( $filters =~ /\bp\./ ) {
+        $filter_joins .= "\nLEFT JOIN fsppilot p ON p.id = g.pilot_id";
+    }
+
     my $querystring = "SELECT
 CASE
   WHEN o.disassemble IS NOT NULL AND SUBSTRING_INDEX(TRIM(o.disassemble), ' ', 1) REGEXP '^[a-zA-Z][a-zA-Z0-9 ]*\$'
   THEN SUBSTRING_INDEX(TRIM(o.disassemble), ' ', 1)
   ELSE NULL
 END AS mnemonic,
-COUNT(DISTINCT CONCAT(t.instr2, '_', t.data_physical_address)) AS count
+COUNT(DISTINCT t.instr2, t.data_physical_address) AS count
 FROM variant v
-JOIN trace t ON v.id = t.variant_id
+JOIN trace t ON v.id = t.variant_id$filter_joins
-LEFT JOIN fspgroup g ON g.variant_id = t.variant_id AND g.instr2 = t.instr2 AND g.data_physical_address = t.data_physical_address
-LEFT JOIN fsppilot p ON p.id = g.pilot_id
 JOIN objdump o ON o.variant_id = v.id AND o.instr_address = t.instr2_absolute
 WHERE v.variant = '$experiment'$filters
 GROUP BY mnemonic

scripts/Queries/ResultsDataWriteGroups.pm

@@ -16,25 +16,77 @@ sub query {
     my $extra =
       Filters::build_filter_clause( $experiment_dir, @filter_config_names );
 
-    # This query asks the same question as ResultsData.pm: How much faultspace
+    # The write branch applies the data filters on the trace and the
-    # area ended up in each possible outcome?
+    # instruction filters on the pilot, so the clause is built per column.
+    my $data_extra =
+      Filters::build_filter_clause_restricted( $experiment_dir, qr/^g\./,
+        @filter_config_names );
+    my $instr_extra =
+      Filters::build_filter_clause_restricted( $experiment_dir, qr/^p\./,
+        @filter_config_names );
+
+    # This is the same as ResultsData.pm (how much faultspace
+    # area ended up in each possible outcome), but applies the
+    # possible writegroups fix.
     #
-    # The difference is how the write equivalence classes are treated.
+    # At first I have written this query similar to ResultsData.pm, with
-    # BasicPruner's write-group INSERT maps every 'W' trace entry to the single
+    # an inner join to reconstruct the "fixed" fspgroup table within the query.
-    # known_outcome=1 write pilot, but fspgroup has PRIMARY KEY (pilot_id), so
+    # This was extremely slow since it resulted in a large cross product
-    # only the first row lands in the table.
+    # between the write groups/classes and the write pilots injections
-    # This query fixes that.
+    # (also I don't think the indices were helping the way I wrote the query).
+    #
+    # This version now constructs the result in two
+    # disjunct branches (as the data is disjunct in known_outcome = 0/1).
+    my $resulttype_order =
+        "FIELD(resulttype, 'OK_MARKER', 'FAIL_MARKER', 'DETECTED_MARKER',"
+      . " 'GROUP1_MARKER', 'GROUP2_MARKER', 'GROUP3_MARKER', 'GROUP4_MARKER',"
+      . " 'TIMEOUT', 'TRAP', 'WRITE_TEXTSEGMENT', 'ACCESS_OUTERSPACE',"
+      . " 'SDC', 'UNKNOWN')";
+
     my $querystring = "SELECT
-benchmark, resulttype, SUM(g.time2 - g.time1 + 1) AS faults
+benchmark, resulttype, SUM(faults) AS faults
+FROM (
+    SELECT v.benchmark AS benchmark, rc.resulttype AS resulttype, SUM(cl.len * rc.cnt) AS faults
     FROM variant v
-JOIN trace g ON v.id = g.variant_id
+    JOIN (
+        SELECT g.variant_id, p.id AS pilot_id, SUM(g.time2 - g.time1 + 1) AS len
+        FROM trace g
         JOIN fsppilot p ON p.variant_id = g.variant_id
-    AND ( (p.known_outcome = 1 AND g.accesstype = 'W')
+            AND p.known_outcome = 0
-        OR (p.known_outcome = 0 AND p.instr2 = g.instr2 AND p.data_physical_address = g.data_physical_address) )
+            AND p.instr2 = g.instr2
+            AND p.data_physical_address = g.data_physical_address
+        WHERE 1 = 1$extra
+        GROUP BY g.variant_id, p.id
+    ) cl ON cl.variant_id = v.id
+    JOIN (
+        SELECT r.pilot_id, r.resulttype, COUNT(*) AS cnt
+        FROM result_GenericExperimentMessage r
+        GROUP BY r.pilot_id, r.resulttype
+    ) rc ON rc.pilot_id = cl.pilot_id
+    WHERE v.variant = '$experiment'
+    GROUP BY v.id, rc.resulttype
+
+    UNION ALL
+
+    SELECT v.benchmark AS benchmark, rc.resulttype AS resulttype, wl.len * rc.cnt AS faults
+    FROM variant v
+    JOIN (
+        SELECT g.variant_id, SUM(g.time2 - g.time1 + 1) AS len
+        FROM trace g
+        WHERE g.accesstype = 'W'$data_extra
+        GROUP BY g.variant_id
+    ) wl ON wl.variant_id = v.id
+    JOIN (
+        SELECT p.variant_id, r.resulttype, COUNT(*) AS cnt
+        FROM fsppilot p
         JOIN result_GenericExperimentMessage r ON r.pilot_id = p.id
-WHERE v.variant = '$experiment'$extra
+        WHERE p.known_outcome = 1$instr_extra
-GROUP BY v.id, resulttype
+        GROUP BY p.variant_id, r.resulttype
-ORDER BY variant, benchmark, resulttype;";
+    ) rc ON rc.variant_id = v.id
+    WHERE v.variant = '$experiment'
+) x
+GROUP BY benchmark, resulttype
+ORDER BY benchmark, $resulttype_order;";
 
     say $querystring;