Merge pull request #12959 from grafana/11890_variables

Fix: Template variables are executing excessive queries

public/app/core/utils/dag.test.ts

+import { Graph } from './dag';
+
+describe('Directed acyclic graph', () => {
+  describe('Given a graph with nodes with different links in between them', () => {
+    let dag = new Graph();
+    let nodeA = dag.createNode('A');
+    let nodeB = dag.createNode('B');
+    let nodeC = dag.createNode('C');
+    let nodeD = dag.createNode('D');
+    let nodeE = dag.createNode('E');
+    let nodeF = dag.createNode('F');
+    let nodeG = dag.createNode('G');
+    let nodeH = dag.createNode('H');
+    let nodeI = dag.createNode('I');
+    dag.link([nodeB, nodeC, nodeD, nodeE, nodeF, nodeG, nodeH], nodeA);
+    dag.link([nodeC, nodeD, nodeE, nodeF, nodeI], nodeB);
+    dag.link([nodeD, nodeE, nodeF, nodeG], nodeC);
+    dag.link([nodeE, nodeF], nodeD);
+    dag.link([nodeF, nodeG], nodeE);
+    //printGraph(dag);
+
+    it('nodes in graph should have expected edges', () => {
+      expect(nodeA.inputEdges).toHaveLength(7);
+      expect(nodeA.outputEdges).toHaveLength(0);
+      expect(nodeA.edges).toHaveLength(7);
+
+      expect(nodeB.inputEdges).toHaveLength(5);
+      expect(nodeB.outputEdges).toHaveLength(1);
+      expect(nodeB.edges).toHaveLength(6);
+
+      expect(nodeC.inputEdges).toHaveLength(4);
+      expect(nodeC.outputEdges).toHaveLength(2);
+      expect(nodeC.edges).toHaveLength(6);
+
+      expect(nodeD.inputEdges).toHaveLength(2);
+      expect(nodeD.outputEdges).toHaveLength(3);
+      expect(nodeD.edges).toHaveLength(5);
+
+      expect(nodeE.inputEdges).toHaveLength(2);
+      expect(nodeE.outputEdges).toHaveLength(4);
+      expect(nodeE.edges).toHaveLength(6);
+
+      expect(nodeF.inputEdges).toHaveLength(0);
+      expect(nodeF.outputEdges).toHaveLength(5);
+      expect(nodeF.edges).toHaveLength(5);
+
+      expect(nodeG.inputEdges).toHaveLength(0);
+      expect(nodeG.outputEdges).toHaveLength(3);
+      expect(nodeG.edges).toHaveLength(3);
+
+      expect(nodeH.inputEdges).toHaveLength(0);
+      expect(nodeH.outputEdges).toHaveLength(1);
+      expect(nodeH.edges).toHaveLength(1);
+
+      expect(nodeI.inputEdges).toHaveLength(0);
+      expect(nodeI.outputEdges).toHaveLength(1);
+      expect(nodeI.edges).toHaveLength(1);
+
+      expect(nodeA.getEdgeFrom(nodeB)).not.toBeUndefined();
+      expect(nodeB.getEdgeTo(nodeA)).not.toBeUndefined();
+    });
+
+    it('when optimizing input edges for node A should return node B and H', () => {
+      const actual = nodeA.getOptimizedInputEdges().map(e => e.inputNode);
+      expect(actual).toHaveLength(2);
+      expect(actual).toEqual(expect.arrayContaining([nodeB, nodeH]));
+    });
+
+    it('when optimizing input edges for node B should return node C', () => {
+      const actual = nodeB.getOptimizedInputEdges().map(e => e.inputNode);
+      expect(actual).toHaveLength(2);
+      expect(actual).toEqual(expect.arrayContaining([nodeC, nodeI]));
+    });
+
+    it('when optimizing input edges for node C should return node D', () => {
+      const actual = nodeC.getOptimizedInputEdges().map(e => e.inputNode);
+      expect(actual).toHaveLength(1);
+      expect(actual).toEqual(expect.arrayContaining([nodeD]));
+    });
+
+    it('when optimizing input edges for node D should return node E', () => {
+      const actual = nodeD.getOptimizedInputEdges().map(e => e.inputNode);
+      expect(actual).toHaveLength(1);
+      expect(actual).toEqual(expect.arrayContaining([nodeE]));
+    });
+
+    it('when optimizing input edges for node E should return node F and G', () => {
+      const actual = nodeE.getOptimizedInputEdges().map(e => e.inputNode);
+      expect(actual).toHaveLength(2);
+      expect(actual).toEqual(expect.arrayContaining([nodeF, nodeG]));
+    });
+
+    it('when optimizing input edges for node F should return zero nodes', () => {
+      const actual = nodeF.getOptimizedInputEdges();
+      expect(actual).toHaveLength(0);
+    });
+
+    it('when optimizing input edges for node G should return zero nodes', () => {
+      const actual = nodeG.getOptimizedInputEdges();
+      expect(actual).toHaveLength(0);
+    });
+
+    it('when optimizing input edges for node H should return zero nodes', () => {
+      const actual = nodeH.getOptimizedInputEdges();
+      expect(actual).toHaveLength(0);
+    });
+  });
+});

public/app/core/utils/dag.ts

+export class Edge {
+  inputNode: Node;
+  outputNode: Node;
+
+  _linkTo(node, direction) {
+    if (direction <= 0) {
+      node.inputEdges.push(this);
+    }
+
+    if (direction >= 0) {
+      node.outputEdges.push(this);
+    }
+
+    node.edges.push(this);
+  }
+
+  link(inputNode: Node, outputNode: Node) {
+    this.unlink();
+    this.inputNode = inputNode;
+    this.outputNode = outputNode;
+
+    this._linkTo(inputNode, 1);
+    this._linkTo(outputNode, -1);
+    return this;
+  }
+
+  unlink() {
+    let pos;
+    let inode = this.inputNode;
+    let onode = this.outputNode;
+
+    if (!(inode && onode)) {
+      return;
+    }
+
+    pos = inode.edges.indexOf(this);
+    if (pos > -1) {
+      inode.edges.splice(pos, 1);
+    }
+
+    pos = onode.edges.indexOf(this);
+    if (pos > -1) {
+      onode.edges.splice(pos, 1);
+    }
+
+    pos = inode.outputEdges.indexOf(this);
+    if (pos > -1) {
+      inode.outputEdges.splice(pos, 1);
+    }
+
+    pos = onode.inputEdges.indexOf(this);
+    if (pos > -1) {
+      onode.inputEdges.splice(pos, 1);
+    }
+
+    this.inputNode = null;
+    this.outputNode = null;
+  }
+}
+
+export class Node {
+  name: string;
+  edges: Edge[];
+  inputEdges: Edge[];
+  outputEdges: Edge[];
+
+  constructor(name: string) {
+    this.name = name;
+    this.edges = [];
+    this.inputEdges = [];
+    this.outputEdges = [];
+  }
+
+  getEdgeFrom(from: string | Node): Edge {
+    if (!from) {
+      return null;
+    }
+
+    if (typeof from === 'object') {
+      return this.inputEdges.find(e => e.inputNode.name === from.name);
+    }
+
+    return this.inputEdges.find(e => e.inputNode.name === from);
+  }
+
+  getEdgeTo(to: string | Node): Edge {
+    if (!to) {
+      return null;
+    }
+
+    if (typeof to === 'object') {
+      return this.outputEdges.find(e => e.outputNode.name === to.name);
+    }
+
+    return this.outputEdges.find(e => e.outputNode.name === to);
+  }
+
+  getOptimizedInputEdges(): Edge[] {
+    let toBeRemoved = [];
+    this.inputEdges.forEach(e => {
+      let inputEdgesNodes = e.inputNode.inputEdges.map(e => e.inputNode);
+
+      inputEdgesNodes.forEach(n => {
+        let edgeToRemove = n.getEdgeTo(this.name);
+        if (edgeToRemove) {
+          toBeRemoved.push(edgeToRemove);
+        }
+      });
+    });
+
+    return this.inputEdges.filter(e => toBeRemoved.indexOf(e) === -1);
+  }
+}
+
+export class Graph {
+  nodes = {};
+
+  constructor() {}
+
+  createNode(name: string): Node {
+    const n = new Node(name);
+    this.nodes[name] = n;
+    return n;
+  }
+
+  createNodes(names: string[]): Node[] {
+    let nodes = [];
+    names.forEach(name => {
+      nodes.push(this.createNode(name));
+    });
+    return nodes;
+  }
+
+  link(input: string | string[] | Node | Node[], output: string | string[] | Node | Node[]): Edge[] {
+    let inputArr = [];
+    let outputArr = [];
+    let inputNodes = [];
+    let outputNodes = [];
+
+    if (input instanceof Array) {
+      inputArr = input;
+    } else {
+      inputArr = [input];
+    }
+
+    if (output instanceof Array) {
+      outputArr = output;
+    } else {
+      outputArr = [output];
+    }
+
+    for (let n = 0; n < inputArr.length; n++) {
+      const i = inputArr[n];
+      if (typeof i === 'string') {
+        inputNodes.push(this.getNode(i));
+      } else {
+        inputNodes.push(i);
+      }
+    }
+
+    for (let n = 0; n < outputArr.length; n++) {
+      const i = outputArr[n];
+      if (typeof i === 'string') {
+        outputNodes.push(this.getNode(i));
+      } else {
+        outputNodes.push(i);
+      }
+    }
+
+    let edges = [];
+    inputNodes.forEach(input => {
+      outputNodes.forEach(output => {
+        edges.push(this.createEdge().link(input, output));
+      });
+    });
+    return edges;
+  }
+
+  createEdge(): Edge {
+    return new Edge();
+  }
+
+  getNode(name: string): Node {
+    return this.nodes[name];
+  }
+}
+
+export const printGraph = (g: Graph) => {
+  Object.keys(g.nodes).forEach(name => {
+    const n = g.nodes[name];
+    let outputEdges = n.outputEdges.map(e => e.outputNode.name).join(', ');
+    if (!outputEdges) {
+      outputEdges = '<none>';
+    }
+    let inputEdges = n.inputEdges.map(e => e.inputNode.name).join(', ');
+    if (!inputEdges) {
+      inputEdges = '<none>';
+    }
+    console.log(`${n.name}:\n - links to:   ${outputEdges}\n - links from: ${inputEdges}`);
+  });
+};

public/app/features/templating/variable_srv.ts

@@ -2,6 +2,7 @@ import angular from 'angular';
 import _ from 'lodash';
 import coreModule from 'app/core/core_module';
 import { variableTypes } from './variable';
+import { Graph } from 'app/core/utils/dag';
 
 export class VariableSrv {
   dashboard: any;
@@ -120,16 +121,13 @@ export class VariableSrv {
       return this.$q.when();
     }
 
-    // cascade updates to variables that use this variable
-    var promises = _.map(this.variables, otherVariable => {
-      if (otherVariable === variable) {
-        return;
-      }
-
-      if (otherVariable.dependsOn(variable)) {
-        return this.updateOptions(otherVariable);
-      }
-    });
+    const g = this.createGraph();
+    const promises = g
+      .getNode(variable.name)
+      .getOptimizedInputEdges()
+      .map(e => {
+        return this.updateOptions(this.variables.find(v => v.name === e.inputNode.name));
+      });
 
     return this.$q.all(promises).then(() => {
       if (emitChangeEvents) {
@@ -288,6 +286,26 @@ export class VariableSrv {
     filter.operator = options.operator;
     this.variableUpdated(variable, true);
   }
+
+  createGraph() {
+    let g = new Graph();
+
+    this.variables.forEach(v1 => {
+      g.createNode(v1.name);
+
+      this.variables.forEach(v2 => {
+        if (v1 === v2) {
+          return;
+        }
+
+        if (v1.dependsOn(v2)) {
+          g.link(v1.name, v2.name);
+        }
+      });
+    });
+
+    return g;
+  }
 }
 
 coreModule.service('variableSrv', VariableSrv);