@grafana/e2e: improvements (#25342)

* Minor changes

* Remove console.* logger plugin

... as it doesn't work in Electron

* Only open/close panel editor options and groups when state is inverted

... meaning, only open when closed and only close when open. This avoids unpredictable states, causing inconsistent results.

* Support for adding multiple datasources and dashboards

... and having them all auto-removed when tests are completed

* Avoid page errors when removing dashboards and datasources [keep?]

* Wait for chart data before saving panel

... so that everything is ready when returning to the dashboard

packages/grafana-e2e-selectors/src/selectors/pages.ts

@@ -6,6 +6,9 @@ export const Pages = {
     submit: 'Login button',
     skip: 'Skip change password button',
   },
+  Home: {
+    url: '/',
+  },
   DataSource: {
     name: 'Data source settings page name input field',
     delete: 'Data source settings page Delete button',

packages/grafana-e2e/cypress/plugins/index.js

@@ -2,21 +2,17 @@ const compareScreenshots = require('./compareScreenshots');
 const extendConfig = require('./extendConfig');
 const readProvisions = require('./readProvisions');
 const typescriptPreprocessor = require('./typescriptPreprocessor');
-const { install: installConsoleLogger } = require('cypress-log-to-output');
 
 module.exports = (on, config) => {
   on('file:preprocessor', typescriptPreprocessor);
   on('task', { compareScreenshots, readProvisions });
   on('task', {
-    // @todo remove
     log({ message, optional }) {
       optional ? console.log(message, optional) : console.log(message);
       return null;
     },
   });
 
-  installConsoleLogger(on);
-
   // Always extend with this library's config and return for diffing
   // @todo remove this when possible: https://github.com/cypress-io/cypress/issues/5674
   return extendConfig(config);

packages/grafana-e2e/cypress/support/commands.ts

@@ -11,7 +11,6 @@ Cypress.Commands.add('compareScreenshots', (config: CompareScreenshotsConfig | s
   });
 });
 
-// @todo remove
 Cypress.Commands.add('logToConsole', (message: string, optional?: any) => {
   cy.task('log', { message, optional });
 });

packages/grafana-e2e/package.json

@@ -50,7 +50,6 @@
     "blink-diff": "1.0.13",
     "commander": "5.0.0",
     "cypress": "^4.7.0",
-    "cypress-log-to-output": "^1.0.8",
     "execa": "4.0.0",
     "resolve-as-bin": "2.1.0",
     "ts-loader": "6.2.1",

packages/grafana-e2e/src/flows/addDashboard.ts

+import { DeleteDashboardConfig } from './deleteDashboard';
 import { e2e } from '../index';
 import { getDashboardUid } from '../support/url';
 
-export const addDashboard = () => {
-  e2e().logToConsole('Adding dashboard');
+export interface AddDashboardConfig {
+  title: string;
+}
+
+// @todo this actually returns type `Cypress.Chainable`
+export const addDashboard = (config?: Partial<AddDashboardConfig>): any => {
+  const fullConfig = {
+    title: `e2e-${Date.now()}`,
+    ...config,
+  } as AddDashboardConfig;
+
+  const { title } = fullConfig;
+
+  e2e().logToConsole('Adding dashboard with title:', title);
+
   e2e.pages.AddDashboard.visit();
 
-  const dashboardTitle = e2e.flows.saveNewDashboard();
-  e2e().logToConsole('Added dashboard with title:', dashboardTitle);
+  e2e.pages.Dashboard.Toolbar.toolbarItems('Save dashboard').click();
+
+  e2e.pages.SaveDashboardAsModal.newName()
+    .clear()
+    .type(title);
+  e2e.pages.SaveDashboardAsModal.save().click();
 
-  e2e()
+  e2e.flows.assertSuccessNotification();
+
+  e2e().logToConsole('Added dashboard with title:', title);
+
+  return e2e()
     .url()
     .then((url: string) => {
-      e2e.setScenarioContext({
-        lastAddedDashboard: dashboardTitle,
-        lastAddedDashboardUid: getDashboardUid(url),
+      const uid = getDashboardUid(url);
+
+      e2e.getScenarioContext().then(({ addedDashboards }: any) => {
+        e2e.setScenarioContext({
+          addedDashboards: [...addedDashboards, { title, uid } as DeleteDashboardConfig],
+        });
+      });
+
+      // @todo remove `wrap` when possible
+      return e2e().wrap({
+        config: fullConfig,
+        uid,
       });
     });
 };

packages/grafana-e2e/src/flows/addDataSource.ts

+import { DeleteDataSourceConfig } from './deleteDataSource';
 import { e2e } from '../index';
 import { fromBaseUrl, getDataSourceId } from '../support/url';
-import { setScenarioContext } from '../support/scenarioContext';
 
 export interface AddDataSourceConfig {
   checkHealth: boolean;
   expectedAlertMessage: string | RegExp;
   form: Function;
   name: string;
+  type: string;
 }
 
-const DEFAULT_ADD_DATA_SOURCE_CONFIG: AddDataSourceConfig = {
-  checkHealth: false,
-  expectedAlertMessage: 'Data source is working',
-  form: () => {},
-  name: 'TestData DB',
-};
+// @todo this actually returns type `Cypress.Chainable`
+export const addDataSource = (config?: Partial<AddDataSourceConfig>): any => {
+  const fullConfig = {
+    checkHealth: false,
+    expectedAlertMessage: 'Data source is working',
+    form: () => {},
+    name: `e2e-${Date.now()}`,
+    type: 'TestData DB',
+    ...config,
+  } as AddDataSourceConfig;
 
-export const addDataSource = (config?: Partial<AddDataSourceConfig>): string => {
-  const { checkHealth, expectedAlertMessage, form, name } = { ...DEFAULT_ADD_DATA_SOURCE_CONFIG, ...config };
+  const { checkHealth, expectedAlertMessage, form, name, type } = fullConfig;
 
   e2e().logToConsole('Adding data source with name:', name);
   e2e.pages.AddDataSource.visit();
-  e2e.pages.AddDataSource.dataSourcePlugins(name)
+  e2e.pages.AddDataSource.dataSourcePlugins(type)
     .scrollIntoView()
     .should('be.visible') // prevents flakiness
     .click();
 
-  const dataSourceName = `e2e-${Date.now()}`;
   e2e.pages.DataSource.name().clear();
-  e2e.pages.DataSource.name().type(dataSourceName);
+  e2e.pages.DataSource.name().type(name);
   form();
   e2e.pages.DataSource.saveAndTest().click();
   e2e.pages.DataSource.alert().should('exist');
   e2e.pages.DataSource.alertMessage().contains(expectedAlertMessage); // assertion
-  e2e().logToConsole('Added data source with name:', dataSourceName);
+  e2e().logToConsole('Added data source with name:', name);
 
-  if (checkHealth) {
-    e2e()
-      .url()
-      .then((url: string) => {
-        const dataSourceId = getDataSourceId(url);
+  return e2e()
+    .url()
+    .then((url: string) => {
+      const id = getDataSourceId(url);
 
-        setScenarioContext({
-          lastAddedDataSource: dataSourceName,
-          lastAddedDataSourceId: dataSourceId,
+      e2e.getScenarioContext().then(({ addedDataSources }: any) => {
+        e2e.setScenarioContext({
+          addedDataSources: [...addedDataSources, { id, name } as DeleteDataSourceConfig],
         });
+      });
 
-        const healthUrl = fromBaseUrl(`/api/datasources/${dataSourceId}/health`);
+      if (checkHealth) {
+        const healthUrl = fromBaseUrl(`/api/datasources/${id}/health`);
         e2e().logToConsole(`Fetching ${healthUrl}`);
         e2e()
           .request(healthUrl)
           .its('body')
           .should('have.property', 'status')
           .and('eq', 'OK');
+      }
+
+      // @todo remove `wrap` when possible
+      return e2e().wrap({
+        config: fullConfig,
+        id,
       });
-  } else {
-    setScenarioContext({
-      lastAddedDataSource: dataSourceName,
     });
-  }
-
-  return dataSourceName;
 };

packages/grafana-e2e/src/flows/addPanel.ts

 import { e2e } from '../index';
+import { getLocalStorage, requireLocalStorage } from '../support/localStorage';
 import { getScenarioContext } from '../support/scenarioContext';
 
 export interface AddPanelConfig {
@@ -7,6 +8,7 @@ export interface AddPanelConfig {
   queriesForm: (config: AddPanelConfig) => void;
   panelTitle: string;
   visualizationName: string;
+  waitForChartData: boolean;
 }
 
 // @todo this actually returns type `Cypress.Chainable`
@@ -18,6 +20,7 @@ export const addPanel = (config?: Partial<AddPanelConfig>): any =>
       panelTitle: `e2e-${Date.now()}`,
       queriesForm: () => {},
       visualizationName: 'Table',
+      waitForChartData: true,
       ...config,
     } as AddPanelConfig;
 
@@ -31,42 +34,98 @@ export const addPanel = (config?: Partial<AddPanelConfig>): any =>
       .get('.ds-picker')
       .click()
       .contains('[id^="react-select-"][id*="-option-"]', dataSourceName)
+      .scrollIntoView()
       .click();
 
+    isOptionsOpen().then((isOpen: any) => {
+      if (!isOpen) {
+        toggleOptions();
+      }
+    });
+
+    openOptionsGroup('settings');
     getOptionsGroup('settings')
       .find('[value="Panel Title"]')
+      .scrollIntoView()
       .clear()
       .type(panelTitle);
-    toggleOptionsGroup('settings');
+    closeOptionsGroup('settings');
 
-    toggleOptionsGroup('type');
+    openOptionsGroup('type');
     e2e()
       .get(`[aria-label="Plugin visualization item ${visualizationName}"]`)
       .scrollIntoView()
       .click();
-    toggleOptionsGroup('type');
+    closeOptionsGroup('type');
+
+    e2e().server();
+    e2e()
+      .route('POST', '/api/ds/query')
+      .as('chartData');
 
     queriesForm(fullConfig);
 
+    e2e().wait('@chartData');
+
     // @todo enable when plugins have this implemented
     //e2e.components.QueryEditorRow.actionButton('Disable/enable query').click();
     //e2e.components.Panels.Panel.containerByTitle(panelTitle).find('.panel-content').contains('No data');
     //e2e.components.QueryEditorRow.actionButton('Disable/enable query').click();
 
-    e2e.components.PanelEditor.OptionsPane.close().click();
+    isOptionsOpen().then((isOpen: any) => {
+      if (isOpen) {
+        toggleOptions();
+      }
+    });
 
     e2e()
       .get('button[title="Apply changes and go back to dashboard"]')
       .click();
 
     // @todo remove `wrap` when possible
-    return e2e().wrap(fullConfig);
+    return e2e().wrap({ config: fullConfig });
+  });
+
+// @todo this actually returns type `Cypress.Chainable`
+const closeOptionsGroup = (name: string): any =>
+  isOptionsGroupOpen(name).then((isOpen: any) => {
+    if (isOpen) {
+      toggleOptionsGroup(name);
+    }
   });
 
 const getOptionsGroup = (name: string) => e2e().get(`.options-group:has([aria-label="Options group Panel ${name}"])`);
 
+// @todo this actually returns type `Cypress.Chainable`
+const isOptionsGroupOpen = (name: string): any =>
+  requireLocalStorage(`grafana.dashboard.editor.ui.optionGroup[Panel ${name}]`).then(({ defaultToClosed }: any) => {
+    // @todo remove `wrap` when possible
+    return e2e().wrap(!defaultToClosed);
+  });
+
+// @todo this actually returns type `Cypress.Chainable`
+const isOptionsOpen = (): any =>
+  getLocalStorage('grafana.dashboard.editor.ui').then((data: any) => {
+    if (data) {
+      // @todo remove `wrap` when possible
+      return e2e().wrap(data.isPanelOptionsVisible);
+    } else {
+      // @todo remove `wrap` when possible
+      return e2e().wrap(true);
+    }
+  });
+
+// @todo this actually returns type `Cypress.Chainable`
+const openOptionsGroup = (name: string): any =>
+  isOptionsGroupOpen(name).then((isOpen: any) => {
+    if (!isOpen) {
+      toggleOptionsGroup(name);
+    }
+  });
+
+const toggleOptions = () => e2e.components.PanelEditor.OptionsPane.close().click();
+
 const toggleOptionsGroup = (name: string) =>
   getOptionsGroup(name)
     .find('.editor-options-group-toggle')
-    .scrollIntoView()
     .click();

packages/grafana-e2e/src/flows/deleteDashboard.ts

 import { e2e } from '../index';
 import { fromBaseUrl } from '../support/url';
 
-export const deleteDashboard = (dashBoardUid: string) => {
-  e2e().logToConsole('Deleting dashboard with uid:', dashBoardUid);
-  e2e().request('DELETE', fromBaseUrl(`/api/dashboards/uid/${dashBoardUid}`));
+export interface DeleteDashboardConfig {
+  title: string;
+  uid: string;
+}
+
+export const deleteDashboard = ({ title, uid }: DeleteDashboardConfig) => {
+  e2e().logToConsole('Deleting dashboard with uid:', uid);
+
+  // Avoid dashboard page errors
+  e2e.pages.Home.visit();
+  e2e().request('DELETE', fromBaseUrl(`/api/dashboards/uid/${uid}`));
 
   /* https://github.com/cypress-io/cypress/issues/2831
-  Flows.openDashboard(dashboardName);
+  Flows.openDashboard(title);
 
   Pages.Dashboard.settings().click();
 
@@ -20,9 +28,17 @@ export const deleteDashboard = (dashBoardUid: string) => {
   Pages.Dashboards.dashboards().each(item => {
     const text = item.text();
     Cypress.log({ message: [text] });
-    if (text && text.indexOf(dashboardName) !== -1) {
-      expect(false).equals(true, `Dashboard ${dashboardName} was found although it was deleted.`);
+    if (text && text.indexOf(title) !== -1) {
+      expect(false).equals(true, `Dashboard ${title} was found although it was deleted.`);
     }
   });
- */
+  */
+
+  e2e.getScenarioContext().then(({ addedDashboards }: any) => {
+    e2e.setScenarioContext({
+      addedDashboards: addedDashboards.filter((dashboard: DeleteDashboardConfig) => {
+        return dashboard.title !== title && dashboard.uid !== uid;
+      }),
+    });
+  });
 };

packages/grafana-e2e/src/flows/deleteDataSource.ts

 import { e2e } from '../index';
 import { fromBaseUrl } from '../support/url';
 
-export const deleteDataSource = (dataSourceName: string) => {
-  e2e().logToConsole('Deleting data source with name:', dataSourceName);
-  e2e().request('DELETE', fromBaseUrl(`/api/datasources/name/${dataSourceName}`));
+export interface DeleteDataSourceConfig {
+  id: string;
+  name: string;
+}
+
+export const deleteDataSource = ({ id, name }: DeleteDataSourceConfig) => {
+  e2e().logToConsole('Deleting data source with name:', name);
+
+  // Avoid datasources page errors
+  e2e.pages.Home.visit();
+  e2e().request('DELETE', fromBaseUrl(`/api/datasources/name/${name}`));
 
   /* https://github.com/cypress-io/cypress/issues/2831
   Pages.DataSources.visit();
-  Pages.DataSources.dataSources(dataSourceName).click();
+  Pages.DataSources.dataSources(name).click();
 
   Pages.DataSource.delete().click();
 
@@ -16,9 +24,17 @@ export const deleteDataSource = (dataSourceName: string) => {
   Pages.DataSources.visit();
   Pages.DataSources.dataSources().each(item => {
     const text = item.text();
-    if (text && text.indexOf(dataSourceName) !== -1) {
-      expect(false).equals(true, `Data source ${dataSourceName} was found although it was deleted.`);
+    if (text && text.indexOf(name) !== -1) {
+      expect(false).equals(true, `Data source ${name} was found although it was deleted.`);
     }
   });
- */
+  */
+
+  e2e.getScenarioContext().then(({ addedDataSources }: any) => {
+    e2e.setScenarioContext({
+      addedDataSources: addedDataSources.filter((dataSource: DeleteDataSourceConfig) => {
+        return dataSource.id !== id && dataSource.name !== name;
+      }),
+    });
+  });
 };

packages/grafana-e2e/src/flows/index.ts

@@ -7,7 +7,6 @@ import { deleteDataSource } from './deleteDataSource';
 import { login } from './login';
 import { openDashboard } from './openDashboard';
 import { saveDashboard } from './saveDashboard';
-import { saveNewDashboard } from './saveNewDashboard';
 import { openPanelMenuItem, PanelMenuItems } from './openPanelMenuItem';
 
 export const Flows = {
@@ -20,7 +19,6 @@ export const Flows = {
   login,
   openDashboard,
   saveDashboard,
-  saveNewDashboard,
   openPanelMenuItem,
   PanelMenuItems,
 };

packages/grafana-e2e/src/flows/saveNewDashboard.ts

-import { e2e } from '../index';
-
-export const saveNewDashboard = () => {
-  e2e.pages.Dashboard.Toolbar.toolbarItems('Save dashboard').click();
-
-  const dashboardTitle = `e2e-${Date.now()}`;
-  e2e.pages.SaveDashboardAsModal.newName().clear();
-  e2e.pages.SaveDashboardAsModal.newName().type(dashboardTitle);
-  e2e.pages.SaveDashboardAsModal.save().click();
-
-  e2e.flows.assertSuccessNotification();
-
-  return dashboardTitle;
-};

packages/grafana-e2e/src/support/index.ts

-export * from './types';
-export * from './selector';
+export * from './localStorage';
 export * from './scenarioContext';
+export * from './selector';
+export * from './types';

packages/grafana-e2e/src/support/localStorage.ts

+import { e2e } from '../index';
+
+// @todo this actually returns type `Cypress.Chainable`
+const get = (key: string): any =>
+  e2e()
+    .wrap({ getLocalStorage: () => localStorage.getItem(key) })
+    .invoke('getLocalStorage');
+
+// @todo this actually returns type `Cypress.Chainable`
+export const getLocalStorage = (key: string): any =>
+  get(key).then((value: any) => {
+    if (value === null) {
+      return value;
+    } else {
+      return JSON.parse(value);
+    }
+  });
+
+// @todo this actually returns type `Cypress.Chainable`
+export const requireLocalStorage = (key: string): any =>
+  get(key) // `getLocalStorage()` would turn 'null' into `null`
+    .should('not.equal', null)
+    .then((value: any) => JSON.parse(value as string));

packages/grafana-e2e/src/support/scenario.ts

@@ -34,14 +34,9 @@ export const e2eScenario = ({
       });
 
       afterEach(() => {
-        getScenarioContext().then(({ lastAddedDashboardUid, lastAddedDataSource }: any) => {
-          if (lastAddedDashboardUid) {
-            Flows.deleteDashboard(lastAddedDashboardUid);
-          }
-
-          if (lastAddedDataSource) {
-            Flows.deleteDataSource(lastAddedDataSource);
-          }
+        getScenarioContext().then(({ addedDashboards, addedDataSources }: any) => {
+          addedDashboards.forEach((dashboard: any) => Flows.deleteDashboard(dashboard));
+          addedDataSources.forEach((dataSource: any) => Flows.deleteDataSource(dataSource));
         });
       });
 

packages/grafana-e2e/src/support/scenarioContext.ts

 import { e2e } from '../index';
+import { DeleteDashboardConfig } from '../flows/deleteDashboard';
+import { DeleteDataSourceConfig } from '../flows/deleteDataSource';
 
 export interface ScenarioContext {
-  lastAddedDashboard: string;
+  addedDashboards: DeleteDashboardConfig[];
+  addedDataSources: DeleteDataSourceConfig[];
+  lastAddedDashboard: string; // @todo rename to `lastAddedDashboardTitle`
   lastAddedDashboardUid: string;
-  lastAddedDataSource: string;
+  lastAddedDataSource: string; // @todo rename to `lastAddedDataSourceName`
   lastAddedDataSourceId: string;
   [key: string]: any;
 }
 
 const scenarioContext: ScenarioContext = {
-  lastAddedDashboard: '',
-  lastAddedDashboardUid: '',
-  lastAddedDataSource: '',
-  lastAddedDataSourceId: '',
+  addedDashboards: [],
+  addedDataSources: [],
+  get lastAddedDashboard() {
+    return lastProperty(this.addedDashboards, 'title');
+  },
+  get lastAddedDashboardUid() {
+    return lastProperty(this.addedDashboards, 'uid');
+  },
+  get lastAddedDataSource() {
+    return lastProperty(this.addedDataSources, 'name');
+  },
+  get lastAddedDataSourceId() {
+    return lastProperty(this.addedDataSources, 'id');
+  },
 };
 
+const lastProperty = <T extends DeleteDashboardConfig | DeleteDataSourceConfig, K extends keyof T>(
+  items: T[],
+  key: K
+) => items[items.length - 1]?.[key] ?? '';
+
 // @todo this actually returns type `Cypress.Chainable`
 export const getScenarioContext = (): any =>
   e2e()