@grafana/e2e: improvements (#27170)

* Minor changes

* Changed default panel screenshot name

* Removed default visualization type from `addPanel` config

... which, unless defined, will use the app's default (graph). This is the new approach for plugins which supports annotations.

* Select timezone from dashboard toolbar instead of settings

... and refactored `setDashboardTimeRange` for reuse on other pages via a now more generalized `setTimeRange`

* Added optional annotations to `addDashboard` config

* Added `explore` flow

… which reuses `configurePanel` which is very similar

packages/grafana-e2e/src/flows/addDashboard.ts

-import { v4 as uuidv4 } from 'uuid';
-import { DashboardTimeRangeConfig, setDashboardTimeRange } from './setDashboardTimeRange';
 import { DeleteDashboardConfig } from './deleteDashboard';
 import { e2e } from '../index';
 import { getDashboardUid } from '../support/url';
-import { selectOption } from './selectOption';
+import { setDashboardTimeRange, TimeRangeConfig } from './setDashboardTimeRange';
+import { v4 as uuidv4 } from 'uuid';
+
+export interface AddAnnotationConfig {
+  dataSource: string;
+  name: string;
+  sources?: string;
+  tags?: string;
+}
 
 export interface AddDashboardConfig {
-  timeRange: DashboardTimeRangeConfig;
-  timezone: string;
+  annotations: AddAnnotationConfig[];
+  timeRange: TimeRangeConfig;
   title: string;
   variables: Array<Partial<AddVariableConfig>>;
 }
@@ -15,7 +21,7 @@ export interface AddDashboardConfig {
 export interface AddVariableConfig {
   constantValue?: string;
   dataSource?: string;
-  hide?: string;
+  hide: string;
   label?: string;
   name: string;
   query?: string;
@@ -27,40 +33,38 @@ export interface AddVariableConfig {
 // @todo this actually returns type `Cypress.Chainable`
 export const addDashboard = (config?: Partial<AddDashboardConfig>): any => {
   const fullConfig = {
+    annotations: [],
+    title: `e2e-${uuidv4()}`,
+    variables: [],
+    ...config,
     timeRange: {
       from: '2020-01-01 00:00:00',
       to: '2020-01-01 06:00:00',
+      zone: 'Coordinated Universal Time',
+      ...config?.timeRange,
     },
-    timezone: 'Coordinated Universal Time',
-    title: `e2e-${uuidv4()}`,
-    variables: [],
-    ...config,
   } as AddDashboardConfig;
 
-  const { timeRange, timezone, title, variables } = fullConfig;
+  const { annotations, timeRange, title, variables } = fullConfig;
 
   e2e().logToConsole('Adding dashboard with title:', title);
 
   e2e.pages.AddDashboard.visit();
 
+  if (annotations.length > 0 || variables.length > 0) {
     e2e.pages.Dashboard.Toolbar.toolbarItems('Dashboard settings').click();
-
-  // @todo use the time range picker's time zone control
-  selectOption(e2e.pages.Dashboard.Settings.General.timezone(), timezone);
-
+    addAnnotations(annotations);
     addVariables(variables);
-
     e2e.components.BackButton.backArrow().click();
+  }
 
   setDashboardTimeRange(timeRange);
 
   e2e.pages.Dashboard.Toolbar.toolbarItems('Save dashboard').click();
-
   e2e.pages.SaveDashboardAsModal.newName()
     .clear()
     .type(title);
   e2e.pages.SaveDashboardAsModal.save().click();
-
   e2e.flows.assertSuccessNotification();
 
   e2e().logToConsole('Added dashboard with title:', title);
@@ -87,13 +91,74 @@ export const addDashboard = (config?: Partial<AddDashboardConfig>): any => {
     });
 };
 
+const addAnnotation = (config: AddAnnotationConfig, isFirst: boolean) => {
+  if (isFirst) {
+    e2e.pages.Dashboard.Settings.Annotations.List.addAnnotationCTA().click();
+  } else {
+    // @todo add to e2e-selectors and `aria-label`
+    e2e()
+      .contains('.btn', 'New')
+      .click();
+  }
+
+  const { dataSource, name, sources, tags } = config;
+
+  // @todo add to e2e-selectors and `aria-label`
+  e2e()
+    .contains('.gf-form', 'Data source')
+    .find('select')
+    .select(dataSource);
+
+  // @todo add to e2e-selectors and `aria-label`
+  e2e()
+    .contains('.gf-form', 'Name')
+    .find('input')
+    .type(name);
+
+  if (sources) {
+    // @todo add to e2e-selectors and `aria-label`
+    e2e()
+      .contains('.gf-form', 'Sources')
+      .find('input')
+      .type(sources);
+  }
+
+  if (tags) {
+    // @todo add to e2e-selectors and `aria-label`
+    e2e()
+      .contains('.gf-form', 'Tags')
+      .find('input')
+      .type(tags);
+  }
+
+  // @todo add to e2e-selectors and `aria-label`
+  e2e()
+    .contains('.btn', 'Add')
+    .click();
+};
+
+// @todo this actually returns type `Cypress.Chainable`
+const addAnnotations = (configs: AddAnnotationConfig[]): any => {
+  if (configs.length > 0) {
+    e2e.pages.Dashboard.Settings.General.sectionItems('Annotations').click();
+  }
+
+  return configs.map((config, i) => addAnnotation(config, i === 0));
+};
+
+export const VARIABLE_HIDE_LABEL = 'Label';
+export const VARIABLE_HIDE_NOTHING = '';
+export const VARIABLE_HIDE_VARIABLE = 'Variable';
+
 export const VARIABLE_TYPE_AD_HOC_FILTERS = 'Ad hoc filters';
 export const VARIABLE_TYPE_CONSTANT = 'Constant';
 export const VARIABLE_TYPE_DATASOURCE = 'Datasource';
 export const VARIABLE_TYPE_QUERY = 'Query';
 
+// @todo this actually returns type `Cypress.Chainable`
 const addVariable = (config: Partial<AddVariableConfig>, isFirst: boolean): any => {
   const fullConfig = {
+    hide: VARIABLE_HIDE_NOTHING,
     type: VARIABLE_TYPE_QUERY,
     ...config,
   } as AddVariableConfig;
@@ -106,7 +171,13 @@ const addVariable = (config: Partial<AddVariableConfig>, isFirst: boolean): any
 
   const { constantValue, dataSource, hide, label, name, query, regex, type } = fullConfig;
 
-  if (hide) {
+  // This field is key to many reactive changes
+  if (type !== VARIABLE_TYPE_QUERY) {
+    e2e.pages.Dashboard.Settings.Variables.Edit.General.generalTypeSelect().select(type);
+  }
+
+  // Avoid '', which is an accepted value
+  if (hide !== undefined) {
     e2e.pages.Dashboard.Settings.Variables.Edit.General.generalHideSelect().select(hide);
   }
 
@@ -116,10 +187,6 @@ const addVariable = (config: Partial<AddVariableConfig>, isFirst: boolean): any
 
   e2e.pages.Dashboard.Settings.Variables.Edit.General.generalNameInput().type(name);
 
-  if (type !== VARIABLE_TYPE_QUERY) {
-    e2e.pages.Dashboard.Settings.Variables.Edit.General.generalTypeSelect().select(type);
-  }
-
   if (
     dataSource &&
     (type === VARIABLE_TYPE_AD_HOC_FILTERS || type === VARIABLE_TYPE_DATASOURCE || type === VARIABLE_TYPE_QUERY)
@@ -141,11 +208,26 @@ const addVariable = (config: Partial<AddVariableConfig>, isFirst: boolean): any
     }
   }
 
+  // Avoid flakiness
+  e2e()
+    .focused()
+    .blur();
+  e2e()
+    .contains('.gf-form-group', 'Preview of values')
+    .within(() => {
+      if (type === VARIABLE_TYPE_CONSTANT) {
+        e2e()
+          .root()
+          .contains(constantValue as string);
+      }
+    });
+
   e2e.pages.Dashboard.Settings.Variables.Edit.General.addButton().click();
 
   return fullConfig;
 };
 
+// @todo this actually returns type `Cypress.Chainable`
 const addVariables = (configs: Array<Partial<AddVariableConfig>>): any => {
   if (configs.length > 0) {
     e2e.pages.Dashboard.Settings.General.sectionItems('Variables').click();

packages/grafana-e2e/src/flows/addDataSource.ts

-import { v4 as uuidv4 } from 'uuid';
 import { DeleteDataSourceConfig } from './deleteDataSource';
 import { e2e } from '../index';
 import { fromBaseUrl, getDataSourceId } from '../support/url';
+import { v4 as uuidv4 } from 'uuid';
 
 export interface AddDataSourceConfig {
   basicAuth: boolean;

packages/grafana-e2e/src/flows/addPanel.ts

-import { v4 as uuidv4 } from 'uuid';
 import { configurePanel, ConfigurePanelConfig } from './configurePanel';
 import { getScenarioContext } from '../support/scenarioContext';
+import { v4 as uuidv4 } from 'uuid';
 
+// @todo `Omit` 'isExplore'
 export interface AddPanelConfig extends ConfigurePanelConfig {
   dataSourceName: string;
   queriesForm: (config: AddPanelConfig) => void;
-  panelTitle: string;
   visualizationName: string;
 }
 
@@ -17,7 +17,6 @@ export const addPanel = (config?: Partial<AddPanelConfig>): any =>
       {
         dataSourceName: lastAddedDataSource,
         panelTitle: `e2e-${uuidv4()}`,
-        visualizationName: 'Table',
         ...config,
       } as AddPanelConfig,
       false

packages/grafana-e2e/src/flows/configurePanel.ts

@@ -2,6 +2,8 @@ import { e2e } from '../index';
 import { getLocalStorage, requireLocalStorage } from '../support/localStorage';
 import { getScenarioContext } from '../support/scenarioContext';
 import { selectOption } from './selectOption';
+import { setDashboardTimeRange } from './setDashboardTimeRange';
+import { setTimeRange, TimeRangeConfig } from './setTimeRange';
 
 export interface ConfigurePanelConfig {
   chartData: {
@@ -10,17 +12,19 @@ export interface ConfigurePanelConfig {
   };
   dashboardUid: string;
   dataSourceName?: string;
+  isExplore: boolean;
   matchScreenshot: boolean;
   queriesForm?: (config: any) => void;
   panelTitle: string;
   screenshotName: string;
+  timeRange?: TimeRangeConfig;
   visitDashboardAtStart: boolean; // @todo remove when possible
   visualizationName?: string;
 }
 
 // @todo improve config input/output: https://stackoverflow.com/a/63507459/923745
 // @todo this actually returns type `Cypress.Chainable`
-export const configurePanel = (config: Partial<ConfigurePanelConfig>, edit: boolean): any =>
+export const configurePanel = (config: Partial<ConfigurePanelConfig>, isEdit: boolean): any =>
   getScenarioContext().then(({ lastAddedDashboardUid }: any) => {
     const fullConfig = {
       chartData: {
@@ -28,9 +32,10 @@ export const configurePanel = (config: Partial<ConfigurePanelConfig>, edit: bool
         route: '/api/ds/query',
       },
       dashboardUid: lastAddedDashboardUid,
+      isExplore: false,
       matchScreenshot: false,
       saveDashboard: true,
-      screenshotName: 'chart',
+      screenshotName: 'panel-visualization',
       visitDashboardAtStart: true,
       ...config,
     } as ConfigurePanelConfig;
@@ -39,25 +44,39 @@ export const configurePanel = (config: Partial<ConfigurePanelConfig>, edit: bool
       chartData,
       dashboardUid,
       dataSourceName,
+      isExplore,
       matchScreenshot,
       panelTitle,
       queriesForm,
       screenshotName,
+      timeRange,
       visitDashboardAtStart,
       visualizationName,
     } = fullConfig;
 
+    if (isExplore) {
+      e2e.pages.Explore.visit();
+    } else {
       if (visitDashboardAtStart) {
         e2e.flows.openDashboard({ uid: dashboardUid });
       }
 
-    if (edit) {
+      if (isEdit) {
         e2e.components.Panels.Panel.title(panelTitle).click();
         e2e.components.Panels.Panel.headerItems('Edit').click();
       } else {
         e2e.pages.Dashboard.Toolbar.toolbarItems('Add panel').click();
         e2e.pages.AddDashboard.addNewPanel().click();
       }
+    }
+
+    if (timeRange) {
+      if (isExplore) {
+        e2e.pages.Explore.Toolbar.navBar().within(() => setTimeRange(timeRange));
+      } else {
+        setDashboardTimeRange(timeRange);
+      }
+    }
 
     e2e().server();
 
@@ -76,8 +95,9 @@ export const configurePanel = (config: Partial<ConfigurePanelConfig>, edit: bool
 
     e2e().wait('@chartData');
 
+    if (!isExplore) {
       // `panelTitle` is needed to edit the panel, and unlikely to have its value changed at that point
-    const changeTitle = panelTitle && !edit;
+      const changeTitle = panelTitle && !isEdit;
 
       if (changeTitle || visualizationName) {
         openOptions();
@@ -89,7 +109,6 @@ export const configurePanel = (config: Partial<ConfigurePanelConfig>, edit: bool
             .scrollIntoView()
             .clear()
             .type(panelTitle);
-        closeOptionsGroup('settings');
         }
 
         if (visualizationName) {
@@ -97,18 +116,25 @@ export const configurePanel = (config: Partial<ConfigurePanelConfig>, edit: bool
           e2e.components.PluginVisualization.item(visualizationName)
             .scrollIntoView()
             .click();
-        closeOptionsGroup('type');
         }
 
+        // Consistently closed
+        closeOptionsGroup('settings');
+        closeOptionsGroup('type');
         closeOptions();
       } else {
-      // Options are consistently closed
+        // Consistently closed
         closeOptions();
       }
+    }
 
     if (queriesForm) {
       queriesForm(fullConfig);
       e2e().wait('@chartData');
+
+      // Wait for a possible complex visualization to render (or something related, as this isn't necessary on the dashboard page)
+      // Can't assert that its HTML changed because a new query could produce the same results
+      e2e().wait(1000);
     }
 
     // @todo enable when plugins have this implemented
@@ -118,13 +144,20 @@ export const configurePanel = (config: Partial<ConfigurePanelConfig>, edit: bool
     //e2e.components.QueryEditorRow.actionButton('Disable/enable query').click();
     //e2e().wait('@chartData');
 
+    if (!isExplore) {
       e2e()
         .get('button[title="Apply changes and go back to dashboard"]')
         .click();
-
       e2e()
         .url()
         .should('include', `/d/${dashboardUid}`);
+    }
+
+    // Avoid annotations flakiness
+    e2e()
+      .get('.refresh-picker-buttons .btn')
+      .first()
+      .click();
 
     e2e().wait('@chartData');
 
@@ -132,9 +165,15 @@ export const configurePanel = (config: Partial<ConfigurePanelConfig>, edit: bool
     e2e().wait(500);
 
     if (matchScreenshot) {
-      e2e.components.Panels.Panel.containerByTitle(panelTitle)
-        .find('.panel-content')
-        .screenshot(screenshotName);
+      let visualization;
+
+      if (isExplore) {
+        visualization = e2e.pages.Explore.General.graph();
+      } else {
+        visualization = e2e.components.Panels.Panel.containerByTitle(panelTitle).find('.panel-content');
+      }
+
+      visualization.scrollIntoView().screenshot(screenshotName);
       e2e().compareScreenshots(screenshotName);
     }
 
@@ -199,3 +238,20 @@ const toggleOptionsGroup = (name: string) =>
   getOptionsGroup(name)
     .find('.editor-options-group-toggle')
     .click();
+
+export const VISUALIZATION_ALERT_LIST = 'Alert list';
+export const VISUALIZATION_BAR_GAUGE = 'Bar gauge';
+export const VISUALIZATION_CLOCK = 'Clock';
+export const VISUALIZATION_DASHBOARD_LIST = 'Dashboard list';
+export const VISUALIZATION_GAUGE = 'Gauge';
+export const VISUALIZATION_GRAPH = 'Graph';
+export const VISUALIZATION_HEAT_MAP = 'Heatmap';
+export const VISUALIZATION_LOGS = 'Logs';
+export const VISUALIZATION_NEWS = 'News';
+export const VISUALIZATION_PIE_CHART = 'Pie Chart';
+export const VISUALIZATION_PLUGIN_LIST = 'Plugin list';
+export const VISUALIZATION_POLYSTAT = 'Polystat';
+export const VISUALIZATION_STAT = 'Stat';
+export const VISUALIZATION_TABLE = 'Table';
+export const VISUALIZATION_TEXT = 'Text';
+export const VISUALIZATION_WORLD_MAP = 'Worldmap Panel';

packages/grafana-e2e/src/flows/editPanel.ts

 import { configurePanel, ConfigurePanelConfig } from './configurePanel';
 
+// @todo `Omit` 'isExplore'
 export interface EditPanelConfig extends ConfigurePanelConfig {
   queriesForm?: (config: EditPanelConfig) => void;
 }

packages/grafana-e2e/src/flows/explore.ts

+import { configurePanel, ConfigurePanelConfig } from './configurePanel';
+import { getScenarioContext } from '../support/scenarioContext';
+
+// @todo `Omit` 'dashboardUid' and 'panelTitle'
+export interface ExploreConfig extends ConfigurePanelConfig {
+  queriesForm: (config: ExploreConfig) => void;
+}
+
+// @todo improve config input/output: https://stackoverflow.com/a/63507459/923745
+// @todo this actually returns type `Cypress.Chainable`
+export const explore = (config: Partial<ExploreConfig>): any =>
+  getScenarioContext().then(({ lastAddedDataSource }: any) =>
+    configurePanel(
+      {
+        dataSourceName: lastAddedDataSource,
+        isExplore: true,
+        screenshotName: 'explore-graph',
+        ...config,
+        timeRange: {
+          from: '2020-01-01 00:00:00',
+          to: '2020-01-01 06:00:00',
+          zone: 'Coordinated Universal Time',
+          ...config.timeRange,
+        },
+      } as ExploreConfig,
+      false
+    )
+  );

packages/grafana-e2e/src/flows/index.ts

@@ -5,9 +5,29 @@ export * from './assertSuccessNotification';
 export * from './deleteDashboard';
 export * from './deleteDataSource';
 export * from './editPanel';
+export * from './explore';
 export * from './login';
 export * from './openDashboard';
 export * from './openPanelMenuItem';
 export * from './revertAllChanges';
 export * from './saveDashboard';
 export * from './selectOption';
+
+export {
+  VISUALIZATION_ALERT_LIST,
+  VISUALIZATION_BAR_GAUGE,
+  VISUALIZATION_CLOCK,
+  VISUALIZATION_DASHBOARD_LIST,
+  VISUALIZATION_GAUGE,
+  VISUALIZATION_GRAPH,
+  VISUALIZATION_HEAT_MAP,
+  VISUALIZATION_LOGS,
+  VISUALIZATION_NEWS,
+  VISUALIZATION_PIE_CHART,
+  VISUALIZATION_PLUGIN_LIST,
+  VISUALIZATION_POLYSTAT,
+  VISUALIZATION_STAT,
+  VISUALIZATION_TABLE,
+  VISUALIZATION_TEXT,
+  VISUALIZATION_WORLD_MAP,
+} from './configurePanel';

packages/grafana-e2e/src/flows/openDashboard.ts

-import { DashboardTimeRangeConfig, setDashboardTimeRange } from './setDashboardTimeRange';
 import { e2e } from '../index';
 import { getScenarioContext } from '../support/scenarioContext';
+import { setDashboardTimeRange, TimeRangeConfig } from './setDashboardTimeRange';
 
 export interface OpenDashboardConfig {
-  timeRange?: DashboardTimeRangeConfig;
+  timeRange?: TimeRangeConfig;
   uid: string;
 }
 

packages/grafana-e2e/src/flows/selectOption.ts

 import { e2e } from '../index';
 
 // @todo this actually returns type `Cypress.Chainable`
-export const selectOption = (select: any, optionText: string): any =>
+export const selectOption = (select: any, optionText: string, clickToOpen = true): any =>
   select.within(() => {
+    if (clickToOpen) {
       e2e()
         .get('[class$="-input-suffix"]')
         .click();
+    }
+
     e2e.components.Select.option()
       .filter(`:contains("${optionText}")`)
       .scrollIntoView()

packages/grafana-e2e/src/flows/setDashboardTimeRange.ts

 import { e2e } from '../index';
+import { setTimeRange, TimeRangeConfig } from './setTimeRange';
 
-export interface DashboardTimeRangeConfig {
-  from: string;
-  to: string;
-}
+export { TimeRangeConfig };
 
-export const setDashboardTimeRange = ({ from, to }: DashboardTimeRangeConfig) =>
-  e2e.pages.Dashboard.Toolbar.navBar().within(() => {
-    e2e()
-      .get('[aria-label="TimePicker Open Button"]')
-      .click();
-    e2e()
-      .get('[aria-label="TimePicker absolute time range"]')
-      .click();
-    e2e()
-      .get('[aria-label="TimePicker from field"]')
-      .clear()
-      .type(from);
-    e2e()
-      .get('[aria-label="TimePicker to field"]')
-      .clear()
-      .type(to);
-    e2e()
-      .get('[aria-label="TimePicker submit button"]')
-      .click();
-  });
+export const setDashboardTimeRange = (config: TimeRangeConfig) =>
+  e2e.pages.Dashboard.Toolbar.navBar().within(() => setTimeRange(config));

packages/grafana-e2e/src/flows/setTimeRange.ts

+import { e2e } from '../index';
+import { selectOption } from './selectOption';
+
+export interface TimeRangeConfig {
+  from: string;
+  to: string;
+  zone?: string;
+}
+
+export const setTimeRange = ({ from, to, zone }: TimeRangeConfig) => {
+  e2e()
+    .get('[aria-label="TimePicker Open Button"]')
+    .click();
+
+  if (zone) {
+    e2e()
+      .contains('button', 'Change time zone')
+      .click();
+    selectOption(e2e.components.TimeZonePicker.container(), zone, false);
+  }
+
+  // For smaller screens
+  e2e()
+    .get('[aria-label="TimePicker absolute time range"]')
+    .click();
+
+  e2e()
+    .get('[aria-label="TimePicker from field"]')
+    .clear()
+    .type(from);
+  e2e()
+    .get('[aria-label="TimePicker to field"]')
+    .clear()
+    .type(to);
+  e2e()
+    .get('[aria-label="TimePicker submit button"]')
+    .click();
+};