Fix/improved csv output (#11740)

* fix: initial cleanup and implementation

* feat: finish special character escaping

* feat: updates fileExport to generate RFC-4180 compliant CSV

* chore: replace html decoder with the lodash version and final cleanup

* fix: restore character html decoding

public/app/core/specs/file_export.jest.ts

@@ -30,17 +30,17 @@ describe('file_export', () => {
     it('should export points in proper order', () => {
     it('should export points in proper order', () => {
       let text = fileExport.convertSeriesListToCsv(ctx.seriesList, ctx.timeFormat);
       let text = fileExport.convertSeriesListToCsv(ctx.seriesList, ctx.timeFormat);
       const expectedText =
       const expectedText =
-        'Series;Time;Value\n' +
-        'series_1;1500026100;1\n' +
-        'series_1;1500026200;2\n' +
-        'series_1;1500026300;null\n' +
-        'series_1;1500026400;null\n' +
-        'series_1;1500026500;null\n' +
-        'series_1;1500026600;6\n' +
-        'series_2;1500026100;11\n' +
-        'series_2;1500026200;12\n' +
-        'series_2;1500026300;13\n' +
-        'series_2;1500026500;15\n';
+        '"Series";"Time";"Value"\r\n' +
+        '"series_1";"1500026100";1\r\n' +
+        '"series_1";"1500026200";2\r\n' +
+        '"series_1";"1500026300";null\r\n' +
+        '"series_1";"1500026400";null\r\n' +
+        '"series_1";"1500026500";null\r\n' +
+        '"series_1";"1500026600";6\r\n' +
+        '"series_2";"1500026100";11\r\n' +
+        '"series_2";"1500026200";12\r\n' +
+        '"series_2";"1500026300";13\r\n' +
+        '"series_2";"1500026500";15';
 
 
       expect(text).toBe(expectedText);
       expect(text).toBe(expectedText);
     });
     });
@@ -50,15 +50,79 @@ describe('file_export', () => {
     it('should export points in proper order', () => {
     it('should export points in proper order', () => {
       let text = fileExport.convertSeriesListToCsvColumns(ctx.seriesList, ctx.timeFormat);
       let text = fileExport.convertSeriesListToCsvColumns(ctx.seriesList, ctx.timeFormat);
       const expectedText =
       const expectedText =
-        'Time;series_1;series_2\n' +
-        '1500026100;1;11\n' +
-        '1500026200;2;12\n' +
-        '1500026300;null;13\n' +
-        '1500026400;null;null\n' +
-        '1500026500;null;15\n' +
-        '1500026600;6;null\n';
+        '"Time";"series_1";"series_2"\r\n' +
+        '"1500026100";1;11\r\n' +
+        '"1500026200";2;12\r\n' +
+        '"1500026300";null;13\r\n' +
+        '"1500026400";null;null\r\n' +
+        '"1500026500";null;15\r\n' +
+        '"1500026600";6;null';
 
 
       expect(text).toBe(expectedText);
       expect(text).toBe(expectedText);
     });
     });
   });
   });
+
+  describe('when exporting table data to csv', () => {
+
+    it('should properly escape special characters and quote all string values', () => {
+      const inputTable = {
+        columns: [
+          { title: 'integer_value' },
+          { text: 'string_value' },
+          { title: 'float_value' },
+          { text: 'boolean_value' },
+        ],
+        rows: [
+          [123, 'some_string', 1.234, true],
+          [0o765, 'some string with " in the middle', 1e-2, false],
+          [0o765, 'some string with "" in the middle', 1e-2, false],
+          [0o765, 'some string with """ in the middle', 1e-2, false],
+          [0o765, '"some string with " at the beginning', 1e-2, false],
+          [0o765, 'some string with " at the end"', 1e-2, false],
+          [0x123, 'some string with \n in the middle', 10.01, false],
+          [0b1011, 'some string with ; in the middle', -12.34, true],
+          [123, 'some string with ;; in the middle', -12.34, true],
+        ],
+      };
+
+      const returnedText = fileExport.convertTableDataToCsv(inputTable, false);
+
+      const expectedText =
+        '"integer_value";"string_value";"float_value";"boolean_value"\r\n' +
+        '123;"some_string";1.234;true\r\n' +
+        '501;"some string with "" in the middle";0.01;false\r\n' +
+        '501;"some string with """" in the middle";0.01;false\r\n' +
+        '501;"some string with """""" in the middle";0.01;false\r\n' +
+        '501;"""some string with "" at the beginning";0.01;false\r\n' +
+        '501;"some string with "" at the end""";0.01;false\r\n' +
+        '291;"some string with \n in the middle";10.01;false\r\n' +
+        '11;"some string with ; in the middle";-12.34;true\r\n' +
+        '123;"some string with ;; in the middle";-12.34;true';
+
+      expect(returnedText).toBe(expectedText);
+    });
+
+    it('should decode HTML encoded characters', function() {
+      const inputTable = {
+        columns: [
+          { text: 'string_value' },
+        ],
+        rows: [
+          ['"&ä'],
+          ['<strong>&quot;some html&quot;</strong>'],
+          ['<a href="http://something/index.html">some text</a>']
+        ],
+      };
+
+      const returnedText = fileExport.convertTableDataToCsv(inputTable, false);
+
+      const expectedText =
+        '"string_value"\r\n' +
+        '"""&ä"\r\n' +
+        '"<strong>""some html""</strong>"\r\n' +
+        '"<a href=""http://something/index.html"">some text</a>"';
+
+      expect(returnedText).toBe(expectedText);
+    });
+  });
 });
 });

public/app/core/utils/file_export.ts

-import _ from 'lodash';
+import { isBoolean, isNumber, sortedUniq, sortedIndexOf, unescape as htmlUnescaped } from 'lodash';
 import moment from 'moment';
 import moment from 'moment';
 import { saveAs } from 'file-saver';
 import { saveAs } from 'file-saver';
+import { isNullOrUndefined } from 'util';
 
 
 const DEFAULT_DATETIME_FORMAT = 'YYYY-MM-DDTHH:mm:ssZ';
 const DEFAULT_DATETIME_FORMAT = 'YYYY-MM-DDTHH:mm:ssZ';
 const POINT_TIME_INDEX = 1;
 const POINT_TIME_INDEX = 1;
 const POINT_VALUE_INDEX = 0;
 const POINT_VALUE_INDEX = 0;
 
 
+const END_COLUMN = ';';
+const END_ROW = '\r\n';
+const QUOTE = '"';
+const EXPORT_FILENAME = 'grafana_data_export.csv';
+
+function csvEscaped(text) {
+  if (!text) {
+    return text;
+  }
+
+  return text.split(QUOTE).join(QUOTE + QUOTE);
+}
+
+const domParser = new DOMParser();
+function htmlDecoded(text) {
+  if (!text) {
+    return text;
+  }
+
+  const regexp = /&[^;]+;/g;
+  function htmlDecoded(value) {
+    const parsedDom = domParser.parseFromString(value, 'text/html');
+    return parsedDom.body.textContent;
+  }
+  return text.replace(regexp, htmlDecoded).replace(regexp, htmlDecoded);
+}
+
+function formatSpecialHeader(useExcelHeader) {
+  return useExcelHeader ? `sep=${END_COLUMN}${END_ROW}` : '';
+}
+
+function formatRow(row, addEndRowDelimiter = true) {
+  let text = '';
+  for (let i = 0; i < row.length; i += 1) {
+    if (isBoolean(row[i]) || isNullOrUndefined(row[i])) {
+      text += row[i];
+    } else if (isNumber(row[i])) {
+      text += row[i].toLocaleString();
+    } else {
+      text += `${QUOTE}${csvEscaped(htmlUnescaped(htmlDecoded(row[i])))}${QUOTE}`;
+    }
+
+    if (i < row.length - 1) {
+      text += END_COLUMN;
+    }
+  }
+  return addEndRowDelimiter ? text + END_ROW : text;
+}
+
 export function convertSeriesListToCsv(seriesList, dateTimeFormat = DEFAULT_DATETIME_FORMAT, excel = false) {
 export function convertSeriesListToCsv(seriesList, dateTimeFormat = DEFAULT_DATETIME_FORMAT, excel = false) {
-  var text = (excel ? 'sep=;\n' : '') + 'Series;Time;Value\n';
-  _.each(seriesList, function(series) {
-    _.each(series.datapoints, function(dp) {
-      text +=
-        series.alias + ';' + moment(dp[POINT_TIME_INDEX]).format(dateTimeFormat) + ';' + dp[POINT_VALUE_INDEX] + '\n';
-    });
-  });
+  let text = formatSpecialHeader(excel) + formatRow(['Series', 'Time', 'Value']);
+  for (let seriesIndex = 0; seriesIndex < seriesList.length; seriesIndex += 1) {
+    for (let i = 0; i < seriesList[seriesIndex].datapoints.length; i += 1) {
+      text += formatRow(
+        [
+          seriesList[seriesIndex].alias,
+          moment(seriesList[seriesIndex].datapoints[i][POINT_TIME_INDEX]).format(dateTimeFormat),
+          seriesList[seriesIndex].datapoints[i][POINT_VALUE_INDEX],
+        ],
+        i < seriesList[seriesIndex].datapoints.length - 1 || seriesIndex < seriesList.length - 1
+      );
+    }
+  }
   return text;
   return text;
 }
 }
 
 
 export function exportSeriesListToCsv(seriesList, dateTimeFormat = DEFAULT_DATETIME_FORMAT, excel = false) {
 export function exportSeriesListToCsv(seriesList, dateTimeFormat = DEFAULT_DATETIME_FORMAT, excel = false) {
-  var text = convertSeriesListToCsv(seriesList, dateTimeFormat, excel);
-  saveSaveBlob(text, 'grafana_data_export.csv');
+  let text = convertSeriesListToCsv(seriesList, dateTimeFormat, excel);
+  saveSaveBlob(text, EXPORT_FILENAME);
 }
 }
 
 
 export function convertSeriesListToCsvColumns(seriesList, dateTimeFormat = DEFAULT_DATETIME_FORMAT, excel = false) {
 export function convertSeriesListToCsvColumns(seriesList, dateTimeFormat = DEFAULT_DATETIME_FORMAT, excel = false) {
-  let text = (excel ? 'sep=;\n' : '') + 'Time;';
   // add header
   // add header
-  _.each(seriesList, function(series) {
-    text += series.alias + ';';
-  });
-  text = text.substring(0, text.length - 1);
-  text += '\n';
-
+  let text =
+    formatSpecialHeader(excel) +
+    formatRow(
+      ['Time'].concat(
+        seriesList.map(function(val) {
+          return val.alias;
+        })
+      )
+    );
   // process data
   // process data
   seriesList = mergeSeriesByTime(seriesList);
   seriesList = mergeSeriesByTime(seriesList);
-  var dataArr = [[]];
-  var sIndex = 1;
-  _.each(seriesList, function(series) {
-    var cIndex = 0;
-    dataArr.push([]);
-    _.each(series.datapoints, function(dp) {
-      dataArr[0][cIndex] = moment(dp[POINT_TIME_INDEX]).format(dateTimeFormat);
-      dataArr[sIndex][cIndex] = dp[POINT_VALUE_INDEX];
-      cIndex++;
-    });
-    sIndex++;
-  });
 
 
   // make text
   // make text
-  for (var i = 0; i < dataArr[0].length; i++) {
-    text += dataArr[0][i] + ';';
-    for (var j = 1; j < dataArr.length; j++) {
-      text += dataArr[j][i] + ';';
-    }
-    text = text.substring(0, text.length - 1);
-    text += '\n';
+  for (let i = 0; i < seriesList[0].datapoints.length; i += 1) {
+    const timestamp = moment(seriesList[0].datapoints[i][POINT_TIME_INDEX]).format(dateTimeFormat);
+    text += formatRow(
+      [timestamp].concat(
+        seriesList.map(function(series) {
+          return series.datapoints[i][POINT_VALUE_INDEX];
+        })
+      ),
+      i < seriesList[0].datapoints.length - 1
+    );
   }
   }
 
 
   return text;
   return text;
@@ -71,15 +120,15 @@ function mergeSeriesByTime(seriesList) {
       timestamps.push(seriesPoints[j][POINT_TIME_INDEX]);
       timestamps.push(seriesPoints[j][POINT_TIME_INDEX]);
     }
     }
   }
   }
-  timestamps = _.sortedUniq(timestamps.sort());
+  timestamps = sortedUniq(timestamps.sort());
 
 
   for (let i = 0; i < seriesList.length; i++) {
   for (let i = 0; i < seriesList.length; i++) {
     let seriesPoints = seriesList[i].datapoints;
     let seriesPoints = seriesList[i].datapoints;
-    let seriesTimestamps = _.map(seriesPoints, p => p[POINT_TIME_INDEX]);
+    let seriesTimestamps = seriesPoints.map(p => p[POINT_TIME_INDEX]);
     let extendedSeries = [];
     let extendedSeries = [];
     let pointIndex;
     let pointIndex;
     for (let j = 0; j < timestamps.length; j++) {
     for (let j = 0; j < timestamps.length; j++) {
-      pointIndex = _.sortedIndexOf(seriesTimestamps, timestamps[j]);
+      pointIndex = sortedIndexOf(seriesTimestamps, timestamps[j]);
       if (pointIndex !== -1) {
       if (pointIndex !== -1) {
         extendedSeries.push(seriesPoints[pointIndex]);
         extendedSeries.push(seriesPoints[pointIndex]);
       } else {
       } else {
@@ -93,27 +142,26 @@ function mergeSeriesByTime(seriesList) {
 
 
 export function exportSeriesListToCsvColumns(seriesList, dateTimeFormat = DEFAULT_DATETIME_FORMAT, excel = false) {
 export function exportSeriesListToCsvColumns(seriesList, dateTimeFormat = DEFAULT_DATETIME_FORMAT, excel = false) {
   let text = convertSeriesListToCsvColumns(seriesList, dateTimeFormat, excel);
   let text = convertSeriesListToCsvColumns(seriesList, dateTimeFormat, excel);
-  saveSaveBlob(text, 'grafana_data_export.csv');
+  saveSaveBlob(text, EXPORT_FILENAME);
 }
 }
 
 
-export function exportTableDataToCsv(table, excel = false) {
-  var text = excel ? 'sep=;\n' : '';
-  // add header
-  _.each(table.columns, function(column) {
-    text += (column.title || column.text) + ';';
-  });
-  text += '\n';
+export function convertTableDataToCsv(table, excel = false) {
+  let text = formatSpecialHeader(excel);
+  // add headline
+  text += formatRow(table.columns.map(val => val.title || val.text));
   // process data
   // process data
-  _.each(table.rows, function(row) {
-    _.each(row, function(value) {
-      text += value + ';';
-    });
-    text += '\n';
-  });
-  saveSaveBlob(text, 'grafana_data_export.csv');
+  for (let i = 0; i < table.rows.length; i += 1) {
+    text += formatRow(table.rows[i], i < table.rows.length - 1);
+  }
+  return text;
+}
+
+export function exportTableDataToCsv(table, excel = false) {
+  let text = convertTableDataToCsv(table, excel);
+  saveSaveBlob(text, EXPORT_FILENAME);
 }
 }
 
 
 export function saveSaveBlob(payload, fname) {
 export function saveSaveBlob(payload, fname) {
-  var blob = new Blob([payload], { type: 'text/csv;charset=utf-8' });
+  let blob = new Blob([payload], { type: 'text/csv;charset=utf-8;header=present;' });
   saveAs(blob, fname);
   saveAs(blob, fname);
 }
 }