Google Sheet Integration

Step 1: Select "Apps Script"

From within a new Google Sheet, select "Extensions" and then "Apps Script".
Step 2: Add the Javascript

Copy the below code and replace all of the existing content on the AppsScript screen.
/**
 * Datastreamer Search -> Google Sheets (Apps Script)
 * Clean, ASCII-only, compile-safe.
 *
 * POST https://api.platform.datastreamer.io/api/search
 * Header: apikey: <YOUR_API_KEY>
 * Body: { "query": { "from": 0, "size": 10, "query": "foo", "data_sources": ["private.my_storage"] } }
 * Resp: { "results": [ ... ] }
 */

/* =========================
 * Menu and bootstrap
 * ========================= */
function onOpen() {
  SpreadsheetApp.getUi()
    .createMenu('Datastreamer')
    .addItem('Configure API...', 'ds_configure')
    .addSeparator()
    .addItem('Run Search...', 'ds_runSearchPrompt')
    .addItem('Clear Sheet', 'ds_clearSheet')
    .addSeparator()
    .addItem('Toggle Append Mode', 'ds_toggleAppendMode')
    .addSubMenu(
      SpreadsheetApp.getUi().createMenu('Saved Queries')
        .addItem('Save Query...', 'ds_saveQueryPrompt')
        .addItem('Run Saved Query...', 'ds_runSavedQuery')
        .addItem('Delete Saved Query...', 'ds_deleteSavedQuery')
    )
    .addToUi();
}

/* =========================
 * Config & helpers
 * ========================= */
function ds_clearSheet() {
  var sh = SpreadsheetApp.getActiveSheet();
  sh.clear({ contentsOnly: true });
}

function ds_toggleAppendMode() {
  var p = PropertiesService.getScriptProperties();
  var curr = p.getProperty('DS_APPEND') === 'true';
  var next = !curr;
  p.setProperty('DS_APPEND', String(next));
  SpreadsheetApp.getUi().alert('Append mode is now ' + (next ? 'ON' : 'OFF') + '.');
}

function ds_isAppend_() {
  return PropertiesService.getScriptProperties().getProperty('DS_APPEND') === 'true';
}

/* Configure only stores API key in Script Properties.
   Base and Path are fixed to Datastreamer defaults. */
function ds_configure() {
  var ui = SpreadsheetApp.getUi();
  var sp = PropertiesService.getScriptProperties();

  var keyResp = ui.prompt('Datastreamer API Key', 'Paste your API key from Datastreamer: Keys & Secrets.', ui.ButtonSet.OK_CANCEL);
  if (keyResp.getSelectedButton() !== ui.Button.OK) return;

  var apiKey = (keyResp.getResponseText() || '').trim();
  sp.setProperty('DS_API_KEY', apiKey);

  // Hard-coded defaults per your request
  sp.setProperty('DS_BASE', 'https://api.platform.datastreamer.io');
  sp.setProperty('DS_PATH', '/api/search');

  ui.alert('Saved', 'Base: https://api.platform.datastreamer.io\nPath: /api/search', ui.ButtonSet.OK);
}

/* =========================
 * Search UI
 * ========================= */
function ds_runSearchPrompt() {
  var ui = SpreadsheetApp.getUi();

  var dsResp = ui.prompt('Data source(s)', 'Comma-separated (e.g., private.my_storage or wsl_instagram)', ui.ButtonSet.OK_CANCEL);
  if (dsResp.getSelectedButton() !== ui.Button.OK) return;

  var dataSources = (dsResp.getResponseText() || '').split(',');
  var cleaned = [];
  for (var i = 0; i < dataSources.length; i++) {
    var s = (dataSources[i] || '').replace(/^\s+|\s+$/g, '');
    if (s) cleaned.push(s);
  }
  if (!cleaned.length) { ui.alert('You must enter at least one data source.'); return; }

  var qResp = ui.prompt('Query (Lucene)', 'Example: coffee AND doc_date:[2025-01-01 TO 2025-01-31]', ui.ButtonSet.OK_CANCEL);
  if (qResp.getSelectedButton() !== ui.Button.OK) return;
  var query = (qResp.getResponseText() || '').trim();
  if (!query) query = '*';

  var sizeResp = ui.prompt('Page size', 'Results per page (default 200).', ui.ButtonSet.OK_CANCEL);
  if (sizeResp.getSelectedButton() !== ui.Button.OK) return;
  var pageSize = Number((sizeResp.getResponseText() || '').trim());
  if (!pageSize || pageSize < 1) pageSize = 200;

  // No "Max pages" prompt by request: always 1 page
  var maxPages = 1;

  ds_runSearch_(cleaned, query, pageSize, maxPages);
}

/* =========================
 * Runner
 * ========================= */
function ds_runSearch_(dataSources, query, size, maxPages) {
  var sh = SpreadsheetApp.getActiveSheet();
  var append = ds_isAppend_();

  if (!append) sh.clear({ contentsOnly: true });

  // headers map to keep stable order: meta first, then alpha
  var headersMap = { id: true, data_source: true, doc_date: true };
  var rows = [];

  var from = 0;
  for (var page = 0; page < maxPages; page++) {
    var resp = ds_fetchPage_(dataSources, query, from, size);
    var results = (resp && resp.results) ? resp.results : [];
    if (!results.length) break;

    for (var i = 0; i < results.length; i++) {
      var r = results[i];
      var flat = ds_flatten_(r, '');
      if (!flat.id) flat.id = r.id || '';
      if (!flat.data_source) flat.data_source = r.data_source || (dataSources[0] || '');
      if (!flat.doc_date) flat.doc_date = r.doc_date || '';
      for (var k in flat) headersMap[k] = true;
      rows.push(flat);
    }

    if (results.length < size) break; // last page
    from = from + size;
    Utilities.sleep(200);
  }

  if (!rows.length) { SpreadsheetApp.getUi().alert('No results found.'); return; }

  var headers = ['id', 'data_source', 'doc_date'];
  var extras = [];
  for (var key in headersMap) {
    if (key !== 'id' && key !== 'data_source' && key !== 'doc_date') extras.push(key);
  }
  extras.sort();
  headers = headers.concat(extras);

  var values = [];
  var lastRow = sh.getLastRow();
  var writeHeaders = (!append || lastRow === 0);
  if (writeHeaders) values.push(headers);

  for (var r = 0; r < rows.length; r++) {
    var obj = rows[r];
    var rowVals = [];
    for (var h = 0; h < headers.length; h++) {
      var field = headers[h];
      var val = obj[field];
      if (val === null || typeof val === 'undefined') val = '';
      rowVals.push(val);
    }
    values.push(rowVals);
  }

  var startRow = writeHeaders ? 1 : (lastRow + 1);
  sh.getRange(startRow, 1, values.length, headers.length).setValues(values);
  if (writeHeaders) sh.setFrozenRows(1);
  sh.autoResizeColumns(1, headers.length);
}

/* =========================
 * Network
 * ========================= */
function ds_fetchPage_(dataSources, query, from, size) {
  var sp = PropertiesService.getScriptProperties();
  var baseUrl = (sp.getProperty('DS_BASE') || 'https://api.platform.datastreamer.io').replace(/\/$/, '');
  var path = sp.getProperty('DS_PATH') || '/api/search';
  var apiKey = sp.getProperty('DS_API_KEY');
  if (!apiKey) throw new Error('Missing DS_API_KEY. Use Datastreamer -> Configure API...');

  var url = baseUrl + path;
  var body = { query: { from: from || 0, size: size || 10, query: query || '*', data_sources: dataSources } };

  var options = {
    method: 'post',
    contentType: 'application/json',
    payload: JSON.stringify(body),
    headers: { 'apikey': apiKey },
    muteHttpExceptions: true
  };

  var res = UrlFetchApp.fetch(url, options);
  var code = res.getResponseCode();
  if (code < 200 || code >= 300) {
    throw new Error('Search failed (' + code + '): ' + res.getContentText());
  }
  return JSON.parse(res.getContentText());
}

/* =========================
 * Utils
 * ========================= */
function ds_flatten_(obj, prefix) {
  var out = {};
  var pre = prefix ? prefix + '.' : '';
  for (var k in obj) {
    if (!obj.hasOwnProperty(k)) continue;
    var v = obj[k];
    var key = pre + k;
    if (v === null || typeof v === 'undefined') {
      out[key] = '';
    } else if (Object.prototype.toString.call(v) === '[object Array]') {
      var acc = [];
      for (var i = 0; i < v.length; i++) {
        var x = v[i];
        acc.push(x && typeof x === 'object' ? JSON.stringify(x) : x);
      }
      out[key] = acc.join(', ');
    } else if (typeof v === 'object') {
      var nested = ds_flatten_(v, key);
      for (var nk in nested) { out[nk] = nested[nk]; }
    } else {
      out[key] = v;
    }
  }
  return out;
}

/* =========================
 * Saved queries (optional)
 * ========================= */
function ds_getSaved_() {
  var sp = PropertiesService.getScriptProperties();
  var raw = sp.getProperty('DS_SAVED_QUERIES');
  if (!raw) return [];
  try { return JSON.parse(raw); } catch (e) { return []; }
}
function ds_setSaved_(arr) {
  PropertiesService.getScriptProperties().setProperty('DS_SAVED_QUERIES', JSON.stringify(arr || []));
}

function ds_saveQueryPrompt() {
  var ui = SpreadsheetApp.getUi();
  var nameResp = ui.prompt('Save Query', 'Give this query a name:', ui.ButtonSet.OK_CANCEL);
  if (nameResp.getSelectedButton() !== ui.Button.OK) return;
  var name = (nameResp.getResponseText() || '').trim();
  if (!name) { ui.alert('Name is required.'); return; }

  var dsResp = ui.prompt('Data source(s)', 'Comma-separated (e.g., private.my_storage,wsl_instagram)', ui.ButtonSet.OK_CANCEL);
  if (dsResp.getSelectedButton() !== ui.Button.OK) return;
  var dataSources = (dsResp.getResponseText() || '').split(',');
  var cleaned = [];
  for (var i = 0; i < dataSources.length; i++) {
    var s = (dataSources[i] || '').replace(/^\s+|\s+$/g, '');
    if (s) cleaned.push(s);
  }
  if (!cleaned.length) { ui.alert('At least one data source required.'); return; }

  var qResp = ui.prompt('Query (Lucene)', 'Example: coffee AND doc_date:[2025-01-01 TO 2025-01-31]', ui.ButtonSet.OK_CANCEL);
  if (qResp.getSelectedButton() !== ui.Button.OK) return;
  var query = (qResp.getResponseText() || '*').trim();

  var sizeResp = ui.prompt('Page size', 'Default 200', ui.ButtonSet.OK_CANCEL);
  if (sizeResp.getSelectedButton() !== ui.Button.OK) return;
  var pageSize = Number((sizeResp.getResponseText() || '').trim());
  if (!pageSize || pageSize < 1) pageSize = 200;

  var saved = ds_getSaved_();
  var idx = -1;
  for (var j = 0; j < saved.length; j++) { if (saved[j].name === name) { idx = j; break; } }
  var item = { name: name, dataSources: cleaned, query: query, pageSize: pageSize, maxPages: 1, ts: new Date().toISOString() };
  if (idx >= 0) saved[idx] = item; else saved.push(item);
  ds_setSaved_(saved);
  ui.alert('Saved query "' + name + '"');
}

function ds_runSavedQuery() {
  var ui = SpreadsheetApp.getUi();
  var saved = ds_getSaved_();
  if (!saved.length) { ui.alert('No saved queries yet.'); return; }
  var names = [];
  for (var i = 0; i < saved.length; i++) names.push(saved[i].name);
  var pick = ui.prompt('Run Saved Query', 'Available: ' + names.join(', ') + '\nType the exact name to run:', ui.ButtonSet.OK_CANCEL);
  if (pick.getSelectedButton() !== ui.Button.OK) return;
  var name = (pick.getResponseText() || '').trim();

  var q = null;
  for (var j = 0; j < saved.length; j++) { if (saved[j].name === name) { q = saved[j]; break; } }
  if (!q) { ui.alert('Not found.'); return; }

  ds_runSearch_(q.dataSources, q.query, q.pageSize, 1);
}

function ds_deleteSavedQuery() {
  var ui = SpreadsheetApp.getUi();
  var saved = ds_getSaved_();
  if (!saved.length) { ui.alert('No saved queries to delete.'); return; }
  var names = [];
  for (var i = 0; i < saved.length; i++) names.push(saved[i].name);
  var pick = ui.prompt('Delete Saved Query', 'Available: ' + names.join(', ') + '\nType the exact name to delete:', ui.ButtonSet.OK_CANCEL);
  if (pick.getSelectedButton() !== ui.Button.OK) return;
  var name = (pick.getResponseText() || '').trim();

  var next = [];
  for (var j = 0; j < saved.length; j++) { if (saved[j].name !== name) next.push(saved[j]); }
  if (next.length === saved.length) { ui.alert('Not found.'); return; }
  ds_setSaved_(next);
  ui.alert('Deleted "' + name + '"');
}
Step 3: Save and Run

Select Save, and then Run will appear after saving. You do not need to deploy. Some authorization may be required for the script to run. It will then say "Execution Completed" on the bottom.
4. Configure API Key

You should now see a new "Datastreamer" menu option, just after the "Help" menu. First "Configure API" using the API key on your Keys & Secrets page within Portal.
5. Ingest into Google Sheets!

You can then "Run Search", please note that any Searchable Storage index begins with "private.(whatever the name is)".
A few other things of note:
You can save, run, and delete a query for repeat usage.
"Toggle Append Mode" makes it so that new queries are added onto the end of the sheet instead of overwriting.