github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 01:38:56 -04:00
Code Issues Projects Releases Packages Wiki Activity

[export saved objects] Use scroll API when exporting all

Browse source
This commit is contained in:
Jonathan Budzenski 2015-12-04 13:59:25 -06:00
parent 5ee11c894b
commit b53d48b49a
5 changed files with 161 additions and 34 deletions
Download patch file Download diff file Expand all files Collapse all files

1
src/plugins/elasticsearch/index.js
View file

@ -37,6 +37,7 @@ module.exports = function (kibana) {
createProxy(server, 'POST', '/_mget');
createProxy(server, 'POST', '/{index}/_search');
createProxy(server, 'POST', '/_msearch');
createProxy(server, 'POST', '/_search/scroll');
function noBulkCheck(request, reply) {
if (/\/_bulk/.test(request.path)) {

53
src/plugins/kibana/public/dashboard/services/saved_dashboards.js
View file

@ -42,6 +42,52 @@ define(function (require) {
});
};
this.scan = function (pageSize = 100, docCount = 1000) {
var allResults = {
hits: [],
total: 0
};
var self = this;
return new Promise(function (resolve, reject) {
es.search({
index: kbnIndex,
type: 'dashboard',
size: pageSize,
body: { query: {match_all: {}}},
searchType: 'scan',
scroll: '1m'
}, function getMoreUntilDone(error, response) {
var scanAllResults = docCount === Number.POSITIVE_INFINITY;
allResults.total = scanAllResults ? response.hits.total : Math.min(response.hits.total, docCount);
var hits = response.hits.hits
.slice(0, allResults.total - allResults.hits.length)
.map(self.mapHits.bind(self));
allResults.hits = allResults.hits.concat(hits);
var collectedAllResults = allResults.total === allResults.hits.length;
if (collectedAllResults) {
resolve(allResults);
} else {
es.scroll({
scrollId: response._scroll_id,
}, getMoreUntilDone);
}
});
});
};
this.scanAll = function (queryString, pageSize = 100) {
return this.scan(pageSize, Number.POSITIVE_INFINITY);
};
this.mapHits = function (hit) {
var source = hit._source;
source.id = hit._id;
source.url = this.urlFor(hit._id);
return source;
};
this.find = function (searchString, size = 100) {
var self = this;
@ -69,12 +115,7 @@ define(function (require) {
.then(function (resp) {
return {
total: resp.hits.total,
hits: resp.hits.hits.map(function (hit) {
var source = hit._source;
source.id = hit._id;
source.url = self.urlFor(hit._id);
return source;
})
hits: resp.hits.hits.map(self.mapHits.bind(self))
};
});
};

55
src/plugins/kibana/public/discover/saved_searches/saved_searches.js
View file

@ -31,6 +31,47 @@ define(function (require) {
nouns: 'saved searches'
};
this.scan = function (pageSize = 100, docCount = 1000) {
var allResults = {
hits: [],
total: 0
};
var self = this;
return new Promise(function (resolve, reject) {
es.search({
index: kbnIndex,
type: 'search',
size: pageSize,
body: { query: {match_all: {}}},
searchType: 'scan',
scroll: '1m'
}, function getMoreUntilDone(error, response) {
var scanAllResults = docCount === Number.POSITIVE_INFINITY;
allResults.total = scanAllResults ? response.hits.total : Math.min(response.hits.total, docCount);
var hits = response.hits.hits
.slice(0, allResults.total - allResults.hits.length)
.map(self.mapHits.bind(self));
allResults.hits = allResults.hits.concat(hits);
var collectedAllResults = allResults.total === allResults.hits.length;
if (collectedAllResults) {
resolve(allResults);
} else {
es.scroll({
scrollId: response._scroll_id,
}, getMoreUntilDone);
}
});
});
};
this.scanAll = function (queryString, pageSize = 100) {
return this.scan(pageSize, Number.POSITIVE_INFINITY);
};
this.get = function (id) {
return (new SavedSearch(id)).init();
};
@ -46,6 +87,13 @@ define(function (require) {
});
};
this.mapHits = function (hit) {
var source = hit._source;
source.id = hit._id;
source.url = this.urlFor(hit._id);
return source;
};
this.find = function (searchString, size = 100) {
var self = this;
var body;
@ -72,12 +120,7 @@ define(function (require) {
.then(function (resp) {
return {
total: resp.hits.total,
hits: resp.hits.hits.map(function (hit) {
var source = hit._source;
source.id = hit._id;
source.url = self.urlFor(hit._id);
return source;
})
hits: resp.hits.hits.map(self.mapHits.bind(self))
};
});
};

2
src/plugins/kibana/public/settings/sections/objects/_objects.js
View file

@ -94,7 +94,7 @@ define(function (require) {
$scope.exportAll = () => {
Promise.map($scope.services, (service) =>
service.service.find('', MAX_SIZE).then((results) =>
service.service.scanAll('').then((results) =>
results.hits.map((hit) => _.extend(hit, {type: service.type}))
)
).then((results) => retrieveAndExportDocs(_.flattenDeep(results)));

84
src/plugins/kibana/public/visualize/saved_visualizations/saved_visualizations.js
View file

@ -41,6 +41,68 @@ define(function (require) {
});
};
this.scan = function (pageSize = 100, docCount = 1000) {
var allResults = {
hits: [],
total: 0
};
var self = this;
return new Promise(function (resolve, reject) {
es.search({
index: kbnIndex,
type: 'visualization',
size: pageSize,
body: { query: {match_all: {}}},
searchType: 'scan',
scroll: '1m'
}, function getMoreUntilDone(error, response) {
var scanAllResults = docCount === Number.POSITIVE_INFINITY;
allResults.total = scanAllResults ? response.hits.total : Math.min(response.hits.total, docCount);
var hits = response.hits.hits
.slice(0, allResults.total - allResults.hits.length)
.map(self.mapHits.bind(self));
allResults.hits = allResults.hits.concat(hits);
var collectedAllResults = allResults.total === allResults.hits.length;
if (collectedAllResults) {
resolve(allResults);
} else {
es.scroll({
scrollId: response._scroll_id,
}, getMoreUntilDone);
}
});
});
};
this.scanAll = function (queryString, pageSize = 100) {
return this.scan(pageSize, Number.POSITIVE_INFINITY);
};
this.mapHits = function (hit) {
var source = hit._source;
source.id = hit._id;
source.url = this.urlFor(hit._id);
var typeName = source.typeName;
if (source.visState) {
try { typeName = JSON.parse(source.visState).type; }
catch (e) { /* missing typename handled below */ } // eslint-disable-line no-empty
}
if (!typeName || !visTypes.byName[typeName]) {
if (!typeName) notify.error('Visualization type is missing. Please add a type to this visualization.', hit);
else notify.error('Visualization type of "' + typeName + '" is invalid. Please change to a valid type.', hit);
return kbnUrl.redirect('/settings/objects/savedVisualizations/{{id}}', {id: source.id});
}
source.type = visTypes.byName[typeName];
source.icon = source.type.icon;
return source;
};
this.find = function (searchString, size = 100) {
var self = this;
var body;
@ -67,27 +129,7 @@ define(function (require) {
.then(function (resp) {
return {
total: resp.hits.total,
hits: _.transform(resp.hits.hits, function (hits, hit) {
var source = hit._source;
source.id = hit._id;
source.url = self.urlFor(hit._id);
var typeName = source.typeName;
if (source.visState) {
try { typeName = JSON.parse(source.visState).type; }
catch (e) { /* missing typename handled below */ } // eslint-disable-line no-empty
}
if (!typeName || !visTypes.byName[typeName]) {
if (!typeName) notify.error('Visualization type is missing. Please add a type to this visualization.', hit);
else notify.error('Visualization type of "' + typeName + '" is invalid. Please change to a valid type.', hit);
return kbnUrl.redirect('/settings/objects/savedVisualizations/{{id}}', {id: source.id});
}
source.type = visTypes.byName[typeName];
source.icon = source.type.icon;
hits.push(source);
}, [])
hits: resp.hits.hits.map(self.mapHits.bind(self))
};
});
};