move bulk error handler to separate function, re-add dates

Signed-off-by: Jakub Sokołowski <jakub@status.im>
This commit is contained in:
Jakub Sokołowski 2018-12-03 11:20:25 +01:00
parent 80251bbbd2
commit f32df64777
No known key found for this signature in database
GPG Key ID: 4EF064D0E6D63020
1 changed files with 26 additions and 18 deletions

View File

@ -22,13 +22,30 @@ const INDEXED_PROPERTIES = [
'title',
'excerpt',
'raw',
'permalink',
'author',
//'date',
//'updated',
//'layout'
];
function bulkCallback(err, resp) {
/* if there's an error show it */
if (err) {
error(`%s`, err);
error('>> ElasticSearch request failed.');
}
/* if there are errors in specific operations show them */
if (resp.errors) {
each(resp.items, (item) => {
if (item.status != 200) {
error(item);
}
})
} else {
info('Successful indexing of %d posts.', resp.items.length);
}
}
function sha1(text) {
return crypto.createHash('sha1').update(text, 'utf8').digest('hex');
}
@ -105,6 +122,13 @@ module.exports = function(args, callback) {
return allPosts.map(function(data) {
var post = pick(data, INDEXED_PROPERTIES);
/* simpler property names */
post.url = data.permalink;
/* dates by default are stored as objects */
post.created = data.date.toISOString();
post.updated = data.updated.toISOString();
if (Array.isArray(data.categories)) {
post.categories = data.categories.map(function(item) {
return pick(item, ['name', 'path']);
@ -132,23 +156,7 @@ module.exports = function(args, callback) {
return [].concat.apply([], actionsAndPosts);
})
.then((postsBatch) => { /* batch upload posts to ElasticSearch */
es.bulk({body: postsBatch},
(err, resp) => {
if (err) {
error(`%s`, err);
error('>> ElasticSearch request failed.');
}
if (resp.errors) {
each(resp.items, (item) => {
error(item);
})
} else {
each(resp.items, (item) => {
info(item);
})
}
}
)
es.bulk({body: postsBatch}, bulkCallback)
})
.then(() => {
info('Indexing done.');