@webiny/api-page-builder-so-ddb-es
Version:
The DynamoDB + Elasticsearch storage operations Webiny Page Builder API.
800 lines (788 loc) • 23.4 kB
JavaScript
"use strict";
var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault").default;
Object.defineProperty(exports, "__esModule", {
value: true
});
exports.createPageStorageOperations = void 0;
var _omit = _interopRequireDefault(require("lodash/omit"));
var _error = _interopRequireDefault(require("@webiny/error"));
var _cleanup = require("@webiny/db-dynamodb/utils/cleanup");
var _configurations = require("../../configurations");
var _apiElasticsearch = require("@webiny/api-elasticsearch");
var _elasticsearchQueryBody = require("./elasticsearchQueryBody");
var _SearchLatestPagesPlugin = require("../../plugins/definitions/SearchLatestPagesPlugin");
var _SearchPublishedPagesPlugin = require("../../plugins/definitions/SearchPublishedPagesPlugin");
var _dbDynamodb = require("@webiny/db-dynamodb");
var _helpers = require("./helpers");
var _keys = require("./keys");
var _PageDynamoDbElasticsearchFieldPlugin = require("../../plugins/definitions/PageDynamoDbElasticsearchFieldPlugin");
var _shouldIgnoreEsResponseError = require("./shouldIgnoreEsResponseError");
var _logIgnoredEsResponseError = require("./logIgnoredEsResponseError");
/**
* This function removes attributes that were once present in the Page record, which we no longer need.
*/
function removePageAttributes(item) {
return (0, _omit.default)(item, ["home", "notFound", "visibility"]);
}
const createPageStorageOperations = params => {
const {
entity,
esEntity,
elasticsearch,
plugins
} = params;
const create = async params => {
const {
page,
input
} = params;
const versionKeys = {
PK: (0, _keys.createPartitionKey)(page),
SK: (0, _keys.createSortKey)(page)
};
const latestKeys = {
...versionKeys,
SK: (0, _keys.createLatestSortKey)()
};
const publishedKeys = {
...versionKeys,
SK: (0, _keys.createPublishedSortKey)()
};
const entityBatch = (0, _dbDynamodb.createEntityWriteBatch)({
entity,
put: [{
...page,
...versionKeys,
TYPE: (0, _keys.createBasicType)()
}, {
...page,
...latestKeys,
TYPE: (0, _keys.createLatestType)()
}]
});
const esData = (0, _helpers.getESLatestPageData)(plugins, page, input);
const elasticsearchEntityBatch = (0, _dbDynamodb.createEntityWriteBatch)({
entity: esEntity,
put: [{
index: _configurations.configurations.es(page).index,
data: esData,
...latestKeys
}]
});
if (page.status === "published") {
entityBatch.put({
...page,
...publishedKeys,
TYPE: (0, _keys.createPublishedType)()
});
entityBatch.put({
...page,
TYPE: (0, _keys.createPublishedPathType)(),
PK: (0, _keys.createPathPartitionKey)(page),
SK: (0, _keys.createPathSortKey)(page)
});
elasticsearchEntityBatch.put({
index: _configurations.configurations.es(page).index,
data: (0, _helpers.getESPublishedPageData)(plugins, page),
...publishedKeys
});
}
try {
await entityBatch.execute();
await elasticsearchEntityBatch.execute();
return page;
} catch (ex) {
throw new _error.default(ex.message || "Could not create new page.", ex.code || "CREATE_PAGE_ERROR", {
versionKeys,
latestKeys,
page
});
}
};
const createFrom = async params => {
const {
page,
latestPage,
original
} = params;
const versionKeys = {
PK: (0, _keys.createPartitionKey)(page),
SK: (0, _keys.createSortKey)(page)
};
const latestKeys = {
...versionKeys,
SK: (0, _keys.createLatestSortKey)()
};
const entityBatch = (0, _dbDynamodb.createEntityWriteBatch)({
entity,
put: [{
...page,
TYPE: (0, _keys.createBasicType)(),
...versionKeys
}, {
...page,
TYPE: (0, _keys.createLatestType)(),
...latestKeys
}]
});
const esData = (0, _helpers.getESLatestPageData)(plugins, page);
try {
await entityBatch.execute();
await (0, _dbDynamodb.put)({
entity: esEntity,
item: {
index: _configurations.configurations.es(page).index,
data: esData,
...latestKeys
}
});
return page;
} catch (ex) {
throw new _error.default(ex.message || "Could not create new page from existing page.", ex.code || "CREATE_PAGE_FROM_ERROR", {
versionKeys,
latestKeys,
latestPage,
original,
page
});
}
};
const update = async params => {
const {
original,
page,
input
} = params;
const keys = {
PK: (0, _keys.createPartitionKey)(page),
SK: (0, _keys.createSortKey)(page)
};
const latestKeys = {
...keys,
SK: (0, _keys.createLatestSortKey)()
};
const latestPage = await (0, _dbDynamodb.getClean)({
entity,
keys: latestKeys
});
const entityBatch = (0, _dbDynamodb.createEntityWriteBatch)({
entity,
put: [{
...page,
TYPE: (0, _keys.createBasicType)(),
...keys
}]
});
const esData = (0, _helpers.getESLatestPageData)(plugins, page, input);
if (latestPage && latestPage?.id === page.id) {
/**
* We also update the regular record.
*/
entityBatch.put({
...page,
TYPE: (0, _keys.createLatestType)(),
...latestKeys
});
}
/**
* Unfortunately we cannot push regular and es record in the batch write because they are two separate tables.
*/
try {
await entityBatch.execute();
await (0, _dbDynamodb.put)({
entity: esEntity,
item: {
index: _configurations.configurations.es(page).index,
data: esData,
...latestKeys
}
});
return page;
} catch (ex) {
throw new _error.default(ex.message || "Could not update existing page.", ex.code || "UPDATE_PAGE_ERROR", {
original,
page,
latestPage,
latestKeys,
keys
});
}
};
/**
* In case of delete, we must delete records:
* - revision
* - path if published
* Update:
* - latest
*/
const deleteOne = async params => {
const {
page,
latestPage,
publishedPage
} = params;
const partitionKey = (0, _keys.createPartitionKey)(page);
const entityBatch = (0, _dbDynamodb.createEntityWriteBatch)({
entity,
delete: [{
PK: partitionKey,
SK: (0, _keys.createSortKey)(page)
}]
});
const elasticsearchEntityBatch = (0, _dbDynamodb.createEntityWriteBatch)({
entity: esEntity
});
if (publishedPage && publishedPage.id === page.id) {
entityBatch.delete({
PK: partitionKey,
SK: (0, _keys.createPublishedSortKey)()
});
entityBatch.delete({
PK: (0, _keys.createPathPartitionKey)(page),
SK: (0, _keys.createPathSortKey)(page)
});
elasticsearchEntityBatch.delete({
PK: partitionKey,
SK: (0, _keys.createPublishedSortKey)()
});
}
let previousLatestPage = null;
if (latestPage && latestPage.id === page.id) {
const previousLatestRecord = await (0, _dbDynamodb.queryOne)({
entity,
partitionKey,
options: {
lt: (0, _keys.createSortKey)(latestPage),
reverse: true
}
});
if (previousLatestRecord) {
entityBatch.put({
...previousLatestRecord,
TYPE: (0, _keys.createLatestType)(),
PK: partitionKey,
SK: (0, _keys.createLatestSortKey)()
});
elasticsearchEntityBatch.put({
PK: partitionKey,
SK: (0, _keys.createLatestSortKey)(),
index: _configurations.configurations.es(page).index,
data: (0, _helpers.getESLatestPageData)(plugins, previousLatestRecord)
});
previousLatestPage = (0, _cleanup.cleanupItem)(entity, previousLatestRecord);
}
}
try {
await entityBatch.execute();
} catch (ex) {
throw new _error.default(ex.message || "Could not batch write all the page records.", ex.code || "BATCH_WRITE_RECORDS_ERROR");
}
try {
await elasticsearchEntityBatch.execute();
} catch (ex) {
throw new _error.default(ex.message || "Could not batch write all the page Elasticsearch records.", ex.code || "BATCH_WRITE_ELASTICSEARCH_RECORDS_ERROR");
}
return [page, previousLatestPage];
};
/**
* In case of deleteAll, we must delete records:
* - latest
* - published
* - path if published
* - revision
* - es latest
* - es published
*/
const deleteAll = async params => {
const {
page
} = params;
const partitionKey = (0, _keys.createPartitionKey)(page);
const queryAllParams = {
entity,
partitionKey,
options: {
gte: " "
}
};
let revisions;
try {
revisions = await (0, _dbDynamodb.queryAll)(queryAllParams);
} catch (ex) {
throw new _error.default(ex.message || "Could not query for all revisions of the page.", ex.code || "LIST_REVISIONS_ERROR", {
params: queryAllParams
});
}
/**
* We need to go through all possible entries and delete them.
* Also, delete the published entry path record.
*/
const entityBatch = (0, _dbDynamodb.createEntityWriteBatch)({
entity
});
const elasticsearchEntityBatch = (0, _dbDynamodb.createEntityWriteBatch)({
entity: esEntity
});
let publishedPathEntryDeleted = false;
for (const revision of revisions) {
if (revision.status === "published" && !publishedPathEntryDeleted) {
publishedPathEntryDeleted = true;
entityBatch.delete({
PK: (0, _keys.createPathPartitionKey)(page),
SK: revision.path
});
}
entityBatch.delete({
PK: revision.PK,
SK: revision.SK
});
}
elasticsearchEntityBatch.delete({
PK: partitionKey,
SK: (0, _keys.createLatestSortKey)()
});
/**
* Delete published record if it is published.
*/
if (publishedPathEntryDeleted) {
elasticsearchEntityBatch.delete({
PK: partitionKey,
SK: (0, _keys.createPublishedSortKey)()
});
}
try {
await entityBatch.execute();
} catch (ex) {
throw new _error.default(ex.message || "Could not delete all the page records.", ex.code || "DELETE_RECORDS_ERROR");
}
try {
await elasticsearchEntityBatch.execute();
} catch (ex) {
throw new _error.default(ex.message || "Could not delete all the page Elasticsearch records.", ex.code || "DELETE_ELASTICSEARCH_RECORDS_ERROR");
}
return [page];
};
const publish = async params => {
const {
page,
latestPage,
publishedPage
} = params;
page.status = "published";
/**
* Update the given revision of the page.
*/
const entityBatch = (0, _dbDynamodb.createEntityWriteBatch)({
entity,
put: [{
...page,
TYPE: (0, _keys.createBasicType)(),
PK: (0, _keys.createPartitionKey)(page),
SK: (0, _keys.createSortKey)(page)
}]
});
const elasticsearchEntityBatch = (0, _dbDynamodb.createEntityWriteBatch)({
entity: esEntity
});
/**
* If we are publishing the latest revision, update the latest revision
* status in ES. We also need to update the latest page revision entry in ES.
*/
if (latestPage.id === page.id) {
entityBatch.put({
...page,
TYPE: (0, _keys.createLatestType)(),
PK: (0, _keys.createPartitionKey)(page),
SK: (0, _keys.createLatestSortKey)()
});
elasticsearchEntityBatch.put({
PK: (0, _keys.createPartitionKey)(page),
SK: (0, _keys.createLatestSortKey)(),
index: _configurations.configurations.es(page).index,
data: (0, _helpers.getESLatestPageData)(plugins, page)
});
}
/**
* If we already have a published revision, and it's not the revision being published:
* - set the existing published revision to "unpublished"
*/
if (publishedPage && publishedPage.id !== page.id) {
entityBatch.put({
...publishedPage,
status: "unpublished",
PK: (0, _keys.createPartitionKey)(publishedPage),
SK: (0, _keys.createSortKey)(publishedPage)
});
/**
* Remove old published path if required.
*/
if (publishedPage.path !== page.path) {
entityBatch.delete({
PK: (0, _keys.createPathPartitionKey)(page),
SK: publishedPage.path
});
}
}
elasticsearchEntityBatch.put({
PK: (0, _keys.createPartitionKey)(page),
SK: (0, _keys.createPublishedSortKey)(),
index: _configurations.configurations.es(page).index,
data: (0, _helpers.getESPublishedPageData)(plugins, page)
});
/**
* Update or insert published path.
*/
entityBatch.put({
...page,
TYPE: (0, _keys.createPublishedPathType)(),
PK: (0, _keys.createPathPartitionKey)(page),
SK: (0, _keys.createPathSortKey)(page)
});
/**
* Update or insert published page.
*/
entityBatch.put({
...page,
TYPE: (0, _keys.createPublishedType)(),
PK: (0, _keys.createPartitionKey)(page),
SK: (0, _keys.createPublishedSortKey)()
});
try {
await entityBatch.execute();
} catch (ex) {
throw new _error.default(ex.message || "Could not update all the page records when publishing.", ex.code || "UPDATE_RECORDS_ERROR");
}
try {
await elasticsearchEntityBatch.execute();
} catch (ex) {
throw new _error.default(ex.message || "Could not update all the page Elasticsearch records when publishing.", ex.code || "UPDATE_ELASTICSEARCH_RECORDS_ERROR");
}
return page;
};
const unpublish = async params => {
const {
page,
latestPage
} = params;
page.status = "unpublished";
const entityBatch = (0, _dbDynamodb.createEntityWriteBatch)({
entity,
delete: [{
PK: (0, _keys.createPartitionKey)(page),
SK: (0, _keys.createPublishedSortKey)()
}, {
PK: (0, _keys.createPathPartitionKey)(page),
SK: (0, _keys.createPathSortKey)(page)
}],
put: [{
...page,
TYPE: (0, _keys.createBasicType)(),
PK: (0, _keys.createPartitionKey)(page),
SK: (0, _keys.createSortKey)(page)
}]
});
const elasticsearchEntityBatch = (0, _dbDynamodb.createEntityWriteBatch)({
entity: esEntity,
delete: [{
PK: (0, _keys.createPartitionKey)(page),
SK: (0, _keys.createPublishedSortKey)()
}]
});
/*
* If we are unpublishing the latest revision, let's also update the latest revision entry's status in ES.
*/
if (latestPage.id === page.id) {
entityBatch.put({
...page,
TYPE: (0, _keys.createLatestType)(),
PK: (0, _keys.createPartitionKey)(page),
SK: (0, _keys.createLatestSortKey)()
});
elasticsearchEntityBatch.put({
PK: (0, _keys.createPartitionKey)(page),
SK: (0, _keys.createLatestSortKey)(),
index: _configurations.configurations.es(page).index,
data: (0, _helpers.getESLatestPageData)(plugins, page)
});
}
try {
await entityBatch.execute();
} catch (ex) {
throw new _error.default(ex.message || "Could not update all the page records when unpublishing.", ex.code || "UPDATE_RECORDS_ERROR");
}
try {
await elasticsearchEntityBatch.execute();
} catch (ex) {
throw new _error.default(ex.message || "Could not update all the page Elasticsearch records when unpublishing.", ex.code || "UPDATE_ELASTICSEARCH_RECORDS_ERROR");
}
return page;
};
const get = async params => {
const {
where
} = params;
const {
pid,
id,
path,
published
} = where;
let {
version
} = where;
/**
* In case of having full ID and not having version we can take the version from the id.
*/
if (id && id.includes("#") && !version) {
version = Number(id.split("#").pop());
}
let partitionKey = null;
let sortKey;
if (path) {
partitionKey = (0, _keys.createPathPartitionKey)(where);
sortKey = path;
} else if (published) {
sortKey = (0, _keys.createPublishedSortKey)();
} else if (version) {
sortKey = (0, _keys.createSortKey)({
version
});
} else {
sortKey = (0, _keys.createLatestSortKey)();
}
/**
* If partition key is still undefined, create one with id or pid
*/
if (!partitionKey) {
partitionKey = (0, _keys.createPartitionKey)({
...where,
id: pid || id
});
}
const keys = {
PK: partitionKey,
SK: sortKey
};
try {
return await (0, _dbDynamodb.getClean)({
entity,
keys
});
} catch (ex) {
throw new _error.default(ex.message || "Could not load page by given params.", ex.code || "GET_PAGE_ERROR", {
where,
keys
});
}
};
const list = async params => {
/**
* We do not allow loading both published and latest at the same time.
* @see PageStorageOperationsListWhere
*/
if (params.where.published && params.where.latest) {
throw new _error.default("Both published and latest cannot be defined at the same time.", "MALFORMED_WHERE_ERROR", {
where: params.where
});
}
const {
after: previousCursor = null,
limit: initialLimit
} = params;
const limit = (0, _apiElasticsearch.createLimit)(initialLimit, 50);
const body = (0, _elasticsearchQueryBody.createElasticsearchQueryBody)({
...params,
where: {
...params.where
},
limit,
after: previousCursor,
plugins
});
let searchPlugins = [];
if (params.where.published) {
searchPlugins = plugins.byType(_SearchPublishedPagesPlugin.SearchPublishedPagesPlugin.type);
} else if (params.where.latest) {
searchPlugins = plugins.byType(_SearchLatestPagesPlugin.SearchLatestPagesPlugin.type);
} else {
throw new _error.default("Only published or latest can be listed. Missing where condition.", "MALFORMED_WHERE_ERROR", {
where: params.where
});
}
for (const plugin of searchPlugins) {
/**
* Apply query modifications
*/
plugin.modifyQuery({
query: body.query,
args: params,
plugins
});
/**
* Apply sort modifications
*/
plugin.modifySort({
sort: body.sort,
args: params,
plugins
});
}
let response;
const esConfig = _configurations.configurations.es(params.where);
try {
response = await elasticsearch.search({
...esConfig,
body
});
} catch (ex) {
/**
* Do not throw the error if Elasticsearch index does not exist.
* In some CRUDs we try to get list of pages but index was not created yet.
*/
if ((0, _shouldIgnoreEsResponseError.shouldIgnoreEsResponseError)(ex)) {
(0, _logIgnoredEsResponseError.logIgnoredEsResponseError)({
error: ex,
indexName: esConfig.index
});
return {
items: [],
meta: {
hasMoreItems: false,
totalCount: 0,
cursor: null
}
};
}
throw new _error.default(ex.message || "Could not load pages by given Elasticsearch body.", ex.code || "LIST_PAGES_ERROR", {
body
});
}
const {
hits,
total
} = response.body.hits;
const items = hits.map(item => item._source).map(item => removePageAttributes(item));
const hasMoreItems = items.length > limit;
if (hasMoreItems) {
/**
* Remove the last item from results, we don't want to include it.
*/
items.pop();
}
/**
* Cursor is the `sort` value of the last item in the array.
* https://www.elastic.co/guide/en/elasticsearch/reference/current/paginate-search-results.html#search-after
*/
const cursor = items.length > 0 && hasMoreItems ? (0, _apiElasticsearch.encodeCursor)(hits[items.length - 1].sort) || null : null;
return {
items,
meta: {
hasMoreItems,
totalCount: total.value,
cursor
}
};
};
const listTags = async params => {
const {
where
} = params;
const tenant = where.tenant;
const body = (0, _elasticsearchQueryBody.createElasticsearchQueryBody)({
...params,
where: {
locale: where.locale,
search: undefined,
tenant
},
sort: [],
after: null,
limit: 100000,
plugins
});
const esConfig = _configurations.configurations.es(where);
try {
const response = await elasticsearch.search({
...esConfig,
body: {
...body,
sort: undefined,
limit: undefined,
size: 0,
aggs: {
tags: {
terms: {
field: "tags.keyword",
include: `.*${where.search}.*`,
size: 10
}
}
}
}
});
const tags = response.body.aggregations["tags"];
if (!tags || Array.isArray(tags.buckets) === false) {
return [];
}
return tags.buckets.map(item => item.key);
} catch (ex) {
if ((0, _shouldIgnoreEsResponseError.shouldIgnoreEsResponseError)(ex)) {
(0, _logIgnoredEsResponseError.logIgnoredEsResponseError)({
error: ex,
indexName: esConfig.index
});
return [];
}
throw new _error.default(ex.message || "Could not list tags by given parameters.", ex.code || "LIST_TAGS_ERROR", {
body,
where
});
}
};
const listRevisions = async params => {
const {
where,
sort
} = params;
const queryAllParams = {
entity,
partitionKey: (0, _keys.createPartitionKey)({
...where,
id: where.pid
}),
options: {
beginsWith: "REV#",
reverse: false
}
};
let items = [];
try {
items = await (0, _dbDynamodb.queryAll)(queryAllParams);
} catch (ex) {
throw new _error.default(ex.message || "Could not load all the revisions from requested page.", ex.code || "LOAD_PAGE_REVISIONS_ERROR", {
params
});
}
const fields = plugins.byType(_PageDynamoDbElasticsearchFieldPlugin.PageDynamoDbElasticsearchFieldPlugin.type);
return (0, _dbDynamodb.sortItems)({
items: items.map(item => removePageAttributes(item)),
fields,
sort
});
};
return {
create,
createFrom,
update,
delete: deleteOne,
deleteAll: deleteAll,
publish,
unpublish,
get,
list,
listRevisions,
listTags
};
};
exports.createPageStorageOperations = createPageStorageOperations;
//# sourceMappingURL=index.js.map