@botonic/plugin-contentful
Version:
Botonic Plugin Contentful is one of the **[available](https://github.com/hubtype/botonic/tree/master/packages)** plugins for Botonic. **[Contentful](http://www.contentful.com)** is a CMS (Content Management System) which manages contents of a great variet
178 lines • 6.51 kB
JavaScript
import { __asyncValues, __awaiter } from "tslib";
import parser from 'csv-parse';
import * as fs from 'fs';
import { CmsException } from '../../cms';
import { CONTENT_FIELDS, ContentFieldType, ContentFieldValueType, } from '../../manage-cms/fields';
import { replaceAll, trim } from '../../nlp/util/strings';
export const PARSE_OPTIONS = {
escape: '"',
delimiter: ';',
quote: '"',
columns: ['Model', 'Code', 'Id', 'Field', 'from', 'to'],
relaxColumnCountMore: true,
bom: true,
};
export function recordId(record) {
return `'${record.Id}/${record.Code}'`;
}
export class CsvImport {
constructor(importer, options = {}) {
this.importer = importer;
this.options = options;
}
import(options) {
var _a, e_1, _b, _c;
return __awaiter(this, void 0, void 0, function* () {
const parse = new parser.Parser(PARSE_OPTIONS);
const reader = fs.createReadStream(options.fname).pipe(parse);
const header = new CsvImport.HeaderSkipper();
const records = [];
try {
for (var _d = true, _e = __asyncValues(reader), _f; _f = yield _e.next(), _a = _f.done, !_a;) {
_c = _f.value;
_d = false;
try {
const record = _c;
if (header.skipFirst(record)) {
this.extraColumnsMustBeEmpty(record);
continue;
}
records.push(record);
}
finally {
_d = true;
}
}
}
catch (e_1_1) { e_1 = { error: e_1_1 }; }
finally {
try {
if (!_d && !_a && (_b = _e.return)) yield _b.call(_e);
}
finally { if (e_1) throw e_1.error; }
}
this.sortRecords(records);
yield this.importRecords(options, records);
});
}
sortRecords(records) {
records.sort((r1, r2) => {
const comp = r1.Model.localeCompare(r2.Model);
if (comp !== 0)
return comp;
return r1.Code.localeCompare(r2.Code);
});
}
importRecords(options, records) {
var _a;
return __awaiter(this, void 0, void 0, function* () {
for (const record of records) {
const recordId = `'${record.Model}' field '${record.Code.trim() || record.Id}.${record.Field}'`;
if ((_a = options.ignoreContentIds) === null || _a === void 0 ? void 0 : _a.includes(record.Id)) {
console.info(`Skipping ${recordId}`);
continue;
}
if (record.from &&
!record.to &&
record.Field != ContentFieldType.SHORT_TEXT //see ImportContentUpdater.mustBeDeleted
) {
console.warn(`Missing translation for ${recordId}`);
console.warn('To remove the content for a locale, remove the keywords and any link to it');
}
if (this.options.nameFilter && !this.options.nameFilter(record.Code)) {
continue;
}
console.log(`Importing ${recordId}`);
yield this.importer.consume(record);
}
yield this.importer.flush();
});
}
extraColumnsMustBeEmpty(record) {
const cols = PARSE_OPTIONS.columns;
if (Object.keys(record).length == cols.length) {
return;
}
for (let i = cols.length; i < Object.keys(record).length; i++) {
if (Object.values(i)) {
throw new CmsException('Too many columns');
}
}
}
}
CsvImport.HeaderSkipper = class {
constructor() {
this.first = true;
}
skipFirst(record) {
if (this.first) {
this.first = false;
if (record.Model == 'Model') {
console.log('Skipping header');
return true;
}
console.error('No header found. Parsing fist line as a normal line');
}
return false;
}
};
export class RecordFixer {
constructor(record) {
this.record = record;
this.field = CONTENT_FIELDS.get(record.Field);
}
fix() {
this.fixCapitals();
this.fixExcelNewLine();
if (this.field.valueType == ContentFieldValueType.STRING_ARRAY) {
this.checkKeywordLen();
this.removeLastEmptyItem();
}
}
fixCapitals() {
const firstChar = this.record.Model[0];
if (firstChar === firstChar.toLowerCase()) {
return;
}
this.record.Model = this.record.Model.toLowerCase();
}
fixTrimmingQuotes() {
//it's typical to forget a " at the first character of the value
this.record.to = trim(this.record.to, '" ');
}
fixExcelNewLine() {
//https://bugs.documentfoundation.org/show_bug.cgi?id=118470
this.record.to = replaceAll(this.record.to, '_x000D_', '');
}
fixStringArraySeparators(stringArray) {
for (const c of ',.?') {
if (stringArray.indexOf(c) >= 0) {
this.complain(`... Replacing '${c}' with ;`);
stringArray = replaceAll(stringArray, c, ';');
}
}
return stringArray;
}
checkKeywordLen() {
if (this.field.fieldType == ContentFieldType.KEYWORDS) {
const fromLen = this.field.parse(this.record.from).length;
const toLen = this.field.parse(this.record.to).length;
if (toLen != fromLen || fromLen == 1) {
this.complain(`'From' has ${fromLen} keywords:\n${this.record.from}\n ` +
`but 'to' has ${toLen}:\n${this.record.to}`);
this.record.to = this.fixStringArraySeparators(this.record.to);
}
}
}
removeLastEmptyItem() {
const emptyLastItem = /; *$/;
if (emptyLastItem.exec(this.record.to)) {
this.complain('Removing final empty keyword');
this.record.to = this.record.to.replace(emptyLastItem, '');
}
}
complain(msg) {
console.error(`Problem in ${this.record.Id} / ${this.record.Code}: ${msg}`);
}
}
//# sourceMappingURL=csv-import.js.map