@antv/data-wizard
Version:
A js/ts library for data processing
341 lines (340 loc) • 11.9 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.analyzeField = exports.isTime = exports.isNominal = exports.isInterval = exports.isContinuous = exports.isDiscrete = exports.isUnique = exports.isOrdinal = exports.isConst = exports.analyzeType = exports.analyzeDate = exports.analyzeNumber = exports.analyzeString = exports.isDateFieldInfo = exports.isNumberFieldInfo = exports.isStringFieldInfo = void 0;
var tslib_1 = require("tslib");
var utils = tslib_1.__importStar(require("../utils"));
var statistics = tslib_1.__importStar(require("../statistics"));
var is_date_1 = require("./is-date");
/**
* @public
*/
function isStringFieldInfo(x) {
return x.recommendation === 'string';
}
exports.isStringFieldInfo = isStringFieldInfo;
/**
* @public
*/
function isNumberFieldInfo(x) {
return x.recommendation === 'integer' || x.recommendation === 'float';
}
exports.isNumberFieldInfo = isNumberFieldInfo;
/**
* @public
*/
function isDateFieldInfo(x) {
return x.recommendation === 'date';
}
exports.isDateFieldInfo = isDateFieldInfo;
function analyzeString(array) {
var lenArray = array.map(function (item) { return item.length; });
return {
maxLength: statistics.max(lenArray),
minLength: statistics.min(lenArray),
meanLength: statistics.mean(lenArray),
containsChar: array.some(function (item) { return /[A-z]/.test(item); }),
containsDigit: array.some(function (item) { return /[0-9]/.test(item); }),
containsSpace: array.some(function (item) { return /\s/.test(item); }),
};
}
exports.analyzeString = analyzeString;
function analyzeNumber(array) {
return {
minimum: statistics.min(array),
maximum: statistics.max(array),
mean: statistics.mean(array),
percentile5: statistics.quantile(array, 5),
percentile25: statistics.quantile(array, 25),
percentile50: statistics.quantile(array, 50),
percentile75: statistics.quantile(array, 75),
percentile95: statistics.quantile(array, 95),
sum: statistics.sum(array),
variance: statistics.variance(array),
standardDeviation: statistics.standardDeviation(array),
zeros: array.filter(function (item) { return item === 0; }).length,
};
}
exports.analyzeNumber = analyzeNumber;
function analyzeDate(array, isInteger) {
if (isInteger === void 0) { isInteger = false; }
var list = array.map(function (item) {
if (isInteger) {
var str = "" + item;
if (str.length === 8)
return new Date(str.substr(0, 4) + "/" + str.substr(4, 2) + "/" + str.substr(6, 2)).getTime();
}
return new Date(item).getTime();
});
return {
minimum: array[statistics.minIndex(list)],
maximum: array[statistics.maxIndex(list)],
};
}
exports.analyzeDate = analyzeDate;
/**
* Determine what type a value is, may be one of [integer float date string null]
*/
function analyzeType(value) {
if (utils.isNull(value))
return 'null';
if (utils.isNumber(value)) {
if (utils.isInteger(value))
return 'integer';
return 'float';
}
if (utils.isString(value)) {
if (utils.isDigit(value)) {
if (value.includes('.'))
return 'float';
return 'integer';
}
}
if (utils.isDate(value, true))
return 'date';
return 'string';
}
exports.analyzeType = analyzeType;
/**
* Checks if field is constant
* @param info - The {@link FieldInfo} to processs
* @public
*/
function isConst(info) {
return info.distinct === 1;
}
exports.isConst = isConst;
/**
* Checks if field is an ordinal
* @param info - Field Info
* @public
*/
function isOrdinal(info) {
var rawData = info.rawData, recommendation = info.recommendation;
if (recommendation !== 'string')
return false;
if (isConst(info))
return false;
var list = rawData.filter(function (item) { return !utils.isNull(item) && utils.isBasicType(item); });
if (list.length === 0)
return false;
var start = null;
var end = null;
var startIndex = -1;
var endIndex = -1;
var through = true;
while (through) {
var through_1 = true;
for (var i = 0; i < list.length; i += 1) {
var item = list[i];
var char = item[startIndex + 1];
if (start === null || i === 0)
start = char;
if (char !== start) {
through_1 = false;
break;
}
}
if (!through_1)
break;
startIndex += 1;
}
through = true;
while (through) {
var through_2 = true;
for (var i = 0; i < list.length; i += 1) {
var item = list[i];
var char = item[item.length - 1 - (endIndex + 1)];
if (end === null || i === 0)
end = char;
if (char !== end) {
through_2 = false;
break;
}
}
if (!through_2)
break;
endIndex += 1;
}
var partners = [/\d+/, /(零|一|二|三|四|五|六|七|八|九|十)+/, /(一|二|三|四|五|六|日)/, /^[a-z]$/, /^[A-Z]$/];
if (startIndex === -1 && endIndex === -1)
return false;
var arr = list.map(function (item) {
return item.slice(startIndex === -1 ? 0 : startIndex + 1, endIndex === -1 ? undefined : item.length - endIndex - 1);
});
var _loop_1 = function (i) {
var p = partners[i];
var notMatch = arr.some(function (item) { return !p.test(item); });
if (!notMatch)
return { value: true };
};
for (var i = 0; i < partners.length; i += 1) {
var state_1 = _loop_1(i);
if (typeof state_1 === "object")
return state_1.value;
}
return false;
}
exports.isOrdinal = isOrdinal;
/**
* Checks if field is an unique
* @param info - The {@link FieldInfo} to processs
* @public
*/
function isUnique(info) {
return info.distinct === info.count;
}
exports.isUnique = isUnique;
/**
* Checks if field is discrete
* @remarks
* @param info - The {@link FieldInfo} to processs
* @public
*/
function isDiscrete(info) {
return info.recommendation === 'integer';
}
exports.isDiscrete = isDiscrete;
/**
* Checks if field is a continuous.
* @param info - The {@link FieldInfo} to processs
* @public
*/
function isContinuous(info) {
return info.recommendation === 'float';
}
exports.isContinuous = isContinuous;
/**
* Checks if field is an interval
* @param info - The {@link FieldInfo} to processs
* @public
*/
function isInterval(info) {
return info.recommendation === 'integer' || info.recommendation === 'float';
}
exports.isInterval = isInterval;
/**
* Checks if field is a nominal
* @param info - The {@link FieldInfo} to processs
* @public
*/
function isNominal(info) {
if (info.recommendation === 'boolean')
return true;
if (info.recommendation === 'string')
return !isOrdinal(info);
return false;
}
exports.isNominal = isNominal;
/**
* Checks if field is a time
* @param info - Field Info
* @public
*/
function isTime(info) {
return info.recommendation === 'date';
}
exports.isTime = isTime;
/**
* Analysis field type
* @param array - data
* @public
*/
function analyzeField(array) {
var list = array.map(function (item) { return (utils.isNull(item) ? null : item); });
var valueMap = statistics.valueMap(list);
var recommendation;
var nonNullArray = valueMap.null ? list.filter(function (item) { return item !== null; }) : list;
var typeArray = list.map(function (item) { return analyzeType(item); });
var types = Object.keys(statistics.valueMap(typeArray)).filter(function (item) { return item !== 'null'; });
// generate recommendation
switch (types.length) {
case 0:
recommendation = 'null';
break;
case 1:
recommendation = types[0];
// an integer field may be a date field
if (recommendation === 'integer') {
var data = list.filter(function (item) { return item !== null; });
if (data.map(function (num) { return "" + num; }).every(function (str) { return is_date_1.isDateString(str); })) {
recommendation = 'date';
}
}
break;
case 2:
if (types.includes('integer') && types.includes('float'))
recommendation = 'float';
else
recommendation = 'string';
break;
default:
recommendation = 'string';
}
var uniqueArray = utils.unique(nonNullArray);
var fieldInfo = {
count: array.length,
distinct: uniqueArray.length,
type: types.length <= 1 ? types[0] || 'null' : 'mixed',
recommendation: recommendation,
missing: valueMap.null || 0,
rawData: array,
valueMap: valueMap,
};
if (types.length > 1) {
var meta_1 = {};
var restNotNullArray_1 = nonNullArray;
types.forEach(function (item) {
if (item === 'date') {
meta_1.date = analyzeField(restNotNullArray_1.filter(function (item) { return utils.isDate(item, true); }));
restNotNullArray_1 = restNotNullArray_1.filter(function (item) { return !utils.isDate(item, true); });
}
else if (item === 'integer') {
meta_1.integer = analyzeField(restNotNullArray_1.filter(function (item) { return utils.isInteger(item, true); }));
restNotNullArray_1 = restNotNullArray_1.filter(function (item) { return !utils.isInteger(item, true); });
}
else if (item === 'float') {
meta_1.float = analyzeField(restNotNullArray_1.filter(function (item) { return utils.isFloat(item, true); }));
restNotNullArray_1 = restNotNullArray_1.filter(function (item) { return !utils.isFloat(item, true); });
}
else if (item === 'string') {
meta_1.string = analyzeField(restNotNullArray_1.filter(function (item) { return analyzeType(item) === 'string'; }));
restNotNullArray_1 = restNotNullArray_1.filter(function (item) { return analyzeType(item) !== 'string'; });
}
});
fieldInfo.meta = meta_1;
}
if (fieldInfo.distinct === 2 && fieldInfo.recommendation !== 'date') {
// temporarily threshold
if (list.length >= 100) {
fieldInfo.recommendation = 'boolean';
}
else if (utils.isBoolean(uniqueArray, true)) {
fieldInfo.recommendation = 'boolean';
}
}
if (recommendation === 'string') {
Object.assign(fieldInfo, analyzeString(nonNullArray.map(function (item) { return "" + item; })));
}
if (recommendation === 'integer' || recommendation === 'float') {
Object.assign(fieldInfo, analyzeNumber(nonNullArray.map(function (item) { return item * 1; })));
}
if (recommendation === 'date') {
Object.assign(fieldInfo, analyzeDate(nonNullArray, fieldInfo.type === 'integer'));
}
var levelOfMeasurements = [];
if (isNominal(fieldInfo))
levelOfMeasurements.push('Nominal');
if (isOrdinal(fieldInfo))
levelOfMeasurements.push('Ordinal');
if (isInterval(fieldInfo))
levelOfMeasurements.push('Interval');
if (isDiscrete(fieldInfo))
levelOfMeasurements.push('Discrete');
if (isContinuous(fieldInfo))
levelOfMeasurements.push('Continuous');
if (isTime(fieldInfo))
levelOfMeasurements.push('Time');
fieldInfo.levelOfMeasurements = levelOfMeasurements;
return fieldInfo;
}
exports.analyzeField = analyzeField;