@extjs/sencha-cmd-linux-32/dist/js/node_modules/fashion/src/parse/Tokenizer.js

Productivity and performance optimization tool for building applications with Sencha Ext JS and Sencha Touch.

"use strict";

var Fashion = require('../export/Base.js'),
    Base = Fashion.Base;

var Scanner = require('./Scanner.js');

class Tokenizer extends Base {
    tokenize(config) {
        var scanner = new Scanner(config),
            tokens = this.tokens,
            token;

        if (!tokens) {
            this.tokens = tokens = [];

            while ((token = scanner.next()) !== undefined) {
                tokens.push(token);
            }
        }

        return tokens;
    }
}

Tokenizer.prototype.tokens = undefined;

module.exports = Tokenizer;