@tensorflow/tfjs-layers/dist/layers/nlp/models/gpt2/gpt2_backbone.d.ts

TensorFlow layers API in JavaScript

/**
 * @license
 * Copyright 2023 Google LLC.
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 * =============================================================================
 */
/// <amd-module name="@tensorflow/tfjs-layers/dist/layers/nlp/models/gpt2/gpt2_backbone" />
/**
 *  Base class for Backbone models.
 */
import { serialization } from '@tensorflow/tfjs-core';
import { Embedding } from '../../../embeddings';
import { Backbone } from '../backbone';
export interface GPT2BackboneArgs {
    /**
     * Integer. The size of the token vocabulary.
     */
    vocabularySize: number;
    /**
     * Integer. The number of transformer layers.
     */
    numLayers: number;
    /**
     * Integer. The number of attention heads for each transformer.
     * The hidden size must be divisible by the number of attention heads.
     */
    numHeads: number;
    /**
     * Integer. The size of the transformer encoding and pooler layers.
     */
    hiddenDim: number;
    /**
     * Integer. The output dimension of the first Dense layer in a two-layer
     * feedforward network for each transformer.
     */
    intermediateDim: number;
    /**
     * Float. Dropout probability for the Transformer encoder.
     * Defaults to 0.2.
     */
    dropout?: number;
    /**
     * Integer. The maximum sequence length that this encoder can consume.
     * If `null`, `maxSequenceLength` uses the value from sequence length.
     * This determines the variable shape for positional embeddings.
     * Defaults to 1024.
     */
    maxSequenceLength?: number;
}
/**
 * GPT-2 core network with hyperparameters.
 *
 * This network implements a Transformer-based decoder network,
 * Generative Pretrained Transformer-2 (GPT-2), as described in
 * ["Language Models are Unsupervised Multitask Learners"](https://cdn.openai.com/better-language-models/language_models_are_unsupervised_multitask_learners.pdf).
 * It includes the embedding lookups and transformer layers.
 *
 * The default constructor gives a fully customizable, randomly initialized
 * GPT-2 model with any number of layers, heads, and embedding
 * dimensions. To load preset architectures and weights, use the `fromPreset`
 * constructor.
 *
 * Disclaimer: Pre-trained models are provided on an "as is" basis, without
 * warranties or conditions of any kind. The underlying model is provided by a
 * third party and subject to a separate license, available
 * [here](https://github.com/openai/gpt-2).
 *
 *
 * Example usage:
 * ```js
 * const tokenIds = tf.ones([1, 12]), dtype="int32");
 * const paddingMask = tf.tensor(
 *  [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0]], 'int32');
 *
 * # Pretrained GPT-2 decoder.
 * model = GPT2Backbone.fromPreset("gpt2_base_en");
 * model.apply(inputData, {paddingMask});
 *
 * # Randomly initialized GPT-2 decoder with custom config.
 * model = kerasNlp.models.GPT2Backbone({
 *     vocabularySize: 50257,
 *     numLayers: 12,
 *     numHeads: 12,
 *     hiddenDim: 768,
 *     intermediateDim: 3072,
 *     maxSequenceLength: 1024,
 * });
 * model.apply(inputData, {paddingMask});
 * ```
 */
export declare class GPT2Backbone extends Backbone {
    /** @nocollapse */
    static className: string;
    private vocabularySize;
    private numLayers;
    private numHeads;
    private hiddenDim;
    private intermediateDim;
    private dropout;
    private maxSequenceLength;
    constructor(args: GPT2BackboneArgs);
    getConfig(): serialization.ConfigDict;
    get tokenEmbedding(): Embedding;
}