@itentialopensource/adapter-openstack_keystone/swagger/openstack_api.js

This adapter integrates with system described as: Openstack Keystone.

/* eslint-disable */
const fs = require('fs');
const util = require('util');
const parse5 = require('parse5');
const htmlparser2 = require("htmlparser2");
const domutils = require("domutils");
const { Parser } = require("htmlparser2");
const { DomHandler } = require("domhandler");

const inspect = obj => util.inspect(obj, { depth: Infinity });

const htmlFile = fs.readFileSync('./openstack_API_docs/IdentityAPIv3-CURRENT-keystone-documentation.html', { encoding: 'utf-8' })
const outputFileName = 'keystone-openapi.json'

const changeToCamelCase = (s) => {
    let operationId = "";
    s.split(" ").forEach((word, index) => {
        let firstLetter;
        if (index === 0) {
            firstLetter = word[0].toLowerCase()
        } else {
            firstLetter = word[0].toUpperCase()
        }
        operationId += firstLetter + word.slice(1);
    })
    return operationId;
}

const getTag = (node) => {
    const siblings = domutils.getSiblings(node);
    const h2 = domutils.filter(elem => elem.name === 'h2', siblings);
    let tag = '';
    if (h2.length !== 1) {
        console.error('Can not obtain tag')
        return tag;
    }
    const tagNode = h2[0];
    const tagNodetext = domutils.getChildren(tagNode).filter(elem => elem.type === 'text', tagNode);
    tag = domutils.innerText(tagNodetext).trim();
    return tag;
}

const getRestMethod = (node) => {
    const n = domutils.findOne((e => e.name === 'span' && e.attribs.class.split(' ').find(item => item === 'label')), [node]);
    const restMethod = domutils.innerText(n).trim().toLocaleLowerCase();
    return restMethod;
}

const getPath = (node) => {
    const n = domutils.findOne((e => e.name === 'div' && e.attribs.class === 'endpoint-container'), [node]);
    const childs = domutils.getElementsByTagName('div', domutils.getChildren(n), false);
    const pathNode = childs[0];
    const path = domutils.innerText(pathNode).trim();
    return path;
}

const getDescription = (node) => {
    const n = domutils.findOne((e => e.name === 'div' && e.attribs.class === 'endpoint-container'), [node]);
    const childs = domutils.getElementsByTagName('div', domutils.getChildren(n), false);
    const descriptionNode = childs[1];
    const path = domutils.innerText(descriptionNode).trim();
    return path;
}

const getDetailsLink = (node) => {
    const n = domutils.findOne((e => e.name === 'div' && e.attribs.class === 'operation'), [node]);
    const childs = domutils.getElementsByTagName('a', domutils.getChildren(n), true);
    if (childs.length !== 1) {
        return 'No link found';
    }
    const detailsLinkNode = childs[0];
    const detailsLink = detailsLinkNode.attribs.href;
    return detailsLink;
}

const getRequest = (node) => {
    const parameters = [];
    const sections = domutils.getElementsByTagName('section', domutils.getChildren(node), false);

    let requestSection = sections.find(s => {
        let textNodes = domutils.getElementsByTagName('h4', domutils.getChildren(s), false);
        return domutils.textContent(textNodes).includes('Request');
    });

    if (!requestSection) return parameters;

    const subSections = domutils.getElementsByTagName('section', domutils.getChildren(requestSection), false);
    let requestParamtersSection = subSections.find(s => {
        let textNodes = domutils.getElementsByTagName('h5', domutils.getChildren(s), false);
        return domutils.textContent(textNodes).includes('Parameters');
    });

    // TODO Add support for multiple requests/responses
    const table = domutils.getElementsByTagName('tbody', requestParamtersSection || requestSection);
    const allRows = domutils.getElementsByTagName('tr', table);

    allRows.forEach((row) => {
        let parameter = {};
        let columns = domutils.getElementsByTagName('p', row);
        columns.forEach((c, index) => {
            let t = domutils.innerText(c).replace(/[\r\n]\s+/g, ' ').trim();
            switch (index) {
                case 0:
                    parameter.name = t;
                    break;
                case 1:
                    parameter.in = t;
                    break;
                case 2:
                    parameter.type = t;
                    break;
                case 3:
                    parameter.description = t;
                    break;
            }
        });
        parameters.push(parameter);
    });
    return parameters;
}

const getResponse = (node) => {
    let response = [];
    const sections = domutils.getElementsByTagName('section', domutils.getChildren(node), false);
    
    let responseSection = sections.find(s => {
        let textNodes = domutils.getElementsByTagName('h4', domutils.getChildren(s), false);
        return domutils.textContent(textNodes).includes('Response');
    });
    if (!responseSection) return response;

    const subSections = domutils.getElementsByTagName('section', domutils.getChildren(responseSection), false);
    let responseParamtersSection = subSections.find(s => {
        let textNodes = domutils.getElementsByTagName('h5', domutils.getChildren(s), false);
        return domutils.textContent(textNodes).includes('Parameters');
    });
    if (!responseParamtersSection) return response
    // TODO Add support for multiple requests/responses
    const table = domutils.getElementsByTagName('tbody', responseParamtersSection);
    const allRows = domutils.getElementsByTagName('tr', table);

    allRows.forEach((row) => {
        let parameter = {};
        let columns = domutils.getElementsByTagName('p', row);
        columns.forEach((c, index) => {
            let t = domutils.innerText(c).replace(/[\r\n]\s+/g, ' ').trim();
            switch (index) {
                case 0:
                    parameter.name = t;
                    break;
                case 1:
                    parameter.in = t;
                    break;
                case 2:
                    parameter.type = t;
                    break;
                case 3:
                    parameter.description = t;
                    break;
            }
        });
        response.push(parameter);
    });
    return response;
}

const getStatusCodes = (node) => {
    let statusCodes = [];
    const sections = domutils.getElementsByTagName('section', domutils.getChildren(node), true);

    let responseSection = sections.find(s => {
        let textNodes = domutils.getElementsByTagName('h4', domutils.getChildren(s), false);
        return domutils.textContent(textNodes).includes('Response');
    });

    let statusCodesSection = sections.find(s => {
        let textNodes = domutils.getElementsByTagName('h5', domutils.getChildren(s), false);
        return domutils.textContent(textNodes).includes('Status Codes');
    });

    let statusCodesSuccessSection = sections.find(s => {
        let textNodes = domutils.getElementsByTagName('h5', domutils.getChildren(s), false);
        return domutils.textContent(textNodes).includes('Success');
    });

    let statusCodesErrorSection = sections.find(s => {
        let textNodes = domutils.getElementsByTagName('h5', domutils.getChildren(s), false);
        return domutils.textContent(textNodes).includes('Error');
    });

    if (statusCodesSuccessSection) {
        const rows = domutils.getElementsByTagName('td', domutils.getChildren(statusCodesSuccessSection));
        rows.forEach((r, index) => {
            if (index % 2 === 0) {
                statusCodes.push({ code: domutils.innerText(r).trim(), reason: domutils.innerText(rows[index + 1]).trim() })
            }
        })
    }
    if (statusCodesErrorSection) {
        const rows = domutils.getElementsByTagName('td', domutils.getChildren(statusCodesErrorSection));
        rows.forEach((r, index) => {
            if (index % 2 === 0) {
                statusCodes.push({ code: domutils.innerText(r).trim(), reason: domutils.innerText(rows[index + 1]).trim() })
            }
        })
    }

    if (statusCodesSection && !statusCodesSuccessSection && !statusCodesErrorSection) {
        const rows = domutils.getElementsByTagName('td', domutils.getChildren(statusCodesSection));
        rows.forEach((r, index) => {
            if (index % 2 === 0) {
                statusCodes.push({ code: domutils.innerText(r).trim(), reason: domutils.innerText(rows[index + 1]).trim() })
            }
        })
    }

    if (!statusCodesSuccessSection && !statusCodesSection && responseSection) {
        statusCodes.push({ code: "200 - OK", reason: "Request was successful." })
    }
    return statusCodes;
}

const extractData = (detailControlNodes, apiDetailControlNodes) => {
    const N = detailControlNodes.length;
    let data = [];
    console.log("Operations found:");
    for (let i = 0; i < N; i += 1) {
        console.log('\n======= OPERATION ========\n');
        const node = detailControlNodes[i];
        const detailsNode = apiDetailControlNodes[i];

        const tag = getTag(node);
        const restMethod = getRestMethod(node);
        const path = getPath(node);
        const description = getDescription(node);
        const detailsLink = getDetailsLink(node);
        console.log(`> Operation:\n| ${restMethod} | ${path} | ${description} | ${detailsLink} |`);
        const request = getRequest(detailsNode);
        console.log(`> Request parameters:\n|${inspect(request)}|`);
        const response = getResponse(detailsNode);
        console.log(`> Response parameters:\n|${inspect(response)}|`);        
        const statusCodes = getStatusCodes(detailsNode);
        console.log(`> Response status codes:\n|${inspect(statusCodes)}|`);
        data.push({ restMethod, path, description, detailsLink, response, request, statusCodes, tag });
    }
    return data;
}

const saveToFile = (obj) => {
    fs.writeFileSync(outputFileName, JSON.stringify(obj));
    console.log(`File '${outputFileName}' saved.`)
}

const getBaseOpenApiObj = () => {
    const openApiObj = {
        openapi: "3.0.0",
        info: {
            version: "3.14",
            title: "Openstack Keystone API"
        },
        servers: [
            {
                url: 'http://{host}:{port}',
                description: 'Identity service (keystone) administrative endpoint',
                variables: {
                    host: {
                        default: "localhost"
                    },
                    port: {
                        default: "5000"
                    }
                }
            }
        ],
        paths: {
        },
        components: { schemas: {}, responses: {}, parameters: {}, examples: {}, requestBodies: {}, headers: {}, securitySchemes: {}, links: {}, callbacks: {} },
        security: [],
        tags: [],
        externalDocs: {
            description: 'Openstack Keystone Identity API Operations',
            url: 'https://docs.openstack.org/api-ref/identity/v3/#'
        }
    }

    openApiObj.components.securitySchemes = {
        token: {
            type: "apiKey",
            description: 'Token authentication',
            name: "X-Auth-Token",
            in: "header"
        }
    }

    openApiObj.security = [{ "token": [] }];

    return openApiObj;
}

const buildQueryParameters = (operation) => {
    let params = []
    let queryParams = operation.filter((item) => item.in === 'query')
    queryParams.forEach((p) => {
        isOptional = false
        if (p.name.indexOf('Optional') !== -1) {
            isOptional = true;
            p.name = p.name.split(' ')[0].trim();
        }
        if (p.type.includes('key-only')) {
            params.push({
                "name": p.name,
                "in": p.in,
                "required": !isOptional,
                "schema": {
                    "type": 'boolean',
                    "description": p.description
                },
                "allowEmptyValue": true
            })
        } else {
            if (p.type.includes('bool')) p.type = 'boolean'
            params.push({
                "name": p.name,
                "in": p.in,
                "required": !isOptional,
                "schema": {
                    "type": p.type,
                    "description": p.description
                }
            })
        }
    });
    return params;
}

const buildHeaderParameters = (operation) => {
    let params = []
    let headerParams = operation.filter((item) => item.in === 'header')

    headerParams.forEach((p) => {
        isOptional = false
        if (p.name.indexOf('Optional') !== -1) {
            isOptional = true;
            p.name = p.name.split(' ')[0].trim();
        }
        params.push({
            "name": p.name,
            "in": p.in,
            "required": !isOptional,
            "schema": {
                "type": p.type,
                "description": p.description
            }
        })
    });
    return params;
}

const buildPathParameters = (operation) => {
    let params = []
    let pathParams = operation.filter((item) => item.in === 'path')
    pathParams.forEach((p) => {
        isOptional = false
        if (p.name.indexOf('Optional') !== -1) {
            isOptional = true;
            p.name = p.name.split(' ')[0].trim();
        }
        params.push({
            "name": p.name,
            "in": p.in,
            "required": !isOptional,
            "schema": {
                "type": p.type,
                "description": p.description
            }
        })
    });
    return params;
}

const buildRequestBody = (operation) => {
    let bodyParams = operation.filter((item) => item.in === 'body')
    let requestBody = {
        "content": {
            "application/json": {
                "schema": {
                    "type": "object",
                    "properties": {'ADD HERE':''}
                }
            }
        }
    };

    isBodyPresent = bodyParams.length > 0;
    // required = [];
    // bodyParams.forEach((bodyParam) => {
    //     let propertyName = bodyParam.name.split(' ')[0];
    //     if (!bodyParam.name.includes('Optional')) {
    //         required.push(propertyName);
    //     }
    //     requestBody.content["application/json"].schema.properties[propertyName] = {};
    //     if (bodyParam.type === 'list') {
    //         bodyParam.type = 'array';
    //     }
    //     requestBody.content["application/json"].schema.properties[propertyName].type = bodyParam.type
    //     if (bodyParam.type === 'array') {
    //         let type = '';
    //         if (bodyParam.description.includes('object')) {
    //             type = 'object';
    //         } else if (bodyParam.description.includes('string')) {
    //             type = 'string';
    //         } else {
    //             type = 'string';
    //         }
    //         requestBody.content["application/json"].schema.properties[propertyName].items = {
    //             type
    //         }
    //     }
    //     requestBody.content["application/json"].schema.properties[propertyName].description = bodyParam.description
    // });

    // if (required.length > 0) {
    //     requestBody.content["application/json"].schema.required = required;
    // }

    return { isBodyPresent, requestBody };
}

const buildResponseBody = (operation) => {
    const responseBody = {
        "headers": {},
        "content": {
            "application/json": {
                "schema": {
                    "type": "object",
                    "properties": {}
                }
            }
        }
    };

    let headerParams = operation.filter((item) => item.in === 'header');
    headerParams.forEach((headerParam) => {
        responseBody.headers[headerParam.name] = {
            "description": headerParam.description,
            "schema": {
                "type": headerParam.type
            }
        };
    });

    let bodyParams = operation.filter((item) => item.in === 'body')
    isBodyPresent = bodyParams.length > 0;
    const required = new Set();
    bodyParams.forEach((bodyParam) => {
        let propertyName = bodyParam.name.split(' ')[0];
        if (!bodyParam.name.includes('Optional')) {
            required.add(propertyName);
        }
        responseBody.content["application/json"].schema.properties[propertyName] = {};
        if (bodyParam.type === 'list') {
            bodyParam.type = 'array';
        }
        responseBody.content["application/json"].schema.properties[propertyName].type = bodyParam.type
        if (bodyParam.type === 'array') {
            let type = '';
            if (bodyParam.description.includes('object')) {
                type = 'object';
            } else if (bodyParam.description.includes('string')) {
                type = 'string';
            } else {
                type = 'string';
            }
            responseBody.content["application/json"].schema.properties[propertyName].items = {
                type
            }
        }
        responseBody.content["application/json"].schema.properties[propertyName].description = bodyParam.description
    });

    if (required.size > 0) {
        responseBody.content["application/json"].schema.required = [...required];
    }

    return { isBodyPresent, responseBody };
}

const buildResponses = (statusCodes, response) => {
    const responses = {}
    statusCodes.forEach((item) => {
        let code = item.code.split(' ')[0];
        responses[code] = { "description": item.reason }
    });
    // const { isBodyPresent, responseBody } = buildResponseBody(response);
    // const successfullResponse = Object.keys(responses).find(k => k.startsWith('2'));
    // console.log('successfullResponse', successfullResponse);
    // if (isBodyPresent) {
    //     console.log('responseBody', responseBody);
    //     Object.assign(responses[successfullResponse], responseBody);
    // }
    return responses;
}

const createOpenApiFile = (data) => {
    const openApiObj = getBaseOpenApiObj();

    const allEndpoints = new Set()
    data.forEach((item) => allEndpoints.add(item.path))
    console.log('\nFound:', allEndpoints.size, 'endpoints\n');

    allEndpoints.forEach(endpoint => {
        console.log('Processing endpoint', endpoint);

        allEndpointOperations = data.filter(item => item.path === endpoint)
        console.log(`Found ${allEndpointOperations.length} operations\n`);

        openApiObj.paths[endpoint] = {};

        let pathOpenAPI = openApiObj.paths[endpoint];

        allEndpointOperations.forEach(op => {
            if (pathOpenAPI[op.restMethod]) {
                console.error('More then 1 operation for same (URIpath, restMethod) set, skipping...');
                return;
            }
            pathOpenAPI[op.restMethod] = { operationId: changeToCamelCase(op.description) };
            pathOpenAPI[op.restMethod].summary = op.description;
            pathOpenAPI[op.restMethod].externalDocs = { url: op.detailsLink }

            let queryParams = buildQueryParameters(op.request);
            let headerParams = buildHeaderParameters(op.request);
            let pathParams = buildPathParameters(op.request);
            if (queryParams.length + headerParams.length + pathParams.length > 0) {
                pathOpenAPI[op.restMethod].parameters = queryParams.concat(headerParams).concat(pathParams);
            }

            let { isBodyPresent, requestBody } = buildRequestBody(op.request);
            if (isBodyPresent) {
                pathOpenAPI[op.restMethod].requestBody = requestBody;
            }

            pathOpenAPI[op.restMethod].responses = buildResponses(op.statusCodes, op.response);

            if (op.tag) {
                if (!pathOpenAPI[op.restMethod].tags) {
                    pathOpenAPI[op.restMethod].tags = [];
                }
                pathOpenAPI[op.restMethod].tags.push(op.tag);
            }
        })
    })

    saveToFile(openApiObj);
}

const handler = new DomHandler((error, dom) => {
    if (error) {
    } else {
        let detailControlNodes = domutils.findAll((element) => element.name === "section" && element.attribs.class === 'detail-control', dom);
        let apiDetailControlNodes = domutils.findAll((element) => element.name === "section" && element.attribs.class === 'api-detail collapse', dom);

        console.log('Nodes count:', detailControlNodes.length, apiDetailControlNodes.length);
        if (detailControlNodes.length !== apiDetailControlNodes.length) {
            console.error('Count mismatch');
            throw new Error();
        }
        const data = extractData(detailControlNodes, apiDetailControlNodes);
        createOpenApiFile(data);
    }
});

const parser = new Parser(handler);
parser.write(htmlFile);
parser.end();