node-red-contrib-ray-serve/lib/ray.html

Deploy Ray Serve applications from Node-RED

<style>
    .node_label_monospace {
        font-family: monospace;
        font-size: 0.82em;
        font-weight: 900;
    }
</style>


<script type="text/javascript">
    RED.nodes.registerType('rayConfig', {
        category: 'config',
        icon: "ray-logo.svg",
        defaults: {
            rayAddress: { value: 'http://ray:8265', required: true },
            rayAddressType: { value: 'str' },
            serveAddress: { value: 'http://ray:8000', required: true },
            serveAddressType: { value: 'str' },
        },
        label: function () {
            return this.serveAddress || 'Ray Server';
        },
        // labelStyle: function () {
        //     return 'node_label_monospace';
        // },
        oneditprepare: function () {
            $('#node-config-input-rayAddress').typedInput({
				default: this.rayAddressType,
				types: ['str', 'global', 'env'],
			});
            $('#node-config-input-serveAddress').typedInput({
				default: this.serveAddressType,
				types: ['str', 'global', 'env'],
			});
        },
    });
</script>

<script type="text/html" data-template-name="rayConfig">
    <div class="form-row">
        <label for="node-config-input-rayAddress">
            <span>Ray API Address (typically port 8065)</span>
        </label>
        <input type="text" id="node-config-input-rayAddress" placeholder="http://localhost:8065" />
    </div>
    <div class="form-row">
        <label for="node-config-input-serveAddress">
            <span>Ray Serve Address (typically port 8000)</span>
        </label>
        <input type="text" id="node-config-input-serveAddress" placeholder="http://localhost:8000" />
    </div>
</script>



<script type="text/javascript">

    const extractServeDeployments = (code) => {
        code = code.replace(/^\s+#.*/g, '');
        const regex = /@serve\.deployment(\([^)]*\))?\s*(?:\s*[#@].*\n)*\s*class\s+(\w+):/g;
        return [...code.matchAll(regex)].map(([, deploymentArgs, name]) => {
            return { name, ...parseDeploymentArgs(deploymentArgs) };
        });
    }

    const parseDeploymentArgs = (deploymentArgs) => {
        // Remove parentheses and split by commas
        // Convert key-value pairs into an object
        return deploymentArgs ? deploymentArgs.slice(1, -1).split(/\s*,\s*/).reduce((acc, arg) => {
            let [key, value] = arg.split('=');
            key = key.match(/\w+\s*$/)?.[0] || key;
            if (key && value) {
                try {
                    acc[key.trim()] = JSON.parse(value.trim());
                } catch (e) {
                    console.error(`Failed to parse deployment argument: ${key}=${value}`);
                }
            }
            return acc;
        }, {}) : {};
    }


    const sampleList = (list) => list[Math.floor(Math.random() * list.length)];
    const randomEndpoint = () => {
        const adjectives = [
            'swift', 'bright', 'cosmic', 'dynamic', 'elastic', 'fluent', 'global', 'hyper', 'infinite', 'joyful', 
            'kinetic', 'lunar', 'mighty', 'nebula', 'orbital', 'plasma', 'quantum', 'radiant', 'solar', 'turbo'];
        const nouns = [
            'api', 'node', 'service', 'endpoint', 'route', 'gateway', 'portal', 'hub', 'nexus', 'core', 
            'engine', 'matrix', 'sphere', 'vertex', 'circuit', 'pulse', 'stream', 'grid', 'net', 'link'];
        return `${sampleList(adjectives)}-${sampleList(nouns)}`;
    }

//     const code = `
// from starlette.requests import Request
// from ray import serve
// from transformers import pipeline

// @serve.deployment
// class Translator:
//     def __init__(self):
//         # Load model
//         self.model = pipeline("translation_en_to_fr", model="t5-small")

//     def translate(self, text: str) -> str:
//         # Run inference and return the translation text
//         translation = self.model(text)[0]["translation_text"]
//         return translation

//     async def __call__(self, http_request: Request) -> str:
//         english_text: str = await http_request.json()
//         return self.translate(summary)
                
// app = Translator.bind()`
const code = `
from ray import serve

@serve.deployment
class HelloWorld:
    async def __call__(self, http_request) -> str:
        msg: dict = await http_request.json()
        return {"payload": "Hello from ray!", "received": msg}
                
app = HelloWorld.bind()`

    RED.nodes.registerType('ray serve', {
        category: 'function',
        color: '#65a9fc',
        inputs: 1,
        outputs: 1,
        // outputs: 2,
        // outputLabels: ['response', 'unprocessable'],
        icon: "ray-logo.svg",
        defaults: {
            server: {
				type: 'rayConfig',
				required: true,
			},
            name: { value: '' },
            route_prefix: { value: '', required: true },
            variable_name: { value: 'app' },
            // runtime_env: { value: {} },
            // container: { value: '' },
            package_manager: { value: 'pip' },
            dependencies: { value: [] },
            env_vars: { value: [] },
            code: { value: code.trim() },
            deployments: { value: extractServeDeployments(code).map(({name}) => ({name})) },
            deploymentArgs: { value: {} },
            // Add default values for other fields as needed
        },
        label: function () {
            return this.name || this.route_prefix || 'Ray Serve';
        },
        // labelStyle: function () {
        //     return 'node_label_monospace';
        // },
        oneditprepare: function() {
            let node = this;
            $("#ray-dashboard-link").on('click', function() {
                let name = $('#node-input-name').val();
                window.open(`http://localhost:8265/#/serve/applications/${name}`, '_blank').focus();
            });

            node.otherEndpoints = {};
            RED.nodes.eachNode(n => {
                if(n.type !== "ray serve" || !n.server || n.id === node.id) return;
                node.otherEndpoints[n.server] = node.otherEndpoints[n.server] || {};
                node.otherEndpoints[n.server][n.route_prefix] = n.id;
            })
            console.log(node.otherEndpoints);

            $('#node-input-route_prefix').val(this.route_prefix || `/${randomEndpoint()}`);
            $('#node-input-route_prefix').on('keyup', function() {
                let route = $(this).val();
                let server = $('#node-input-server').val();
                // console.log(route, server, node.otherEndpoints[server]?.[route]);
                if(node.otherEndpoints[server]?.[route] && node.otherEndpoints[server]?.[route] !== node.id) {
                    $(this).addClass("input-error");
                    $(this).next('.route-prefix-error').text('Endpoint must be unique.');
                } else {
                    $(this).removeClass("input-error");
                    $(this).next('.route-prefix-error').text('');
                }
            });
            
            this.editor = RED.editor.createEditor({
                id: 'node-input-ray-serve-editor',
                mode: 'ace/mode/python',
                value: this.code || "",
                height: '700px',
                // stateId: stateId,
                focus: true,
                // extraLibs: extraLibs
            });

            const moduleExamples = ['numpy', 'torch>=2.0.0', 'pandas[postgresql]', 'requests', 'scipy', 'transformers', 'librosa']
            this.pipList = $("#node-input-dependencies-x").editableList({
                addItem: function(row, index, data) {
                    let example = moduleExamples[index % moduleExamples.length];
                    let pkg = $(`<input type="text" class="node-input-item-name" placeholder="e.g. ${example}" />`).attr('value', data.name);
                    $(row).append(pkg);
                },
                removable: true,
            });
            for(let data of this.dependencies) {
                this.pipList.editableList('addItem', data)
            }

            this.envList = $("#node-input-env_vars-x").editableList({
                addItem: function(row, index, data) {
                    $(`<input class='node-input-key' type="text" placeholder="VAR_NAME" />`).appendTo(row).attr('value', data.key)
                    $(`<input class='node-input-value' placeholder="value" />`).appendTo(row).attr('value', data.value).typedInput({
                        defaultType: data.type || 'str',
                        types: ['str','num','jsonata','env'], // ,'flow','global'
                    });
                },
                removable: true,
            });
            for(let data of this.env_vars) {
                this.envList.editableList('addItem', data)
            }

            this.deploymentList = $("#node-input-deployments-x").editableList({
                addItem: function(row, index, { name, ...value }) {
                    let val = JSON.stringify(value, null, 2);
                    $(`<input class='node-input-name' type="text" placeholder="Deployment Name" />`).appendTo(row).attr('value', name)
                    $(`<input class='node-input-value' placeholder="value" />`).appendTo(row).attr('value', val).typedInput({
                        type:"json",
                        types: ['json'],
                    });
                },
                removable: true,
            });
            for(let data of this.deployments) {
                this.deploymentList.editableList('addItem', data)
            }

            const renderDeployments = () => {
                let code = this.editor.getValue();
                let deployments = extractServeDeployments(code);
                let oldDeployments = this.deploymentList.editableList('items').toArray().reduce((acc, item) => {
                    let name = $(item).find('.node-input-name').val();
                    let value = $(item).find('.node-input-value').typedInput('value');
                    acc[name] = JSON.parse(value || '{}');
                    return acc;
                }, {});
                
                this.deploymentList.editableList('empty');
                for(let {name, ...data} of deployments) {
                    this.deploymentList.editableList('addItem', {name, ...oldDeployments?.[name], DETECTED_FROM_PYTHON: data});
                }
                node.deploymentArgs = {...node.deploymentArgs, ...oldDeployments};
            }
            renderDeployments();
            this.editor.onDidChangeModelContent((event) => {
                clearTimeout(this.debounceTimer);
                this.debounceTimer = setTimeout(renderDeployments, 500);
            });


            // Tabs

            var tabs = RED.tabs.create({
                id: "ray-tabs",
                onchange: function(tab) {
                    $("#ray-tabs-content").children().hide();
                    $("#" + tab.id).show();
                }
            });
            // tabs.addTab({
            //     id: "func-tab-config",
            //     iconClass: "fa fa-cog",
            //     label: that._("function.label.setup")
            // });
            tabs.addTab({
                id: "ray-tab-app",
                label: "Application"
            });
            tabs.addTab({
                id: "ray-tab-deps",
                label: "Environment"
            });

            tabs.activateTab("ray-tab-app");
        },
        oneditsave: function() {
            let node = this;
            this.code = this.editor.getValue();
            this.editor.destroy();
            delete this.editor;
            node.dependencies = [];
            $("#node-input-dependencies-x").editableList('items').each(function(i) {
                let name = $(this).find(".node-input-item-name").val()
                name && node.dependencies.push({ name })
            })
            node.env_vars = [];
            $("#node-input-env_vars-x").editableList('items').each(function(i) {
                let key = $(this).find(".node-input-value").attr('value');
                let type = $(this).find(".node-input-value").typedInput('type');
                let value = $(this).find(".node-input-value").typedInput('value');
                key && node.env_vars.push({ key, type, value });
            })
            node.deployments = [];
            $("#node-input-deployments-x").editableList('items').each(function(i) {
                let name = $(this).find(".node-input-name").val();
                let value = $(this).find(".node-input-value").typedInput('value');
                value = value.replace(/^\s+\/\/.*/g, '');
                let data = JSON.parse(value || '{}');
                name && node.deployments.push({ name, ...data });
            })

            // node.deployments = extractServeDeployments(node.code);
            delete this.pipList;
            delete this.envList;
            delete this.deploymentList;

        },
        oneditcancel: function() {
            this.editor.destroy();
            delete this.editor;
        },
    });
</script>

<script type="text/html" data-template-name="ray serve">
    <style>
        #ray-tabs-content .red-ui-editableList-container {
            padding: 0px;
            border: 0;
        }
        #ray-tabs-content .red-ui-editableList-container li {
            padding:0px;
        }
        #ray-tabs-content .red-ui-editableList-item-remove {
            right: 5px;
        }

        #ray-tabs-content .node-input-list-row > label {
            /* width: 100%; */
            font-size: 1em;
            font-weight: 700;
            margin: 0 0 0 4px;
        }
        #ray-tabs-content .node-input-list-row .red-ui-editableList-container input {
            /* width: auto; */
            border-radius: 0;
        }
        #ray-tabs-content .red-ui-editableList-container .red-ui-editableList-list li:not(:first-child) input {
            border-top: 0;
        }

        #ray-tabs-content .red-ui-editableList-container li {
            border-bottom: 0px;
        }

        #ray-tabs-content .red-ui-editableList-item-content {
            display: flex;
            justify-content: stretch;
        }
        #ray-tabs-content .red-ui-editableList-item-content > * {
            flex-grow: 1;
        }
        #node-input-env_vars-x .red-ui-typedInput-container {
            border-radius: 0;
        }
        /* #ray-tabs-content .red-ui-editableList-container input {
            width: auto;
        } */

        #ray-tabs-content .red-ui-editableList-container input.node-input-name {
            flex-basis: 100px;
            flex-grow: 0;
            /* width: 70%; */
        }
        #node-input-env_vars-x input.node-input-key {
            width: auto;
        }

        #node-input-route_prefix {
            font-family: monospace;
            font-weight: 900;
            font-size: 1.2em;
            width: 100%;
        }
        .route-prefix-error {
            font-weight: 900;
            color: red;
            display: block;
        }
    </style>
    
    <div class="form-row ray-tabs-row">
        <ul style="min-width: 600px; margin-bottom: 20px;" id="ray-tabs"></ul>
    </div>
    <div id="ray-tabs-content" style="min-height: calc(100% - 95px);">
        <div id="ray-tab-app" style="display:none">
            <div class="form-row">
                <label for="node-input-name">
                    <!-- <span>Server</span> -->
                    <button id="ray-dashboard-link">Server</button>
                </label>
                <input type="text" id="node-input-server" />
            </div>
            <div class="form-row">
                <label for="node-input-name">
                    <span>Name</span>
                </label>
                <input type="text" id="node-input-name" />
            </div>
            <div class="form-row">
                <label for="node-input-route_prefix">
                    <span>URL</span>
                </label>
                <div>
                    <input type="text" id="node-input-route_prefix" placeholder="What should the URL endpoint be?" />
                    <small class="route-prefix-error"></small>
                </div>
            </div>
            <div id="func-tab-body">
                <div class="form-row node-text-editor-row" style="position:relative">
                    <div style="height: 400px; min-height:150px;" class="node-text-editor" id="node-input-ray-serve-editor" ></div>
                </div>
            </div>
            <div class="form-row">
                <p>What is the variable name of your Deployment?</p>
                <code>serve.run(<input type="text" id="node-input-variable_name" />)</code>
            </div>
            <div class="form-row node-input-list-row">
                <p>
                    The list of deployments detected in your Python code. You can modify the deployment configuration here.
                </p>
                <label for="node-input-deployments-x">
                    <span>
                        Deployments [ <a target="_blank" href="https://docs.ray.io/en/latest/serve/configure-serve-deployment.html">docs</a> ]
                    </span>
                </label>
                <ol id="node-input-deployments-x"></ol>
            </div>

            <div>
                <ul>
                    <li><a href="https://docs.ray.io/en/latest/serve/advanced-guides/advanced-autoscaling.html#optional-define-how-the-system-reacts-to-changing-traffic" target="_blank">
                        Autoscaling Docs (<code>autoscaling_config</code>)
                    </a></li>
                </ul>
            </div>

        </div>

        <div id="ray-tab-deps" style="display:none">
            <!-- <div class="form-row">
                <label for="node-input-name">
                    <span>Container Image</span>
                </label>
                <input type="text" id="node-input-container" />
            </div> -->
            <div class="form-row">
                <p>
                    Your Ray application might depend on Python packages (for example, pendulum or requests) via import statements.

                    You can specify these dependencies here. Ray will install them in the environment where your application runs.
                </p>
            </div>
            <div class="form-row">
                <label for="node-input-package_manager">
                    <span>Package Manager</span>
                </label>
                <select id="node-input-package_manager">
                    <option value="pip">pip</option>
                    <option value="conda">conda</option>
                </select>
            </div>
            <div class="form-row node-input-list-row">
                <label for="node-input-dependencies-x">
                    <span>Python Dependencies</span>
                    <small>[<a target="_blank" href="https://docs.ray.io/en/latest/ray-core/handling-dependencies.html#using-conda-or-pip-packages">docs</a>]</small>
                </label>
                <ol id="node-input-dependencies-x"></ol>
            </div>
            <div class="form-row node-input-list-row">
                <label for="node-input-env_vars-x">
                    <span>Environment Variables</span>
                </label>
                <p>
                    You can specify environment variables that will be available to your application at runtime. 
                </p>
                <ol id="node-input-env_vars-x"></ol>
            </div>
        </div>
    </div>
    
</script>