@hclsoftware/secagent/src/Distributor/AsocConnector.js

IAST agent

//IASTIGNORE

/*
 * ****************************************************
 * Licensed Materials - Property of HCL.
 * (c) Copyright HCL Technologies Ltd. 2017, 2025.
 * Note to U.S. Government Users *Restricted Rights.
 * ****************************************************
 */

'use strict'

const fs = require('fs')
const os = require('os')
const path = require('path')

const https = require('https')
const http = require('http')
const querystring = require('querystring')
const Readable = require('stream').Readable
const FormData = require('form-data')
const { URL } = require('url');

const Distributor = require('./Distributor')
const IastLogger = require('../Logger/IastLogger')
const HookParser = require("../Hooks/HookParser")
const ConfigFileManager = require("../ConfigFile/ConfigFileManager")
const {ConfigInfo} = require("../ConfigFile/ConfigInfo")
const {MemoryInfo} = require('../Utils/MemoryInfo')

// ASoC Defs
const ASOC_API_START = '/api/'

// constants for timing definitions
const SECOND = 1000
const MIN_INTERVAL_SEC = 1 // minimum polling interval in s
const DEFAULT_INTERVAL_SEC = 10 // default polling interval for enabled state in seconds
const FORCE_UPGRADE_TIMEOUT = 60 * 60 * 12 // seconds * minutes * hours

const REQUEST_TIMEOUT_MILLISEC = 30000 // 30 seconds

const CONFIG_HASH = 'ConfigHash'

let pollingInterval = DEFAULT_INTERVAL_SEC; // start as enabled
let pollingCounter = 0

const MAX_SKIPPED_CYCLES = 6; // maximum number of cycles we skip before polling asoc if there are no new vulnerabilities or if agent is disabled
let skippedCycles = MAX_SKIPPED_CYCLES; // initialize to max so that in the first cycle we enter the code


let updateTriggerTime

let intervalObj // timer object - https://javascript.info/settimeout-setinterval#setinterval
let sizeCounter = 0
let lastPollTimestamp // last successful polling time
let protocol = 'https'
let timerActive = false
let requestInProgress = false // avoid getting into the task when previous execution is not over
let activeExecutions = true
let memorySafe = true

const ASOC_DEFAULT_HOST = 'cloud.appscan.com'
const ASOC_SSL_HOSTS = [
    "cloud.appscan.com", // PRODUCTION
    "eu.cloud.appscan.com" // PRODUCTION-EU
]

let asocConfig = {
  host: undefined,
  accessToken: undefined, // will not work if no access token defined
  port: undefined,
  endpointStart: '/IAST/api/',
  rejectUnauthorized: true
}

const AsocAccessTokenEnvVarName = "IAST_ACCESS_TOKEN"
const AsocHostEnvVarName = "IAST_HOST"
const ASOC_CONFIG_NAME = 'asoc-config.json'

const MEMORY_OK = "Memory usage below threshold. Enabling hooks.";

module.exports.ConfigureAsocConnector = () => {
  // host and token priority:
  // 1. env var
  // 2. agent-config.json
  // 3. asocConfig.json (legacy)


  const agentConfig = loadAgentConfig()
  IastLogger.eventLog.debug(`agent-config.json: ${JSON.origStringify(agentConfig)}`)

  // first, try to get values from environment variable
  if (process.env[AsocAccessTokenEnvVarName] != null) {
    IastLogger.eventLog.info(`Setting access token from environment variable ${AsocAccessTokenEnvVarName}. Tokens from agent-config.js file and ${ASOC_CONFIG_NAME} file are ignored.`)
    setAccessToken(process.env[AsocAccessTokenEnvVarName])
  }
  else if (agentConfig.accessToken != null && agentConfig.accessToken !== 'KeyPlaceholder') {
    IastLogger.eventLog.info(`Setting access token from agent-config.js file. Token from ${ASOC_CONFIG_NAME} file is ignored.`)
    setAccessToken(agentConfig.accessToken)
  }
  if (process.env[AsocHostEnvVarName] != null)
  {
    IastLogger.eventLog.info(`Setting host from environment variable ${AsocHostEnvVarName}. Tokens from agent-config.js file and ${ASOC_CONFIG_NAME} file are ignored.`)
    setAsocHost(process.env[AsocHostEnvVarName])
  }
  else if (agentConfig.host != null && agentConfig.host !== 'HostPlaceholder') {
    IastLogger.eventLog.info(`Setting host from agent-config.js file. Token from ${ASOC_CONFIG_NAME} file is ignored.`)
    setAsocHost(agentConfig.host)
  }
  // if not all values are found in env, get values from asoc-config.json
  if (asocConfig.host == null || asocConfig.accessToken == null) {
    setAsocConfigFromFile()
  }
  // if host value not found in env var nor in a file, it has default value
  if (asocConfig.host == null) {
    IastLogger.eventLog.info(`Setting host to default - ${ASOC_DEFAULT_HOST}`)
    asocConfig.host = ASOC_DEFAULT_HOST // default value
  }
}

function loadAgentConfig() {
  const configPath = path.resolve(__dirname, '../../agent-config.json');
  if (fs.existsSync(configPath)) {
    try {
      const jsonData = fs.readFileSync(configPath, 'utf8');
      return JSON.origParse(jsonData);
    } catch (e) {
      IastLogger.eventLog.error("Error parsing agent-config.json: " + e);
      return {};
    }
  } else {
    IastLogger.eventLog.info("agent-config.json file does not exist");
    return {};
  }
}

const setAsocConfigFromFile = () => {
  let tempConfig = getAsocConfigFromFile()
  if (asocConfig.host == null && tempConfig != null && tempConfig.host != null) {
    IastLogger.eventLog.info(`Setting host from ${ASOC_CONFIG_NAME} file`)
    setAsocHost(tempConfig.host)
  }
  if (asocConfig.accessToken == null && tempConfig != null && tempConfig.accessToken != null) {
    IastLogger.eventLog.info(`Setting access token from ${ASOC_CONFIG_NAME} file`)
    setAccessToken(tempConfig.accessToken)
  }
}

const setAccessToken = (accessToken) => {
  IastLogger.eventLog.info(`Setting access token to ${accessToken}`)
  asocConfig.accessToken = accessToken
}

const setAsocHost = (origHost) => {
  const url = new URL(origHost)

  asocConfig.host = url.hostname
  asocConfig.port = url.port
  asocConfig.endpointStart = url.pathname.replace(/\/?$/, '') + ASOC_API_START
  IastLogger.eventLog.info(`Setting host to ${asocConfig.host}`)
  if (ASOC_SSL_HOSTS.includes(asocConfig.host)) {
    asocConfig.rejectUnauthorized = true
    IastLogger.eventLog.info(`Creating secured client for host ${asocConfig.host}`);
  } else {  // allow for self signed certificate, for ASE, staging and mirror
    asocConfig.rejectUnauthorized = false
    IastLogger.eventLog.info(`Creating trusted client for host ${asocConfig.host}`);
  }
}

const getAsocConfigFromFile = () => {
  const jsonPath = path.join(os.tmpdir(), ASOC_CONFIG_NAME)
  try {
    const rawdata = fs.readFileSync(jsonPath)
    IastLogger.eventLog.info('config file uploaded:')
    IastLogger.eventLog.debug(`asocConfig: ${rawdata}`)
    return JSON.origParse(rawdata)
  } catch (err) {
    IastLogger.eventLog.info(`${jsonPath} does not exist.`)
  }
}

const DistributorStatus = {
  SUCCESS: 'success',
  FAIL: 'fail',
  DATA_PENDING: 'data_pending'
}

// Task to handle reporting to ASoC
// it first queries ASoC for execution state since last time it was called
// then it gets report from the distributor for the past interval and sends it back to ASoC
async function AsocTimerTask() {
  try {
    if (requestInProgress) return
    pollingCounter = pollingCounter >= pollingInterval ? 1 : pollingCounter + 1 // increment counter
    const timerActiveSampled = timerActive
    let persistentMemoryWarning = ! memorySafe
    let memoryWarning = MemoryInfo.checkMemoryWarning()
    memorySafe = !memoryWarning
    if (memoryWarning && !persistentMemoryWarning){
      IastLogger.eventLog.error(MemoryInfo.getMemoryDebugInfo());
      IastLogger.eventLog.error(`Memory Error!!! Process passed the memory threshold of ${ConfigInfo.ConfigInfo.memoryThreshold}. Disabling hooks.`)
      HookParser.hooksActive = false
    }
    else if (!memoryWarning && persistentMemoryWarning){
      IastLogger.eventLog.info(MEMORY_OK)
      if (activeExecutions){
        HookParser.hooksActive = true
      }
    }
    if (pollingCounter === pollingInterval || !timerActiveSampled  || (memoryWarning && !persistentMemoryWarning)) {
      await doCommunicationCycle(memoryWarning)
    }
    if (!timerActiveSampled) {
      IastLogger.eventLog.info('Stopping AppScan client')
      clearInterval(intervalObj) // stop timer
    }
  } catch (err) {
    IastLogger.eventLog.error(`TimerTask failed: ${err}`)
  }
}

async function doCommunicationCycle(memoryWarning) {
  // we want to reduce asoc communication to once every minute if there are no active executions or
  // no pending issues
  if (!Distributor.vulnerabilitiesPending() && skippedCycles < MAX_SKIPPED_CYCLES) {
    skippedCycles ++ ;
    return;
  }

  skippedCycles = 0;

  requestInProgress = true
  const currentTime = new Date()
  sizeCounter = 0
  let reportSucceeded = true
  Distributor.takeSnapshot() // save current issues status
  let currentExecutionsResult
  try {
    currentExecutionsResult = await getExecutions(currentTime)
  } catch (err) {
    IastLogger.eventLog.error(`getExecutions from ${asocConfig.host} failed: ${err}`)
  }
  const timeAfterGetExecutions = new Date()
  let status
  if (currentExecutionsResult !== undefined) {
    const configFileHash = currentExecutionsResult[CONFIG_HASH]
    if (currentExecutionsResult.UpdateAvailable && updateTriggerTime === undefined) { updateTriggerTime = plusMillis(currentTime, FORCE_UPGRADE_TIMEOUT * 1000) }
    if (currentExecutionsResult.Executions.length === 0) {
      if (activeExecutions) {
        // deactivate hooks if no active execution
        IastLogger.eventLog.info("No active executions. Disabling hooks.")
        activeExecutions = false
        HookParser.hooksActive = false
        Distributor.clearAllIssues()
      }
    } else if (!activeExecutions && memorySafe && !HookParser.hooksActive) {
      IastLogger.eventLog.info("An execution was activated. Enabling hooks.")
      activeExecutions = true
      HookParser.hooksActive = true
    }
    for (const execution of currentExecutionsResult.Executions) {
      let executionStatus
      try {
        executionStatus = await reportToAsoc(execution.StartBeforeMilliSec, execution.EndBeforeMilliSec, execution.ExecutionId, currentTime, timeAfterGetExecutions)
      } catch (err) {
        IastLogger.eventLog.error(`report failed: ${err}`)
        executionStatus = DistributorStatus.FAIL
      }
      if (executionStatus !== DistributorStatus.FAIL) {
        // if report succeeded
        if (execution.EndBeforeMilliSec != null && execution.EndBeforeMilliSec > 0 && executionStatus !== DistributorStatus.DATA_PENDING) { // this execution is done, no need to keep the map
          Distributor.clearReported(execution.ExecutionId)
        } else {
          Distributor.stagedToReported(execution.ExecutionId)
        }
        if (executionStatus === DistributorStatus.DATA_PENDING) {
          status = DistributorStatus.DATA_PENDING
        }
      } else { // report to asoc failed
        reportSucceeded = false
      }
    }
    await readConfigFile(configFileHash)
  }
  // if one of the reports did not succeed, we return the map. the map may contain issues from mixed
  // executions so it is done per the whole polling period.
  // but the set of the 'already reported to asoc' is per execution, so we will not re-send the issues,
  // we will just iterate them again
  if (currentExecutionsResult === undefined || !reportSucceeded) {
    Distributor.returnToMap()
  } else {
    // update timestamp only if there is not pending data, so that we get same execution again
    if (status !== DistributorStatus.DATA_PENDING) {
      lastPollTimestamp = currentTime // if all succeeded, update timestamp of last successful poll
    }
    IastLogger.eventLog.debug(`Polling took ${millisecSince(currentTime, new Date())}ms. ${sizeCounter} chars sent.`)
  }
  if (reportSucceeded && memoryWarning){
    // after we send a report we may clear enough memory to turn on the hooks again
    if (!MemoryInfo.checkMemoryWarning())
    {
      memorySafe = true;
      IastLogger.eventLog.info(MEMORY_OK);
      if (activeExecutions)
      {
        HookParser.hooksActive = true
      }
    }
  }
  requestInProgress = false

}

// called when asocEnabled
module.exports.start = () => {
  if (asocConfig.accessToken == null) return; // can't work if no access token
  if (!timerActive) {
    pollingCounter = 0
    timerActive = true
    intervalObj = setInterval(() => {
      try {
        AsocTimerTask()
      } catch (err) {
        IastLogger.eventLog.error(`starting TimerTask failed: ${err}`)
      }
    }, SECOND)
    IastLogger.eventLog.info(`AppScan client started with interval of ${pollingInterval} seconds`)
    lastPollTimestamp = new Date()
  }
}

// // useful for cases where we stop and start immediately, e.g. set new polling interval
// function stopWithoutSending () {
//   timerActive = false
//   clearInterval(intervalObj) // stop timer
// }

module.exports.stopAfterNext = () => {
  if (timerActive) {
    IastLogger.eventLog.debug('Sending final report.')
    timerActive = false
  } else {
    // notifyExecutionDone();
  }
}

function getExecutions (currentTime) {
  const lastPollMilliSec = millisecSince(lastPollTimestamp, currentTime)
  const queryParams = {
    lastPollMilliSec: Math.floor(lastPollMilliSec * 1.1),
    AgentId: Distributor.getAgentId(),
    AgentVersion: '0.0.1' // TODO: Distributor.getDistributor().getAgentVersion())
  }
  return sendGetRequest('CurrentExecution', queryParams)
}

function processGetResponse (res, resolve, reject, fullURL) {
  const statusCode = res.statusCode
  const statusMessage = res.statusMessage

  // cumulate data
  let body = ''
  res.on('data', function (chunk) {
    body += chunk
  })

  // resolve on end
  res.on('end', function () {
    if (statusCode !== 200) {
      IastLogger.eventLog.error(`Request GET ${fullURL} failed: ${statusCode} ${statusMessage}, error: ${body}`)
      reject(body) // error return of promise
    }
    try {
      IastLogger.eventLog.debug(`Getting Execution info: GET ${fullURL} response: ${body}`)
      body = JSON.origParse(body)
    } catch (e) {
      reject(e) // error return of promise
    }
    resolve(body) // good return of promise
  })
}

// get data to report from the distributor and send to asoc
function reportToAsoc (startBeforeMilliSec, endBeforeMilliSec, execution, currentTime, timeAfterGetExecutions) {
  return new Promise(function (resolve, reject) {
    let status = DistributorStatus.SUCCESS
    const startTime = minusMillis(currentTime, startBeforeMilliSec)
    const endTime = minusMillis(timeAfterGetExecutions, endBeforeMilliSec)
    const result = Distributor.getReport(startTime, endTime, execution)
    if (result.pending) {
      status = DistributorStatus.DATA_PENDING
    }
    if (result.data === undefined) {
      resolve(status)
      return
    }
    sizeCounter += result.data.length

    // convert the data into stream
    const dataStream = new Readable()
    dataStream.push(result.data) // the string you want
    dataStream.push(null) // indicates end-of-file basically - the end of the stream

    // create form data
    const form = new FormData()
    form.append('fileToUpload', dataStream, {
      filename: 'IAST-sample.json', // ... or:
      contentType: 'application/json'
    })

    // prepare the multi part post request
    const queryParams = {
      executionId: execution
    }

    const multiPartHeaders = form.getHeaders()
    multiPartHeaders.Authorization = 'Bearer ' + asocConfig.accessToken
    multiPartHeaders.Accept = 'application/json'

    const fullPath = asocConfig.endpointStart + 'UploadResults?' + querystring.stringify(queryParams)
    const options = getRequestOptions(fullPath, 'POST', multiPartHeaders)

    let req
    const fullURL = getFullUrl(options.hostname, fullPath)
    if (protocol === 'https') {
      req = https.request(options, (res) => { sendPostRequest(res, resolve, reject, status, result.data, fullURL) }).setTimeout(REQUEST_TIMEOUT_MILLISEC)
    } else {
      req = http.request(options, (res) => { sendPostRequest(res, resolve, reject, status, result.data, fullURL) }).setTimeout(REQUEST_TIMEOUT_MILLISEC)
    }

    req.on('error', (err) => {
      IastLogger.eventLog.error('Error sending data: ' + err) // usually caused by 'write EPIPE' error
      reject(err)
    })
    req.on('timeout', () => {
      reject('Request has timed out.')
    })
    try {
      form.pipe(req)
    } catch (err) {
      IastLogger.eventLog.error('Error in pipe: ' + err)
      reject(err)
    }
  })
}

function getRequestOptions(fullpath, httpMethod, httpHeaders) {
  const options = {
    hostname: asocConfig.host,
    path: fullpath,
    method: httpMethod,
    headers: httpHeaders,
    rejectUnauthorized: asocConfig.rejectUnauthorized,
  }
  if (asocConfig.port != null) {
    options.port = asocConfig.port
  }
  return options
}

function getFullUrl(hostname, fullpath) {
  return `${protocol}://${hostname}${fullpath}`
}

function sendPostRequest (res, resolve, reject, status, data, fullURL) {
  const statusCode = res.statusCode
  let responseMessage = ''

  res.setEncoding('utf8')
  res.on('data', (d) => {
    responseMessage = d
  })
  res.on("error", function(error) {
    IastLogger.eventLog.error(`POST ${fullURL} ${error} : ${responseMessage}`)
    reject(error)
  })

  res.on('end', () => {
    if (Math.floor(statusCode/100) !== 2) {
      IastLogger.eventLog.error(`POST ${fullURL} ${statusCode} : ${responseMessage}`)
      if ( Math.floor(statusCode/100) === 4){
        // 400 status code means the issues were sent but rejected by ASoc
        // no reason to try sending these issues again
        IastLogger.findingsLog.error("Data rejected by AppScan server:\n" + data);
        status = DistributorStatus.SUCCESS;
      }
      else {
        status = DistributorStatus.FAIL
      }
    }
    else {
      IastLogger.eventLog.debug(`POST ${fullURL} ${statusCode} : ${responseMessage}`)
    }
    resolve(status)
  })
}

// config updates
// wait some interval until pending data is sent
module.exports.setTestMode = (testMode) => {
  const waitTime = timerActive ? (pollingInterval + 1) * SECOND : 0
  setTimeout(function() {
    protocol = testMode ? 'http' : 'https';
    IastLogger.eventLog.debug(`setting protocol to ${protocol}`)
  }, waitTime);
}

module.exports.setPollingInterval = (interval) => {
  if (interval == null) { interval = DEFAULT_INTERVAL_SEC }
  if (interval < MIN_INTERVAL_SEC) {
    IastLogger.eventLog.warning(`Requested polling interval (${interval} sec) is smaller than minimum (${MIN_INTERVAL_SEC} sec). Not setting.`)
  } else {
    if (interval !== pollingInterval) {
      IastLogger.eventLog.debug(`Setting polling interval to ${interval} sec`)
      pollingCounter = 0
      pollingInterval = interval
    }
  }
}

// time utils

// calc milliseconds between to timestamps
function millisecSince (from, to) {
  return to.getTime() - from.getTime()
}

// return new timestamp with given time minus given millisec
function minusMillis (currentTime, millis) {
  const result = new Date()
  result.setTime(currentTime.getTime() - millis)
  return result
}

// return new timestamp with given time plus given millisec
function plusMillis (currentTime, millis) {
  const result = new Date()
  result.setTime(currentTime.getTime() + millis)
  return result
}

async function readConfigFile (fileHash) {
  if (ConfigFileManager.isServerConfigExists() || fileHash == null || fileHash === ConfigFileManager.getCurrentAsocConfigHash()) {
    return
  }
  let configResult
  try {
    configResult = await sendGetRequest('DownloadUserConfig')
  }
  catch (err) {
    return
  }

  ConfigFileManager.notifyConfigFileUploadedFromAsoc(fileHash, configResult)
}

function sendGetRequest(endpoint, parameters) {
  return new Promise(function (resolve, reject) {
    let fullPath = asocConfig.endpointStart + endpoint
    const query = querystring.stringify(parameters)
    if (query !== "") {
      fullPath += '?' + query
    }
    const headers = {Authorization: 'Bearer ' + asocConfig.accessToken}
    const options = getRequestOptions(fullPath, 'GET', headers)

    let req
    const fullURL = getFullUrl(options.hostname, fullPath)
    if (protocol === 'https') {
      req = https.request(options, (res) => { processGetResponse(res, resolve, reject, fullURL) }).setTimeout(REQUEST_TIMEOUT_MILLISEC)
    } else {
      req = http.request(options, (res) => { processGetResponse(res, resolve, reject, fullURL) }).setTimeout(REQUEST_TIMEOUT_MILLISEC)
    }

    req.on('error', (e) => {
      IastLogger.eventLog.error(`Get request to endpoint ${endpoint} failed: ${e}`)
      skippedCycles = MAX_SKIPPED_CYCLES // try again next cycle
      reject(e)
    })
    req.on('timeout', () => {
      skippedCycles = MAX_SKIPPED_CYCLES // try again next cycle
      reject('Request has timed out.')
    })
    req.end()
  })
}