@hyperlane-xyz/core

name: CCIP Load Test on: push: branches: - ccip-develop tags: - '*' workflow_dispatch: inputs: base64_test_input: # base64 encoded toml for test input description: 'Base64 encoded toml test input' required: false test_secrets_override_key: description: 'Key to run tests with custom test secrets' required: false type: string # Only run 1 of this workflow at a time per PR concurrency: group: load-ccip-tests-chainlink-${{ github.ref }} cancel-in-progress: true env: CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink CHAINLINK_VERSION: ${{ github.sha}} INPUT_CHAINLINK_TEST_VERSION: ${{ github.sha}} ENV_JOB_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-ccip-tests:${{ github.sha }} INTERNAL_DOCKER_REPO: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com AWS_ECR_REPO_PUBLIC_REGISTRY: public.ecr.aws MOD_CACHE_VERSION: 1 E2E_TEST_CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink E2E_TEST_LOKI_TENANT_ID: ${{ vars.LOKI_TENANT_ID }} E2E_TEST_LOKI_ENDPOINT: ${{ secrets.LOKI_URL }} E2E_TEST_LOKI_BASIC_AUTH: ${{ secrets.LOKI_BASIC_AUTH }} E2E_TEST_GRAFANA_BASE_URL: ${{ vars.GRAFANA_URL }} # Default private key test secret loaded from Github Secret as only security team has access to it. # this key secrets.QA_SHARED_803C_KEY has a story behind it. To know more, see CCIP-2875 and SECHD-16575 tickets. E2E_TEST_ETHEREUM_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }} E2E_TEST_ARBITRUM_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }} E2E_TEST_BASE_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }} E2E_TEST_WEMIX_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }} E2E_TEST_AVALANCHE_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }} E2E_TEST_ZKSYNC_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }} E2E_TEST_MODE_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }} E2E_TEST_METIS_ANDROMEDA_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }} E2E_TEST_OPTIMISM_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }} E2E_TEST_KROMA_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }} E2E_TEST_GNOSIS_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }} E2E_TEST_POLYGON_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }} E2E_TEST_BSC_MAINNET_WALLET_KEY: ${{ secrets.QA_SHARED_803C_KEY }} jobs: build-chainlink: environment: integration permissions: id-token: write contents: read name: Build Chainlink Image runs-on: ubuntu20.04-16cores-64GB steps: - name: Checkout the repo uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 - name: Check if image exists id: check-image uses: smartcontractkit/chainlink-github-actions/docker/image-exists@b49a9d04744b0237908831730f8553f26d73a94b # v2.3.17 with: repository: chainlink tag: ${{ env.CHAINLINK_VERSION }} AWS_REGION: ${{ secrets.QA_AWS_REGION }} AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} - name: Build Image if: steps.check-image.outputs.exists == 'false' uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/build-image@b49a9d04744b0237908831730f8553f26d73a94b # v2.3.17 env: GH_TOKEN: ${{ github.token }} with: cl_repo: smartcontractkit/chainlink-ccip cl_ref: ${{ env.CHAINLINK_VERSION }} push_tag: ${{ env.CHAINLINK_IMAGE }}:${{ env.CHAINLINK_VERSION }} QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} - name: Collect Metrics if: always() id: collect-gha-metrics uses: smartcontractkit/push-gha-metrics-action@dea9b546553cb4ca936607c2267a09c004e4ab3f # v3.0.0 with: id: ccip-load-test-build-chainlink-image org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} this-job-name: Build Chainlink Image continue-on-error: true build-test-image: environment: integration permissions: id-token: write contents: read name: Build Test Image runs-on: ubuntu20.04-16cores-64GB steps: - name: Collect Metrics id: collect-gha-metrics uses: smartcontractkit/push-gha-metrics-action@dea9b546553cb4ca936607c2267a09c004e4ab3f # v3.0.0 with: id: ccip-load-test-build-test-image org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} this-job-name: Build Test Image continue-on-error: true - name: Checkout the repo uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 - name: Build Test Image uses: ./.github/actions/build-test-image with: tag: ${{ env.INPUT_CHAINLINK_TEST_VERSION }} QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} ccip-load-test: environment: integration needs: [ build-chainlink, build-test-image ] if: ${{ always() && !contains(needs.*.result, 'failure') }} permissions: issues: read checks: write pull-requests: write id-token: write contents: read env: CHAINLINK_ENV_USER: ${{ github.actor }} SLACK_API_KEY: ${{ secrets.QA_SLACK_API_KEY }} SLACK_CHANNEL: ${{ secrets.QA_SLACK_CHANNEL }} TEST_LOG_LEVEL: info REF_NAME: ${{ github.head_ref || github.ref_name }} strategy: fail-fast: false matrix: type: - name: stable-load run: ^TestLoadCCIPStableRPS$ os: ubuntu-latest - name: load-with-arm-curse-uncurse run: ^TestLoadCCIPStableRPSAfterARMCurseAndUncurse$ config_path: ./integration-tests/ccip-tests/testconfig/tomls/load-with-arm-curse-uncurse.toml os: ubuntu-latest runs-on: ${{ matrix.type.os }} name: CCIP ${{ matrix.type.name }} steps: - name: Collect Metrics id: collect-gha-metrics uses: smartcontractkit/push-gha-metrics-action@dea9b546553cb4ca936607c2267a09c004e4ab3f # v3.0.0 with: id: ccip-load-test-${{ matrix.type.name }} org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} this-job-name: CCIP ${{ matrix.type.name }} test-results-file: '{"testType":"go","filePath":"/tmp/gotest.log"}' continue-on-error: true - name: Checkout the repo uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 with: ref: ${{ env.REF_NAME }} - name: Sets env vars id: set_override_config shell: bash run: | # if the matrix.type.config_path is set, use it as the override config if [ -n "${{ matrix.type.config_path }}" ]; then BASE64_CCIP_CONFIG_OVERRIDE=$(base64 -w 0 -i ${{ matrix.type.config_path }}) echo ::add-mask::$BASE64_CCIP_CONFIG_OVERRIDE echo "base_64_override=$BASE64_CCIP_CONFIG_OVERRIDE" >> $GITHUB_OUTPUT fi if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then BASE64_CCIP_CONFIG_OVERRIDE=$(jq -r '.inputs.base64_test_input' $GITHUB_EVENT_PATH) echo ::add-mask::$BASE64_CCIP_CONFIG_OVERRIDE echo "base_64_override=$BASE64_CCIP_CONFIG_OVERRIDE" >> $GITHUB_OUTPUT fi - name: step summary shell: bash run: | echo "### chainlink image used for this test run :link:" >>$GITHUB_STEP_SUMMARY echo "\`${{ env.CHAINLINK_VERSION }}\`" >> $GITHUB_STEP_SUMMARY echo "### chainlink-tests image tag for this test run :ship:" >>$GITHUB_STEP_SUMMARY echo "\`${{ env.INPUT_CHAINLINK_TEST_VERSION }}\`" >> $GITHUB_STEP_SUMMARY - name: Prepare Base64 TOML override for CCIP secrets uses: ./.github/actions/setup-create-base64-config-ccip id: setup_create_base64_config_ccip with: runId: ${{ github.run_id }} testLogCollect: ${{ vars.TEST_LOG_COLLECT }} chainlinkVersion: ${{ github.sha }} logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }} - name: Run Tests uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@94cb11f4bd545607a2f221c6685052b3abee723d # v2.3.32 env: TEST_SUITE: load TEST_ARGS: -test.timeout 900h DATABASE_URL: postgresql://postgres:node@localhost:5432/chainlink_test?sslmode=disable RR_MEM: 8Gi RR_CPU: 4 TEST_TRIGGERED_BY: ccip-load-test-ci-${{ matrix.type.name }} BASE64_CCIP_CONFIG_OVERRIDE: ${{ steps.set_override_config.outputs.base_64_override }},${{ steps.setup_create_base64_config_ccip.outputs.base64_config }} TEST_BASE64_CCIP_CONFIG_OVERRIDE: ${{ steps.set_override_config.outputs.base_64_override }},${{ steps.setup_create_base64_config_ccip.outputs.base64_config }} E2E_TEST_GRAFANA_DASHBOARD_URL: "/d/6vjVx-1V8/ccip-long-running-tests" with: test_command_to_run: cd ./integration-tests/ccip-tests && go test -v -timeout 70m -count=1 -json -run ${{ matrix.type.run }} ./load 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci test_download_vendor_packages_command: cd ./integration-tests && go mod download # Load default test secrets test_secrets_defaults_base64: ${{ secrets.CCIP_DEFAULT_TEST_SECRETS }} # Override default test secrets with custom test secrets if provided test_secrets_override_base64: ${{ secrets[inputs.test_secrets_override_key] }} token: ${{ secrets.GITHUB_TOKEN }} go_mod_path: ./integration-tests/go.mod QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }} triggered_by: ${{ env.TEST_TRIGGERED_BY }} publish_check_name: ${{ matrix.type.name }} artifacts_location: ./integration-tests/load/logs/payload_ccip.json aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} cache_key_id: ccip-load-${{ env.MOD_CACHE_VERSION }} cache_restore_only: "true" should_cleanup: "true" # Reporting Jobs start-slack-thread: name: Start Slack Thread if: ${{ failure() && needs.ccip-load-test.result != 'skipped' && needs.ccip-load-test.result != 'cancelled' }} environment: integration outputs: thread_ts: ${{ steps.slack.outputs.thread_ts }} permissions: checks: write pull-requests: write id-token: write contents: read runs-on: ubuntu-latest needs: [ccip-load-test] steps: - name: Debug Result run: echo ${{ join(needs.*.result, ',') }} - name: Main Slack Notification uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0 id: slack with: channel-id: "#ccip-testing" payload: | { "attachments": [ { "color": "${{ contains(join(needs.*.result, ','), 'failure') && '#C62828' || '#2E7D32' }}", "blocks": [ { "type": "header", "text": { "type": "plain_text", "text": "CCIP load tests results ${{ contains(join(needs.*.result, ','), 'failure') && ':x:' || ':white_check_mark:'}}", "emoji": true } }, { "type": "divider" }, { "type": "section", "text": { "type": "mrkdwn", "text": "<${{ github.server_url }}/${{ github.repository }}/${{contains(github.ref_name, 'release') && 'releases/tag' || 'tree'}}/${{ github.ref_name }}|${{ github.ref_name }}> | <${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }}|${{ github.sha }}> | <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|Run>" } } ] } ] } env: SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }} post-test-results-to-slack: name: Post Test Results if: ${{ failure() && needs.start-slack-thread.result != 'skipped' && needs.start-slack-thread.result != 'cancelled' }} environment: integration permissions: checks: write pull-requests: write id-token: write contents: read runs-on: ubuntu-latest needs: start-slack-thread steps: - name: Checkout the repo uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 with: ref: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha }} - name: Post Test Results uses: ./.github/actions/notify-slack-jobs-result with: github_token: ${{ github.token }} github_repository: ${{ github.repository }} workflow_run_id: ${{ github.run_id }} github_job_name_regex: ^CCIP (.*)$ message_title: CCIP Jobs slack_channel_id: "#ccip-testing" slack_bot_token: ${{ secrets.QA_SLACK_API_KEY }} slack_thread_ts: ${{ needs.start-slack-thread.outputs.thread_ts }} # End Reporting Jobs