node-rdkafka/deps/librdkafka/.semaphore/semaphore.yml

Node.js bindings for librdkafka

version: v1.0
name: 'librdkafka build and release artifact pipeline'
agent:
  machine:
    type: s1-prod-ubuntu24-04-arm64-00
execution_time_limit:
  hours: 3

auto_cancel:
  running:
    when: "branch != 'master'"

global_job_config:
  prologue:
    commands:
      - checkout
      - mkdir artifacts
      - mkdir dest
  env_vars:
    - name: CACHE_TAG
      value: '7'
    - name: CI
      value: 'true'
    - name: KAFKA_VERSION
      value: '4.0.0'
    - name: CP_VERSION
      value: '8.0.0'
    # Shows plain output from docker build (no progress bars)
    - name: BUILDKIT_PROGRESS
      value: plain

blocks:
  - name: 'OSX arm64/m1'
    dependencies: []
    run:
      when: "tag =~ '^v[0-9]\\.'"
    task:
      agent:
        machine:
          type: s1-macos-15-arm64-8
      env_vars:
        - name: ARTIFACT_KEY
          value: p-librdkafka__plat-osx__arch-arm64__lnk-all
      epilogue:
        commands:
          - '[[ -z $SEMAPHORE_GIT_TAG_NAME ]] || artifact push workflow artifacts/ --destination artifacts/${ARTIFACT_KEY}/'
      jobs:
        - name: 'Build'
          commands:
            - ./configure --install-deps --source-deps-only --enable-static --disable-lz4-ext --enable-strip
            - make -j all examples check
            - examples/rdkafka_example -X builtin.features
            - otool -L src/librdkafka.dylib
            - otool -L src-cpp/librdkafka++.dylib
            - make -j -C tests build
            - make -C tests run_local_quick
            - DESTDIR="$PWD/dest" make install
            - (cd dest && tar cvzf ../artifacts/librdkafka.tgz .)


  - name: 'OSX x64'
    dependencies: []
    run:
      when: "tag =~ '^v[0-9]\\.'"
    task:
      agent:
        machine:
          type: s1-macos-15-amd64-6
      env_vars:
        - name: ARTIFACT_KEY
          value: p-librdkafka__plat-osx__arch-x64__lnk-all
      epilogue:
        commands:
          - '[[ -z $SEMAPHORE_GIT_TAG_NAME ]] || artifact push workflow artifacts/ --destination artifacts/${ARTIFACT_KEY}/'
      jobs:
        - name: 'Build'
          commands:
            - ./configure --install-deps --source-deps-only --enable-static --disable-lz4-ext --enable-strip
            - make -j all examples check
            - examples/rdkafka_example -X builtin.features
            - otool -L src/librdkafka.dylib
            - otool -L src-cpp/librdkafka++.dylib
            - make -j -C tests build
            - make -C tests run_local_quick
            - DESTDIR="$PWD/dest" make install
            - (cd dest && tar cvzf ../artifacts/librdkafka.tgz .)


  - name: 'Style check'
    dependencies: []
    skip:
      # Skip for release tags, we don't want style checks
      # to fail the release build.
      when: "tag =~ '^v[0-9]\\.'"
    task:
      agent:
        machine:
          type: s1-prod-ubuntu24-04-arm64-00
      jobs:
        - name: 'Style check'
          commands:
            - packaging/tools/run-in-docker.sh test-runner-arm64-${CACHE_TAG} packaging/tools/run-style-check.sh

  - name: 'Build documentation'
    dependencies: []
    task:
      agent:
        machine:
          type: s1-prod-ubuntu24-04-arm64-1
      jobs:
        - name: 'Generate documentation'
          commands:
            - packaging/tools/run-in-docker.sh test-runner-arm64-${CACHE_TAG} make docs
            - (cd staging-docs && tar cvzf ../artifacts/librdkafka-docs.tgz .)
            - '[[ -z $SEMAPHORE_GIT_TAG_NAME ]] || artifact push workflow artifacts/librdkafka-docs.tgz --destination artifacts/librdkafka-docs.tgz'


  - name: 'Linux Ubuntu amd64: source build'
    dependencies: []
    skip:
      # Skip for release tags, we don't want flaky CI tests
      # to fail the release build.
      when: "tag =~ '^v[0-9]\\.'"
    task:
      agent:
        machine:
          type: s1-prod-ubuntu24-04-amd64-1
      env_vars:
        - name: CFLAGS
          value: -std=gnu90 # Test minimum C standard, default in CentOS 7
      prologue:
        commands:
          - '[[ -z $DOCKERHUB_APIKEY ]] || docker login --username $DOCKERHUB_USER --password $DOCKERHUB_APIKEY'
      jobs:
        - name: 'Build configuration checks'
          commands:
            - packaging/tools/run-in-docker.sh test-runner-amd64-${CACHE_TAG} ./packaging/tools/build-configurations-checks.sh


  - name: 'Linux x64: release artifact docker builds'
    dependencies: []
    run:
      when: "tag =~ '^v[0-9]\\.'"
    task:
      agent:
        machine:
          type: s1-prod-ubuntu24-04-amd64-1
      prologue:
        commands:
          - '[[ -z $DOCKERHUB_APIKEY ]] || docker login --username $DOCKERHUB_USER --password $DOCKERHUB_APIKEY'
      epilogue:
        commands:
          - '[[ -z $SEMAPHORE_GIT_TAG_NAME ]] || artifact push workflow artifacts/ --destination artifacts/${ARTIFACT_KEY}/'
      jobs:
        - name: 'Build: centos8 glibc +gssapi'
          env_vars:
            - name: ARTIFACT_KEY
              value: p-librdkafka__plat-linux__dist-centos8__arch-x64__lnk-std__extra-gssapi
          commands:
            - packaging/tools/build-release-artifacts.sh quay.io/pypa/manylinux_2_28_x86_64:2024.07.01-1 artifacts/librdkafka.tgz

        - name: 'Build: centos8 glibc'
          env_vars:
            - name: ARTIFACT_KEY
              value: p-librdkafka__plat-linux__dist-centos8__arch-x64__lnk-all
          commands:
            - packaging/tools/build-release-artifacts.sh --disable-gssapi quay.io/pypa/manylinux_2_28_x86_64:2024.07.01-1 artifacts/librdkafka.tgz

        - name: 'Build: alpine musl +gssapi'
          env_vars:
            - name: ARTIFACT_KEY
              value: p-librdkafka__plat-linux__dist-alpine__arch-x64__lnk-std__extra-gssapi
          commands:
            - packaging/tools/build-release-artifacts.sh alpine:3.16.9 artifacts/librdkafka.tgz

        - name: 'Build: alpine musl'
          env_vars:
            - name: ARTIFACT_KEY
              value: p-librdkafka__plat-linux__dist-alpine__arch-x64__lnk-all
          commands:
            - packaging/tools/build-release-artifacts.sh --disable-gssapi alpine:3.16.9 artifacts/librdkafka.tgz


  - name: 'Linux arm64: release artifact docker builds'
    dependencies: []
    run:
      when: "tag =~ '^v[0-9]\\.'"
    task:
      agent:
        machine:
          type: s1-prod-ubuntu24-04-arm64-1
      prologue:
        commands:
          - '[[ -z $DOCKERHUB_APIKEY ]] || docker login --username $DOCKERHUB_USER --password $DOCKERHUB_APIKEY'
      epilogue:
        commands:
          - '[[ -z $SEMAPHORE_GIT_TAG_NAME ]] || artifact push workflow artifacts/ --destination artifacts/${ARTIFACT_KEY}/'
      jobs:
        - name: 'Build: centos8 glibc +gssapi'
          env_vars:
            - name: ARTIFACT_KEY
              value: p-librdkafka__plat-linux__dist-centos8__arch-arm64__lnk-std__extra-gssapi
          commands:
            - packaging/tools/build-release-artifacts.sh quay.io/pypa/manylinux_2_28_aarch64:2024.07.01-1 artifacts/librdkafka.tgz

        - name: 'Build: centos8 glibc'
          env_vars:
            - name: ARTIFACT_KEY
              value: p-librdkafka__plat-linux__dist-centos8__arch-arm64__lnk-all
          commands:
            - packaging/tools/build-release-artifacts.sh --disable-gssapi quay.io/pypa/manylinux_2_28_aarch64:2024.07.01-1 artifacts/librdkafka.tgz

        - name: 'Build: alpine musl +gssapi'
          env_vars:
            - name: ARTIFACT_KEY
              value: p-librdkafka__plat-linux__dist-alpine__arch-arm64__lnk-all__extra-gssapi
          commands:
            - packaging/tools/build-release-artifacts.sh alpine:3.16.9 artifacts/librdkafka.tgz

        - name: 'Build: alpine musl'
          env_vars:
            - name: ARTIFACT_KEY
              value: p-librdkafka__plat-linux__dist-alpine__arch-arm64__lnk-all
          commands:
            - packaging/tools/build-release-artifacts.sh --disable-gssapi alpine:3.16.9 artifacts/librdkafka.tgz


  - name: 'Windows x64: MinGW-w64'
    dependencies: []
    run:
      when: "tag =~ '^v[0-9]\\.'"
    task:
      agent:
        machine:
          type: s1-prod-windows
      env_vars:
        - name: CHERE_INVOKING
          value: 'yes'
        - name: MSYSTEM
          value: UCRT64
      prologue:
        commands:
          # Set up msys2
          - "& .\\win32\\setup-msys2.ps1"
      epilogue:
        commands:
          - if ($env:SEMAPHORE_GIT_TAG_NAME -ne "") { artifact push workflow artifacts/ --destination artifacts/$Env:ARTIFACT_KEY/ }
      jobs:
        - name: 'Build: MinGW-w64 Dynamic'
          env_vars:
            - name: ARTIFACT_KEY
              value: p-librdkafka__plat-windows__dist-mingw__arch-x64__lnk-std
          commands:
            - C:\msys64\usr\bin\bash -lc './packaging/mingw-w64/semaphoreci-build.sh ./artifacts/librdkafka.tgz'

        - name: 'Build: MinGW-w64 Static'
          env_vars:
            - name: ARTIFACT_KEY
              value: p-librdkafka__plat-windows__dist-mingw__arch-x64__lnk-static
          commands:
            - C:\msys64\usr\bin\bash -lc './packaging/mingw-w64/semaphoreci-build.sh --static ./artifacts/librdkafka.tgz'

  - name: 'Windows x64: Windows SDK 10.0 / MSVC v142 / VS 2019'
    dependencies: []
    run:
      when: "tag =~ '^v[0-9]\\.'"
    task:
      agent:
        machine:
          type: s1-prod-windows
      env_vars:
        # Disable vcpkg telemetry
        - name: VCPKG_DISABLE_METRICS
          value: 'yes'
      prologue:
        commands:
          # install vcpkg in the parent directory.
          - cd ..
          # Setup vcpkg
          - "& .\\librdkafka\\win32\\setup-vcpkg.ps1"
          - cd librdkafka
      epilogue:
        commands:
          - Get-ChildItem . -include *.dll -recurse
          - Get-ChildItem . -include *.lib -recurse
          - if ($env:SEMAPHORE_GIT_TAG_NAME -ne "") { artifact push workflow artifacts/ --destination artifacts/$Env:ARTIFACT_KEY/ }
      jobs:
        - name: 'Build: MSVC x64'
          env_vars:
            - name: triplet
              value: x64-windows
            - name: ARTIFACT_KEY
              value: p-librdkafka__plat-windows__dist-msvc__arch-x64__lnk-std
          commands:
            - "& .\\win32\\msbuild.ps1 -platform x64"
            - "& .\\win32\\package-zip.ps1 -platform x64"
        - name: 'Build: MSVC x86'
          env_vars:
            - name: triplet
              value: x86-windows
            - name: ARTIFACT_KEY
              value: p-librdkafka__plat-windows__dist-msvc__arch-x86__lnk-std
          commands:
            - "& .\\win32\\msbuild.ps1 -platform Win32"
            - "& .\\win32\\package-zip.ps1 -platform Win32"

  - name: 'Packaging'
    dependencies:
      - 'Build documentation'
      - 'OSX arm64/m1'
      - 'OSX x64'
      - 'Linux x64: release artifact docker builds'
      - 'Linux arm64: release artifact docker builds'
      - 'Windows x64: MinGW-w64'
      - 'Windows x64: Windows SDK 10.0 / MSVC v142 / VS 2019'
    run:
      when: "tag =~ '^v[0-9]\\.'"
    task:
      agent:
        machine:
          type: s1-prod-ubuntu24-04-amd64-2
      jobs:
        - name: 'Build NuGet and static packages'
          commands:
            # Get all artifacts from previous jobs in this workflow/pipeline.
            - artifact pull workflow artifacts
            - mkdir -p packages
            # Prepare packaging tools
            - cd packaging/nuget
            - python3 -m pip install -U -r requirements.txt
            # Create NuGet package
            # We need --ignore-tag since the jobs don't add the tag to
            # the artifact path, and they don't need to since these artifacts
            # are part of the same workflow.
            - ./release.py --directory ../../artifacts --ignore-tag --class NugetPackage ${SEMAPHORE_GIT_TAG_NAME}
            - cp -v librdkafka.redist.*.nupkg ../../packages
            # Create static package
            - ./release.py --directory ../../artifacts --ignore-tag --class StaticPackage ${SEMAPHORE_GIT_TAG_NAME}
            - cp -v librdkafka-static-bundle*.tgz ../../packages
            - cd ../../
            # Copy generated docs to packages for inclusion in the tar ball
            - cp -v artifacts/librdkafka-docs.tgz packages/
            # Maker super tar ball of all packages
            - cd packages
            - tar cvf librdkafka-packages-${SEMAPHORE_GIT_TAG_NAME}-${SEMAPHORE_WORKFLOW_ID}.tar .
            # Provide some extra details
            - ls -la
            - sha256sum *
            - cd ..
            # Upload all packages to project artifact store
            - artifact push project packages --destination librdkafka-packages-${SEMAPHORE_GIT_TAG_NAME}-${SEMAPHORE_WORKFLOW_ID}
            - echo Thank you

promotions:
  - name: Run local tests on all architectures and integration tests
    pipeline_file: semaphore-integration.yml
    auto_promote_on:
      - result: passed
        branch:
        - "master"
  # Manual promotion only, the auto promotion on master happens after 
  # integration tests pass.
  - name: Run all tests
    pipeline_file: run-all-tests.yml
    parameters:
      env_vars:
        - required: true
          name: TEST_KAFKA_GIT_REF
          default_value: 4.0.0
        - required: true
          name: TEST_TYPE
          default_value: plaintext,ssl
        - required: true
          name: TEST_ARCHES
          default_value: x86_64,aarch64
        - required: true
          name: TEST_PARALLEL
          default_value: "1"