@contiamo/dev/docker-compose.yml

Dev environment for contiamo

version: "3.7"
services:
  hub:
    image: ${HUB_IMAGE:-eu.gcr.io/dev-and-test-env/hub:dev}
    environment:
      JAEGER_DISABLED: "false"
      JAEGER_AGENT_HOST: tracing
      TENJIN_ENABLE_TEST_CONNECTION: "${TENJIN_ENABLE_TEST_CONNECTION:-true}"
      TENJIN_ENABLE_METADATA: "${TENJIN_ENABLE_METADATA:-true}"
      PROFILER_ENABLE_V2: "${PROFILER_ENABLE_V2:-true}"
    command:
      - "/bin/hub"
      - "--debug=true"
      - "--log-level=DEBUG"
      - "--mode=server"
      - "--http-address=:9192"
      - "--metrics-health-address=:9090"
      - "--metadb-host=metadb"
      - "--metadb-port=5433"
      - "--metadb-name=hub"
      - "--metadb-user=user"
      - "--metadb-password-path=/db/password"
      - "--metadb-conn-pool-size=10"
      - "--profiling=:9292"
      - "--jwt-public-key-path=/config/auth/idp.crt"
      - "--jwt-private-key-path=/config/auth/idp.key"
      - "--pantheon-url=http://pantheon:4300"
      - "--tenjin-url=http://tenjin:8080"
      - "--datastore-url=http://datastore:9191"
      - "--profiler-url=http://profiler:8080"
      - "--encryption-passphrase-path=/db/password"
      - "--messages-endpoint=auth:3001"
      - "--messages-ca-cert=/config/auth/idp.crt"
      - "--messages-auth-header-name=token"
      - "--messages-insecure"
      - "--idp-endpoint=auth:3001"
      - "--idp-ca-cert=/config/auth/idp.crt"
      - "--idp-insecure"
      # must be the url of the service the provides the ingest endpoint
      # we call it the hub-url because we plan to deprecate the sync-ingester
      # and implement the ingest endpoint direclty in Hub
      - "--hub-url=http://sync-ingester:3333"
    volumes:
      - ./config/db-password:/db/password
      - ./config/auth:/config/auth
    ports:
      - "127.0.0.1:9192:9192"
    depends_on:
      - datastore
      - pantheon
      - metadb
      - profiler
      - auth

  hub-worker:
    image: ${HUB_IMAGE:-eu.gcr.io/dev-and-test-env/hub:dev}
    # max retry count is an undocumented feature found here
    # https://github.com/docker/compose/blob/master/compose/config/types.py#L85
    environment:
      JAEGER_DISABLED: "false"
      JAEGER_AGENT_HOST: tracing
      TENJIN_ENABLE_TEST_CONNECTION: "${TENJIN_ENABLE_TEST_CONNECTION:-true}"
      TENJIN_ENABLE_METADATA: "${TENJIN_ENABLE_METADATA:-true}"
      PROFILER_ENABLE_V2: "${PROFILER_ENABLE_V2:-true}"
    command:
      - "/bin/hub"
      - "--debug=true"
      - "--log-level=DEBUG"
      - "--mode=worker"
      - "--metrics-health-address=:9090"
      - "--metadb-host=metadb"
      - "--metadb-port=5433"
      - "--metadb-name=hub"
      - "--metadb-user=user"
      - "--metadb-password-path=/db/password"
      - "--metadb-conn-pool-size=10"
      - "--profiling=:9292"
      - "--jwt-public-key-path=/config/auth/idp.crt"
      - "--jwt-private-key-path=/config/auth/idp.key"
      - "--pantheon-url=http://pantheon:4300"
      - "--tenjin-url=http://tenjin:8080"
      - "--datastore-url=http://datastore:9191"
      - "--profiler-url=http://profiler:8080"
      - "--encryption-passphrase-path=/db/password"
      - "--messages-endpoint=auth:3001"
      - "--messages-ca-cert=/config/auth/idp.crt"
      - "--messages-auth-header-name=token"
      - "--messages-insecure"
      - "--idp-endpoint=auth:3001"
      - "--idp-ca-cert=/config/auth/idp.crt"
      - "--idp-insecure"
      # must be the url of the service the provides the ingest endpoint
      # we call it the hub-url because we plan to deprecate the sync-ingester
      # and implement the ingest endpoint direclty in Hub
      - "--hub-url=http://sync-ingester:3333"

    volumes:
      - ./config/db-password:/db/password
      - ./config/auth:/config/auth
    depends_on:
      - datastore
      - pantheon
      - metadb
      - profiler
      - auth

  graphql:
    image: ${GRAPHQL_IMAGE:-eu.gcr.io/dev-and-test-env/pgql-server:dev}
    environment:
      JAEGER_DISABLED: "false"
      JAEGER_AGENT_HOST: tracing
      PGQL_ENABLE_PLAYGROUND: "false"
      PGQL_LOG_LEVEL: "DEBUG"
      PGQL_PANTHEON_HOST: pantheon
      PGQL_PANTHEON_PORT: "5431"
      PGQL_PANTHEON_DISABLE_SSL: "true"
      PGQL_PGBOUNCER_HOST: pgbouncer
      PGQL_PGBOUNCER_PORT: "6432"
      PGQL_PGBOUNCER_DISABLE_SSL: "true"
      PGQL_HUB_URL: http://hub:9192
      PGQL_JWT_PUBLIC_KEY_PATH: /config/auth/idp.crt
      PGQL_JWT_PRIVATE_KEY_PATH: /config/auth/key.pem
    volumes:
      - ./config/auth:/config/auth
    depends_on:
      - hub
      - pantheon

  datastore:
    image: ${DATASTORE_IMAGE:-eu.gcr.io/dev-and-test-env/datastore-manager:dev}
    ports:
      - "127.0.0.1:9191:9191"
    environment:
      JAEGER_DISABLED: "false"
      JAEGER_AGENT_HOST: tracing
    command:
      - "/bin/ds-manager"
      - "--log-level=DEBUG"
      - "--http-address=:9191"
      - "--metrics-health-address=:9090"
      - "--encryption-passphrase-path=/enc/passphrase"
      - "--pantheon-url=http://pantheon:4300"
      - "--metadb-host=metadb"
      - "--metadb-port=5433"
      - "--metadb-name=datastore"
      - "--metadb-user=user"
      - "--metadb-password-path=/db/password"
      - "--metadb-conn-pool-size=10"
      - "--storagedb-host=metadb"
      - "--storagedb-port=5433"
      - "--storagedb-user=user"
      - "--storagedb-password-path=/db/password"
      - "--storagedb-conn-pool-size=10"
      - "--blobstorage-host=blobstorage:9000"
      - "--blobstorage-access-key=minioaccesskey"
      - "--blobstorage-secret-key-path=/blobstorage/secret"
      - "--blobstorage-ssl=false"
      - "--blobstorage-bucket=ds-mgr-uploads"
      - "--blobstorage-path-prefix=local/lemon"
      - "--profiling=:9292"
      - "--jwt-public-key-path=/config/auth/idp.crt"
    volumes:
      - ./config/enc-passphrase:/enc/passphrase
      - ./config/db-password:/db/password
      - ./config/blobstore/secret_key:/blobstorage/secret
      - ./config/auth:/config/auth
    depends_on:
      - metadb
      - blobstorage

  ds-worker:
    image: ${DATASTORE_IMAGE:-eu.gcr.io/dev-and-test-env/datastore-manager:dev}
    # max retry count is an undocumented feature found here
    # https://github.com/docker/compose/blob/master/compose/config/types.py#L85
    restart: on-failure:5
    environment:
      JAEGER_AGENT_HOST: tracing
    command:
      - "/bin/ds-ingest"
      - "--log-level=DEBUG"
      - "--metrics-health-address=:9090"
      - "--metadb-host=metadb"
      - "--metadb-port=5433"
      - "--metadb-name=datastore"
      - "--metadb-user=user"
      - "--metadb-password-path=/db/password"
      - "--metadb-conn-pool-size=10"
      - "--storagedb-host=metadb"
      - "--storagedb-port=5433"
      - "--storagedb-user=user"
      - "--storagedb-password-path=/db/password"
      - "--storagedb-conn-pool-size=10"
      - "--blobstorage-host=blobstorage:9000"
      - "--blobstorage-access-key=minioaccesskey"
      - "--blobstorage-secret-key-path=/blobstorage/secret"
      - "--blobstorage-ssl=false"
      - "--blobstorage-bucket=ds-mgr-uploads"
      - "--blobstorage-path-prefix=local/lemon"
      - "--pantheon-postgres=postgres://pantheon:5431?user=datastore&sslmode=disable"
      - "--pantheon-jwt-signing-key=/config/auth/idp.key"
    volumes:
      - ./config/db-password:/db/password
      - ./config/blobstore/secret_key:/blobstorage/secret
      - ./config/auth:/config/auth
    depends_on:
      - metadb
      - blobstorage
      - datastore

  pantheon:
    image: ${PANTHEON_IMAGE:-eu.gcr.io/dev-and-test-env/pantheon:latest}
    command:
      - -Dconfig.resource=environments/production.conf
      - -J-javaagent:conf/jmx_prometheus_javaagent.jar=8081:/config/pantheon/metrics.yaml
      - -Djava.rmi.server.hostname=localhost
      - -Dcom.sun.management.jmxremote.port=5001
      - -Dcom.sun.management.jmxremote.rmi.port=5001
      - -Dcom.sun.management.jmxremote.ssl=false
      - -Dcom.sun.management.jmxremote.local.only=true
      - -Dcom.sun.management.jmxremote.access.file=/tmp/jmx-security/jmxremote.access
      - -Dcom.sun.management.jmxremote.password.file=/tmp/jmx-security/jmxremote.password
      - -J-XX:MaxRAMPercentage=50.0
      - -J-XX:MinRAMPercentage=10.0
      - -J-XX:InitialRAMPercentage=10.0
      # enabling policies
      # - -Dpantheon.flags.policies.onlytoplevel=true
      # - -Dpantheon.flags.policies.disable=false
    environment:
      JDBC_DATABASE_URL: jdbc:postgresql://metadb:5433/pantheon?user=pantheon&password=test
      PLAY_HTTP_SECRET_KEY: secret
      DATASTORE_API_URL: http://datastore:9191/internal/api/v1/
      HUB_BASE_URL: http://hub:9192/internal/api/v1/
      IDP_BASE_URL: https://auth:8443/api/v2/ # pg_wire
      SIGN_AND_VERIFY_ALGO: RS512
      SERVICE_TOKEN_KEY: /config/auth/idp.key
      AUTHENTICATE_CERT: /config/auth/idp.crt
      TRACING_ENABLED: "true"
      TRACING_HOST: tracing
      TRACING_SYNC: "true"
      LOKI_ENABLED: "false"
      # async query creds
      S3_BUCKET: pantheon-datasource-test
      S3_HOST: http://blobstorage:9000/
      S3_PATH_STYLE_ACCESS: "true"
      AWS_ACCESS_KEY_ID: minioaccesskey
      AWS_SECRET_ACCESS_KEY: miniosecretkey
      # datasets creds
      DATASETS_S3_BUCKET: ${DATASETS_S3_BUCKET:-}
      DATASETS_AWS_ACCESS_KEY_ID: ${DATASETS_AWS_ACCESS_KEY_ID:-}
      DATASETS_AWS_SECRET_ACCESS_KEY: ${DATASETS_AWS_SECRET_ACCESS_KEY:-}
    ports:
      - "127.0.0.1:5429:5431"
      - "127.0.0.1:5001:5001"
      - "127.0.0.1:4040:4040"
    volumes:
      - ./config/auth:/config/auth
      - ./config/pantheon:/config/pantheon
    depends_on:
      - auth
      - metadb
      - blobstorage

  pgbouncer:
    image: ${PGBOUNCER_IMAGE:-eu.gcr.io/dev-and-test-env/pgbouncer:latest}
    ports:
      - "127.0.0.1:6432:6432"
    depends_on:
      - hub
      - auth
      - metadb

  pantheon-stunnel:
    image: eu.gcr.io/dev-and-test-env/stunnel:v0.0.19
    ports:
      - "5435:5433"
    volumes:
      - ./config/pg-stunnel/cert:/cert
      - ./config/pg-stunnel/config:/etc/stunnel # Override config
    depends_on:
      - pantheon

  postgresql-foodmart:
    image: eu.gcr.io/dev-and-test-env/foodmart-postgres:v2.0.2-postgis-preloaded
    environment:
      PGPORT: "5434"
      POSTGRES_USER: "foodmart"
      POSTGRES_DB: "foodmart"
      POSTGRES_PASSWORD: "test"
    ports:
      - "127.0.0.1:5434:5434"
    logging:
      options:
        max-size: "10m"
        max-file: "3"

  metadb:
    image: postgres:12
    #  ${POSTGRES_ARGS:--c log_statement=all}
    command: -p 5433 ${POSTGRES_ARGS:--c log_connections=on}
    environment:
      POSTGRES_USER: user
      POSTGRES_PASSWORD_FILE: /db/password
      PGPORT: "5433"
    ports:
      - "127.0.0.1:5433:5433"
    logging:
      options:
        max-size: "10m"
        max-file: "3"
    volumes:
      - ./config/db-password:/db/password
      - ./config/dumps:/docker-entrypoint-initdb.d/dumps
      - ./config/datastore/db-bootstrap.sh:/docker-entrypoint-initdb.d/datastore-db-bootstrap.sh
      - ./config/hub/db-bootstrap.sh:/docker-entrypoint-initdb.d/hub-db-bootstrap.sh
      - ./config/auth/db-bootstrap.sh:/docker-entrypoint-initdb.d/auth-db-bootstrap.sh
      - ./config/pantheon/db-bootstrap.sh:/docker-entrypoint-initdb.d/pantheon-db-bootstrap.sh
      - ./config/profiler/db-bootstrap.sh:/docker-entrypoint-initdb.d/profiler-db-bootstrap.sh
      - contiamo-metadb-datavolume:/var/lib/postgresql/data

  blobstorage:
    image: minio/minio:latest # RELEASE.2019-07-24T02-02-23Z
    volumes:
      - contiamo-minio-data-volume:/data
      - ./config/blobstore:/config/blobstore
    ports:
      - "127.0.0.1:9001:9000"
    env_file:
      # should be able to use the secret file, via the env, but haven't gotten it to work yet
      - ./config/blobstore/config.env
    command: server /data

  ui:
    image: ${UI_IMAGE:-eu.gcr.io/dev-and-test-env/contiamo-ui:dev}
    environment:
      DEPLOYMENTURL: http://localhost:9898
    volumes:
      - ./config/ui-config.js:/usr/share/nginx/html/config.js

  auth:
    image: ${AUTH_IMAGE:-eu.gcr.io/dev-and-test-env/idp:dev}
    environment:
      JAEGER_DISABLED: "false"
      JAEGER_AGENT_HOST: tracing
    ports:
      - "127.0.0.1:8443:8443"
    command:
      - "/bin/idp"
      - "--log-level=DEBUG"
      - "--secure-cookie=false"
      - "--bootstrap-file=/config/auth/bootstrap.json"
      - "--domain="
      - "--external-link-base-url=http://localhost:9898"
      - "--external-link-path-prefix=/auth"
      - "--postgres-host=metadb"
      - "--postgres-port=5433"
      - "--postgres-db=simpleidp"
      - "--postgres-user=user"
      - "--postgres-password-path=/db/password"
      - "--grpc-addr=0.0.0.0:3001"
      - "--http-addr=0.0.0.0:8443"
      - "--ca-cert=" # put a CA cert file path here to enable internal TLS
      - "--key=/config/auth/idp.key"
      - "--cert=/config/auth/idp.crt"
      - "--template-dir=/templates"
      - "--encryption-secret-path=/config/auth/encryption_pass.txt"
      - "--smtp-server=smtpmock:2500" # uncomment if you want to use mailslurper for debugging emails (see below)
      - "--smtp-password-path=/db/password"
      - "--tracing=tracing:6831"
      - "--auth-providers-config-file=/config/auth/providers.json"
      - "--integration-config-file=/config/auth/integrations.json"
      # - "--auth-enforce-tenant-prefix=false"
      - "--dev"

    depends_on:
      - metadb
    volumes:
      - ./config/auth:/config/auth
      - ./config/db-password:/db/password

  smtpmock:
    image: adampresley/mailslurper:latest
    ports:
      - "8025:8080"
      - "2500:2500"
      - "8085:8085"
    networks:
      - default

  profiler:
    image: ${PROFILER_IMAGE:-eu.gcr.io/dev-and-test-env/profiler:dev}
    environment:
      PROFILER_PANTHEON_PG_HOST: "pantheon"
      PROFILER_PANTHEON_PG_PORT: "5431"
      PROFILER_PANTHEON_ETL_URL: "http://pantheon:4300"
      PROFILER_USE_ETL: "false"
      PROFILER_REUSE_CACHE: "true"
      PROFILER_DEBUG: "true"
      PROFILER_LOG_LEVEL: "DEBUG"
      PROFILER_METADB_HOST: "metadb"
      PROFILER_METADB_PORT: "5433"
      PROFILER_METADB_USERNAME: "user"
      PROFILER_METADB_PASSWORD_PATH: "/db/password"
      PROFILER_JWT_PUBLIC_KEY_PATH: "/config/auth/idp.crt"
      PROFILER_JWT_PRIVATE_KEY_PATH: "/config/auth/idp.key"
      JAEGER_DISABLED: "false"
      JAEGER_AGENT_HOST: tracing
      TENJIN_URL: "http://tenjin:8080"
      # disable caching for now
      # BLOBSTORAGE_HOST: "blobstorage:9000"
      # BLOBSTORAGE_ACCESS_KEY: "minioaccesskey"
      # BLOBSTORAGE_SECRET_KEY_PATH: "/blobstorage/secret"
      # BLOBSTORAGE_SSL: "false"
      # BLOBSTORAGE_BUCKET: "profiler"
      # BLOBSTORAGE_PATH_PREFIX: "local/profiler/queries"
      # BLOBSTORAGE_TTL: "1h"
    volumes:
      - ./config/db-password:/db/password
      - ./config/blobstore/secret_key:/blobstorage/secret
      - ./config/auth:/config/auth
    # ports:
    #   - "127.0.0.1:8181:8080"
    depends_on:
      - metadb
      - pantheon

  nginx:
    image: nginx:alpine
    ports:
      - "127.0.0.1:9898:80"
    volumes:
      - ./config/ingress/nginx.conf:/etc/nginx/conf.d/default.conf
      - ./config/ingress:/ingress
    depends_on:
      - datastore
      - pantheon
      - ui

  tracing:
    image: jaegertracing/all-in-one:1.22
    environment:
      COLLECTOR_ZIPKIN_HTTP_PORT: "9411"
    ports:
      - "5775:5775/udp"
      - "6831:6831/udp"
      - "6832:6832/udp"
      - "5778:5778"
      - "16686:16686"
      - "14268:14268"
      - "14250:14250"
      - "9411:9411"

  tenjin:
    image: ${TENJIN_IMAGE:-eu.gcr.io/dev-and-test-env/tenjin:latest}
    environment:
      KEY_FILE: "/config/auth/idp.key"
      JAEGER_AGENT_HOST: tracing
    volumes:
      - ./config/auth:/config/auth
    depends_on:
      - hub
    ports:
      - 8881:8080

  marquez-api:
    image: ${MARQUEZ_IMAGE:-eu.gcr.io/dev-and-test-env/marquez:latest}
    environment:
      - MARQUEZ_PORT=${MARQUEZ_API_PORT:-5000}
      - MARQUEZ_ADMIN_PORT=${MARQUEZ_API_ADMIN_PORT:-5001}
    volumes:
      - ./marquez/wait-for-it.sh:/usr/src/app/wait-for-it.sh
    links:
      - "marquez-db:postgres"
    depends_on:
      - marquez-db
    entrypoint: ["./wait-for-it.sh", "marquez-db:5432", "--", "./entrypoint.sh"]

  marquez-web:
    image: ${MARQUEZ_WEB_IMAGE:-eu.gcr.io/dev-and-test-env/marquez-web:latest}
    environment:
      - MARQUEZ_HOST=marquez-api
      # - MARQUEZ_HOST=host.docker.internal
      - MARQUEZ_PORT=${MARQUEZ_API_PORT:-5000}
      - MARQUEZ_BASE_PATH=/marquez
    stdin_open: true
    tty: true
    ports:
      - 3000:3000
    depends_on:
      - marquez-api

  marquez-db:
    image: postgres:12.1
    environment:
      - POSTGRES_USER=postgres
      - POSTGRES_PASSWORD=password
      - MARQUEZ_DB=marquez
      - MARQUEZ_USER=marquez
      - MARQUEZ_PASSWORD=marquez
    volumes:
      - ./marquez/init-db.sh:/docker-entrypoint-initdb.d/init-db.sh
    # Enables SQL statement logging (see: https://www.postgresql.org/docs/12/runtime-config-logging.html#GUC-LOG-STATEMENT)
    # command: ["postgres", "-c", "log_statement=all"]

volumes:
  contiamo-metadb-datavolume: {}
  contiamo-minio-data-volume: {}
  contiamo-pgadmin-volume: {}