mashr/templates/mashrTemplates/psql_config.yml

Simple data pipeline framework for GCP's BigQuery

# Template for psql input. Change the required fields.
# Add a (ex: 'created_at') column at the end to include your column
mashr:
  service_account_email: ''
  # the keyfile must be in current working directory
  json_keyfile: ''
  table_id:     ''
  dataset_id:   ''
  project_id:   ''
  # name the integration for bucket, instance and function name.
  # integration_name must match regex (?:[a-z](?:[-a-z0-9]{0,61}[a-z0-9])?)
  integration_name: ''
  # you can set the command that runs embulk here. The example given is for
  # incremental loading. embulk_config.yml is the default name of the config 
  # file generated for embulk:
  embulk_run_command: 'embulk run embulk_config.yml -c embulk_diff.yml'
  embulk_gems:
    - embulk-filter-add_time
    - embulk-filter-timestamp_format
    - embulk-input-postgresql
embulk: # read about embulk here: https://github.com/embulk/embulk
  exec:
    min_output_tasks: 1
  in:
    type: postgresql
    host: your.public.ip
    user: postgres
    password: "password"
    database: postgres
    table: users
    incremental: true
    incremental_columns: [created_at, id]
  filters:
    - type: add_time
      to_column:
        name: mashr_added_at
        type: timestamp
      from_value:
        mode: upload_time
        timestamp_format: "%Y-%m-%d %H:%M:%S"
    - type: timestamp_format
      default_from_timestamp_format: ["yyyy-MM-dd HH:mm:ss.SSS z", "yyyy-MM-dd HH:mm:ss z", "yyyy-MM-dd HH:mm:ss"]
      default_to_timestamp_format: "yyyy-MM-dd HH:mm:ss"
      columns:
        - {name: mashr_added_at}