x-templates:
  # `x-templates` field contains reusable templates for metadata extraction services that user can define under `services` field.
  # See docker-compose docs on re-using configuration fragments: https://docs.docker.com/compose/compose-file/compose-file-v3/#extension-fields
  crawler-script: &crawler-script |    
    echo "Running Kafka crawler script..."
    mkdir -p "/tmp/credentials"
    
    echo "Preparing credential file..."
    export CLIENT_CONFIG="$$(bash -c 'cat $$KAFKA_CLIENT_CONFIG')"
    
    if [ -f "$$KAFKA_CA_CERT" ]; 
    then
    echo "CA certificate found, will be added to credentials"
    export CA_CERT="$$(bash -c 'cat $$KAFKA_CA_CERT')"
    fi
    
    if [ -f "$$KAFKA_ACCESS_CERT" ]; 
    then
    echo "Client certificate (access certificate) found, will be added to credentials"
    export ACCESS_CERT="$$(bash -c 'cat $$KAFKA_ACCESS_CERT')"
    fi
    
    if [ -f "$$KAFKA_ACCESS_KEY" ]; 
    then
    echo "Client certificate (access key) found, will be added to credentials"
    export ACCESS_KEY="$$(bash -c 'cat $$KAFKA_ACCESS_KEY')"
    fi
    
    jq -n --arg host "$$KAFKA_BOOTSTRAP_SERVERS" --arg config "$$CLIENT_CONFIG" --arg ca_cert "$$CA_CERT" --arg access_cert "$$ACCESS_CERT" --arg access_key "$$ACCESS_KEY" '{"host":$$host, "config":$$config, "ca_cert":$$ca_cert, "access_cert":$$access_cert, "access_key":$$access_key}' > "/run/secrets/credential.json"
    ./marketplace_scripts/kafka/extraction.sh "/tmp/kafka/kafka/bin" "/run/secrets/credential.json" "$$OUTPUT_PREFIX" "$$SKIP_INTERNAL_TOPICS"

  kafka: &kafka-defaults # Kafka default environment variables
    OUTPUT_PREFIX: /output
    KAFKA_CLIENT_CONFIG: /run/secrets/kafka_client_config
    ADDITIONAL_KAFKA_CONFIG: /run/secrets/additional_config
    KAFKA_CA_CERT: /run/secrets/kafka_ca_cert
    KAFKA_ACCESS_CERT: /run/secrets/kafka_access_cert
    KAFKA_ACCESS_KEY: /run/secrets/kafka_access_key
    KAFKA_BOOTSTRAP_SERVERS: "<HOSTS>"
    SKIP_INTERNAL_TOPICS: "true"


  extract-kafka-service: &extract
    image: kafka-extractor-master
    command:
      - "bash"
      - "-c"
      - *crawler-script
    privileged: false
    network_mode: host
    environment:
      <<: *kafka-defaults
    secrets:
      - kafka_client_config
    # Testing only, remove these
    stdin_open: true # docker run -i
    tty: true        # docker run -t

# Create a separate service for each Kafka connection:
services:
  # Example Apache Kafka connection
  apache-kafka-example:
    <<: *extract
    environment:
      <<: *kafka-defaults
      # Kafka bootstrap servers (semicolon-separated)
      KAFKA_BOOTSTRAP_SERVERS: "localhost:9092"
      # Skip topics that are internal to Kafka (e.g. __consumer_offsets)
      SKIP_INTERNAL_TOPICS: "true"
    volumes:
      # You can change './output/kafka-example' to any output location you want
      - ./output/kafka-example:/output

  # Example Confluent Kafka connection
  confluent-kafka-example:
    <<: *extract
    environment:
      <<: *kafka-defaults
      # Kafka bootstrap servers (semicolon-separated)
      KAFKA_BOOTSTRAP_SERVERS: "localhost:9092"
      # Skip topics that are internal to Kafka (e.g. __consumer_offsets)
      SKIP_INTERNAL_TOPICS: "true"
    volumes:
      # You can change './output/confluent-kafka-example' to any output location you want
      - ./output/confluent-kafka-example:/output

  # Example Aiven Kafka connection
  aiven-kafka-example:
    <<: *extract
    secrets:
      - kafka_client_config
      - kafka_ca_cert
      # Uncomment the following lines if you are using Aiven Kafka with Client Certificate Authentication
#      - kafka_access_cert
#      - kafka_access_key
    environment:
      <<: *kafka-defaults
      # Kafka bootstrap servers (semicolon-separated)
      KAFKA_BOOTSTRAP_SERVERS: "localhost:9092"
      # Skip topics that are internal to Kafka (e.g. __consumer_offsets)
      SKIP_INTERNAL_TOPICS: "true"
    volumes:
      # You can change './output/aiven-kafka-example' to any output location you want
      - ./output/aiven-kafka-example:/output

  # Example Redpanda Kafka connection
  redpanda-kafka-example:
    <<: *extract
    environment:
      <<: *kafka-defaults
      # Kafka bootstrap servers (semicolon-separated)
      KAFKA_BOOTSTRAP_SERVERS: "localhost:9092"
      # Skip topics that are internal to Kafka (e.g. __consumer_offsets)
      SKIP_INTERNAL_TOPICS: "true"
    volumes:
      # You can change './output/redpanda-kafka-example' to any output location you want
      - ./output/redpanda-kafka-example:/output

secrets:
  kafka_client_config:
    # Change it to the actual location of your kafka config file
    file: ./kafka.client.config
  kafka_ca_cert:
    # Change it to the actual location of your kafka CA cert file
    file: ./ca.pem
  kafka_access_cert:
    # Change it to the actual location of your kafka access cert file
    file: ./service.cert
  kafka_access_key:
    # Change it to the actual location of your kafka access key file
    file: ./service.key
