x-templates:
  # `x-templates` field contains reusable templates for metadata extraction services that user can define under `services` field.
  # See docker-compose docs on re-using configuration fragments: https://docs.docker.com/compose/compose-file/compose-file-v3/#extension-fields
  databricks: &databricks-defaults # Databricks default environment variables
    OUTPUT_PREFIX: /output
    DATABRICKS_CONFIG: /run/secrets/databricks_config
  extract-databricks-service: &extract
    image: databricks-extractor-master
    command:
      - "python"
      - "-m"
      - "marketplace_scripts.databricks.extract_metadata.main"
    privileged: false
    network_mode: host
    environment:
      <<: *databricks-defaults
    secrets:
      - databricks_config

  extract-databricks-lineage-service: &extract-lineage
    image: databricks-extractor-master
    command:
      - "python"
      - "-m"
      - "marketplace_scripts.databricks.extract_lineage.main"
    privileged: false
    network_mode: host
    environment:
      <<: *databricks-defaults
    secrets:
      - databricks_config


# Create a separate service for each Databricks connection:
services:
  # Example Databricks connection
  databricks-example:
    <<: *extract
    environment:
      <<: *databricks-defaults
      INCLUDE_FILTER: '{"DB_1": [], "DB_2": ["SCHEMA_1", "SCHEMA_2"]}'
      EXCLUDE_FILTER: '{"DB_1": ["SCHEMA_1", "SCHEMA_2"]}'
      TEMP_TABLE_REGEX: '.*temp.*|.*tmp.*|.*TEMP.*|.*TMP.*'
      SYSTEM_SCHEMA_REGEX: '["^global_temp$", "^information_schema$"]'
    volumes:
      # You can change './output/databricks-example' to any output location you want
      - ./output/databricks-example:/output

  # Example Databricks lineage connection
  databricks-lineage-example:
    <<: *extract-lineage
    environment:
      <<: *databricks-defaults
      EXTRACT_QUERY_HISTORY: true
      QUERY_HISTORY_START_TIME_MS: 0
    volumes:
      # You can change './output/databricks-lineage-example' to any output location you want
      - ./output/databricks-lineage-example:/output

secrets:
  databricks_config:
    # Change it to the actual location of your databricks.ini file
    file: ./databricks.ini
