sadfzxcv

#!/usr/bin/envbash
#
# python3

This

importscript osloads all tables in /stuff/kdb, runs a hardcoded multi-line Q-syntax query,
import# requestsand writes the results into /home/stuff/output with a timestamped filename.

# DummyMove loggerto importthe kdb directory
importcd logger/stuff/kdb #|| Provided{
 by yourecho environment"Error: Failed to cd into /stuff/kdb. Check that it exists."
  exit 1
}

# ExampleHardcoded Kdb+multi-line REST endpoint
KDB_URL = "http://kdbserver:8080/execute"

def run_query(query, check_name):
    """
    Send a QQ/SQL-like query
# (Adjust the actual code to Kdb+your viause REST,case)
read returning-r True on success, False on failure.
    We do NOT parse/store the response data because the API handles that.
    """
    payload = {
        "query": query,
        "responseFormat": "json"
    }
    try:
        logger.info(f"Sending query for check-d '{check_name}' toMULTI_LINE_SQL Kdb.")
        resp = requests.post(KDB_URL, json=payload, timeout=30)
        resp.raise_for_status()  # Raises error if status != 200
        logger.info(f"Check <<'{check_name}' succeeded. HTTP Status {resp.status_code}.")
        return True
    except requests.exceptions.RequestException as e:
        logger.error(f"Check '{check_name}' failed with error: {e}")
        return False

def main():
    """
    1) Looks in a specified directory (sql_checks_dir) for all .sql files.
    2) Each file's name is treated as the "check" label (minus the .sql extension).
    3) The content of each file is treated as the Q query to run.
    4) We only log success/failure; the REST API handles storing data.
    """
    logger.info("=== Starting FX Data Daily Checks ===")

    # Directory where your .sql files live
    sql_checks_dir = "./checks"

    # If needed, make sure the directory exists
    if not os.path.isdir(sql_checks_dir):
        logger.error(f"SQL checks directory '{sql_checks_dir}' not found.")
        return

    # List all .sql files in that directory
    sql_files = [f for f in os.listdir(sql_checks_dir) if f.endswith(".sql")]

    if not sql_files:
        logger.info(f"No .sql files found in '{sql_checks_dir}'. Nothing to run.")
        return

    for sql_file in sql_files:
        # The check name is just the filename minus extension
        check_name = os.path.splitext(sql_file)[0]
        sql_path = os.path.join(sql_checks_dir, sql_file)

        logger.info(f"Processing check file: {sql_path}")

        # Read the Q query from the file
        try:
            with open(sql_path, "r", encoding="utf-8") as f:
                query = f.read()
        except Exception as e:
            logger.error(f"Failed to read '{sql_path}': {e}")
            continue

        # Run the query
        success = run_query(query, check_name)
        if success:
            logger.info(f"Check '{check_name}' completed successfully.")
        else:
            logger.error(f"Check '{check_name}' encountered an error.")

    logger.info("=== End of FX Data Daily Checks ===")

if __name__ == "__main__":
    main()

 

 

 

 

 

 

 

 

 

 

-- Missing_Values.sqlEQUERY'
select from fxDatatrades
 where null timestamps
   or null symbol
   or null source
   or null date
   or any each null bidSize
   or any each null bidPrice
   or any each null offerSize
   or any each null offerPrice

 

-- Zero_Negative_Quotes.sql
select from fxData
 where any each bidPrice=0.0
    or any each offerPrice=0.0
    or any each bidSize<=0.0
    or any each offerSize<=0.0

-- Large_Rates.sql
select from fxData
 where any each bidPrice>1000.0
    or any each offerPrice>1000.0

-- Crazy_Timestamps.sql
select from fxData
 where year timestamps < 1980
    or year timestamps > 2100

-- Unexpected_Sources.sql
select from fxData
 where not sourcesym in `Bloomberg`Reuters`MyFeedAAPL`MSFT,
       time within 09:30 16:00
EQUERY

# Generate a timestamp
TIMESTAMP=$(date +"%Y%m%d_%H%M%S")

# Hardcoded output directory (ensure this directory exists or create it beforehand)
OUTPUT_DIR="/home/stuff/output"

# Construct the output filename (e.g., output_20250310_143042.csv)
OUTPUT_CSV="${OUTPUT_DIR}/output_${TIMESTAMP}.csv"

# Run q with 1GB workspace
q --w Duplicates.sql
select1000 duplicates: count i <<EOF
 by timestamps,\l symbol, source, bidPrice, offerPrice.
 from fxDataresults: $MULTI_LINE_SQL
 having count\`$OUTPUT_CSV i0: >results
 1 \\
EOF

echo "Query results saved to: $OUTPUT_CSV"