读懂 Scion：用一份 CI 账单分清查询、文件物化与网络流量

#!/usr/bin/env python3
"""Fictional CI logical materialization accounting; no network or repository access."""
import csv
import json
import sys

FIELDS = ['job', 'jobs_per_day', 'full_tree_bytes', 'sparse_tree_bytes',
          'needs_files', 'metadata_rpcs_per_job']


def natural(value, field):
    if not value or not value.isascii() or not value.isdigit():
        raise ValueError(f'{field} must be a nonnegative integer')
    return int(value)


def exact_gb(value):
    whole, remainder = divmod(value, 10**9)
    return str(whole) + (('.' + f'{remainder:09d}'.rstrip('0')) if remainder else '')


def calculate(path):
    totals = dict(jobs_per_day=0, file_jobs_per_day=0,
                  full_tree_logical_bytes_per_day=0,
                  sparse_only_logical_bytes_per_day=0,
                  classified_logical_bytes_per_day=0,
                  classified_metadata_rpcs_per_day=0)
    seen = set()
    with open(path, newline='', encoding='utf-8') as f:
        reader = csv.DictReader(f)
        if reader.fieldnames != FIELDS:
            raise ValueError('CSV header must exactly match: ' + ','.join(FIELDS))
        for row in reader:
            if set(row) != set(FIELDS) or any(v is None for v in row.values()):
                raise ValueError(f'line {reader.line_num}: wrong column count')
            job = row['job'].strip()
            if not job or job in seen:
                raise ValueError('job names must be nonempty and unique')
            seen.add(job)
            n, full, sparse, rpcs = [natural(row[k], k) for k in
                ['jobs_per_day', 'full_tree_bytes', 'sparse_tree_bytes',
                 'metadata_rpcs_per_job']]
            if full <= 0 or sparse > full:
                raise ValueError('full_tree_bytes must be positive; sparse cannot exceed full')
            if row['needs_files'] not in ('0', '1'):
                raise ValueError('needs_files must be 0 or 1')
            needs = int(row['needs_files'])
            if needs and sparse == 0:
                raise ValueError('file jobs need positive sparse_tree_bytes')
            totals['jobs_per_day'] += n
            totals['file_jobs_per_day'] += n * needs
            totals['full_tree_logical_bytes_per_day'] += n * full
            totals['sparse_only_logical_bytes_per_day'] += n * sparse
            totals['classified_logical_bytes_per_day'] += n * needs * sparse
            totals['classified_metadata_rpcs_per_day'] += n * rpcs
    if not seen:
        raise ValueError('CSV must contain at least one job row')
    totals['logical_gb_per_day'] = {
        key: exact_gb(totals[key + '_logical_bytes_per_day'])
        for key in ('full_tree', 'sparse_only', 'classified')}
    totals['actual_network_bytes_per_day'] = None
    return totals


def main():
    if len(sys.argv) != 2:
        print('usage: python3 workload.py jobs.csv', file=sys.stderr)
        return 2
    try:
        result = calculate(sys.argv[1])
        output = json.dumps(result, ensure_ascii=False, indent=2)
    except (OSError, ValueError, csv.Error) as exc:
        print(f'error: {exc}', file=sys.stderr)
        return 2
    print(output)
    return 0


if __name__ == '__main__':
    sys.exit(main())

job,jobs_per_day,full_tree_bytes,sparse_tree_bytes,needs_files,metadata_rpcs_per_job
ancestry_gate,200,2000000000,120000000,0,1
changed_path_router,200,2000000000,120000000,0,2
unit_tests,300,2000000000,120000000,1,0
lint,200,2000000000,120000000,1,0
package_check,100,2000000000,120000000,1,0

{
  "jobs_per_day": 1000,
  "file_jobs_per_day": 600,
  "full_tree_logical_bytes_per_day": 2000000000000,
  "sparse_only_logical_bytes_per_day": 120000000000,
  "classified_logical_bytes_per_day": 72000000000,
  "classified_metadata_rpcs_per_day": 600,
  "logical_gb_per_day": {
    "full_tree": "2000",
    "sparse_only": "120",
    "classified": "72"
  },
  "actual_network_bytes_per_day": null
}

job,jobs_per_day,full_tree_bytes,sparse_tree_bytes,needs_files,metadata_rpcs_per_job
ancestry_gate,200,2000000000,120000000,0,1
changed_path_router,200,2000000000,120000000,1,2
unit_tests,300,2000000000,120000000,1,0
lint,200,2000000000,120000000,1,0
package_check,100,2000000000,120000000,1,0

{
  "jobs_per_day": 1000,
  "file_jobs_per_day": 800,
  "full_tree_logical_bytes_per_day": 2000000000000,
  "sparse_only_logical_bytes_per_day": 120000000000,
  "classified_logical_bytes_per_day": 96000000000,
  "classified_metadata_rpcs_per_day": 600,
  "logical_gb_per_day": {
    "full_tree": "2000",
    "sparse_only": "120",
    "classified": "96"
  },
  "actual_network_bytes_per_day": null
}