# mypy: allow-untyped-defs import json import os import re from collections import OrderedDict from copy import deepcopy import yaml here = os.path.dirname(__file__) def first(iterable): # First item from a list or iterator if not hasattr(iterable, "next"): if hasattr(iterable, "__iter__"): iterable = iter(iterable) else: raise ValueError("Object isn't iterable") return next(iterable) def load_task_file(path): with open(path) as f: return yaml.safe_load(f) def update_recursive(data, update_data): for key, value in update_data.items(): if key not in data: data[key] = value else: initial_value = data[key] if isinstance(value, dict): if not isinstance(initial_value, dict): raise ValueError("Variable %s has inconsistent types " "(expected object)" % key) update_recursive(initial_value, value) elif isinstance(value, list): if not isinstance(initial_value, list): raise ValueError("Variable %s has inconsistent types " "(expected list)" % key) initial_value.extend(value) else: data[key] = value def resolve_use(task_data, templates): rv = {} if "use" in task_data: for template_name in task_data["use"]: update_recursive(rv, deepcopy(templates[template_name])) update_recursive(rv, task_data) rv.pop("use", None) return rv def resolve_name(task_data, default_name): if "name" not in task_data: task_data["name"] = default_name return task_data def resolve_chunks(task_data): if "chunks" not in task_data: return [task_data] rv = [] total_chunks = task_data["chunks"] for i in range(1, total_chunks + 1): chunk_data = deepcopy(task_data) chunk_data["chunks"] = {"id": i, "total": total_chunks} rv.append(chunk_data) return rv def replace_vars(input_string, variables): # TODO: support replacing as a non-string type? variable_re = re.compile(r"(?