diff options
Diffstat (limited to 'integrations/gen_integrations.py')
| -rwxr-xr-x | integrations/gen_integrations.py | 135 |
1 files changed, 118 insertions, 17 deletions
diff --git a/integrations/gen_integrations.py b/integrations/gen_integrations.py index 043c22a77..9510fdc7f 100755 --- a/integrations/gen_integrations.py +++ b/integrations/gen_integrations.py @@ -14,7 +14,6 @@ from referencing.jsonschema import DRAFT7 from ruamel.yaml import YAML, YAMLError AGENT_REPO = 'netdata/netdata' -GO_REPO = 'netdata/go.d.plugin' INTEGRATIONS_PATH = Path(__file__).parent TEMPLATE_PATH = INTEGRATIONS_PATH / 'templates' @@ -23,15 +22,14 @@ JSON_PATH = INTEGRATIONS_PATH / 'integrations.json' CATEGORIES_FILE = INTEGRATIONS_PATH / 'categories.yaml' REPO_PATH = INTEGRATIONS_PATH.parent SCHEMA_PATH = INTEGRATIONS_PATH / 'schemas' -GO_REPO_PATH = REPO_PATH / 'go.d.plugin' DISTROS_FILE = REPO_PATH / '.github' / 'data' / 'distros.yml' METADATA_PATTERN = '*/metadata.yaml' COLLECTOR_SOURCES = [ - (AGENT_REPO, REPO_PATH / 'collectors', True), - (AGENT_REPO, REPO_PATH / 'collectors' / 'charts.d.plugin', True), - (AGENT_REPO, REPO_PATH / 'collectors' / 'python.d.plugin', True), - (GO_REPO, GO_REPO_PATH / 'modules', True), + (AGENT_REPO, REPO_PATH / 'src' / 'collectors', True), + (AGENT_REPO, REPO_PATH / 'src' / 'collectors' / 'charts.d.plugin', True), + (AGENT_REPO, REPO_PATH / 'src' / 'collectors' / 'python.d.plugin', True), + (AGENT_REPO, REPO_PATH / 'src' / 'go' / 'collectors' / 'go.d.plugin' / 'modules', True), ] DEPLOY_SOURCES = [ @@ -39,14 +37,18 @@ DEPLOY_SOURCES = [ ] EXPORTER_SOURCES = [ - (AGENT_REPO, REPO_PATH / 'exporting', True), + (AGENT_REPO, REPO_PATH / 'src' / 'exporting', True), ] NOTIFICATION_SOURCES = [ - (AGENT_REPO, REPO_PATH / 'health' / 'notifications', True), + (AGENT_REPO, REPO_PATH / 'src' / 'health' / 'notifications', True), (AGENT_REPO, INTEGRATIONS_PATH / 'cloud-notifications' / 'metadata.yaml', False), ] +AUTHENTICATION_SOURCES = [ + (AGENT_REPO, INTEGRATIONS_PATH / 'cloud-authentication' / 'metadata.yaml', False), +] + COLLECTOR_RENDER_KEYS = [ 'alerts', 'metrics', @@ -68,6 +70,12 @@ NOTIFICATION_RENDER_KEYS = [ 'troubleshooting', ] +AUTHENTICATION_RENDER_KEYS = [ + 'overview', + 'setup', + 'troubleshooting', +] + CUSTOM_TAG_PATTERN = re.compile('\\{% if .*?%\\}.*?\\{% /if %\\}|\\{%.*?%\\}', flags=re.DOTALL) FIXUP_BLANK_PATTERN = re.compile('\\\\\\n *\\n') @@ -119,6 +127,11 @@ NOTIFICATION_VALIDATOR = Draft7Validator( registry=registry, ) +AUTHENTICATION_VALIDATOR = Draft7Validator( + {'$ref': './authentication.json#'}, + registry=registry, +) + COLLECTOR_VALIDATOR = Draft7Validator( {'$ref': './collector.json#'}, registry=registry, @@ -386,6 +399,51 @@ def load_notifications(): return ret +def _load_authentication_file(file, repo): + debug(f'Loading { file }.') + data = load_yaml(file) + + if not data: + return [] + + try: + AUTHENTICATION_VALIDATOR.validate(data) + except ValidationError: + warn(f'Failed to validate { file } against the schema.', file) + return [] + + if 'id' in data: + data['integration_type'] = 'authentication' + data['_src_path'] = file + data['_repo'] = repo + data['_index'] = 0 + + return [data] + else: + ret = [] + + for idx, item in enumerate(data): + item['integration_type'] = 'authentication' + item['_src_path'] = file + item['_repo'] = repo + item['_index'] = idx + ret.append(item) + + return ret + + +def load_authentications(): + ret = [] + + for repo, path, match in AUTHENTICATION_SOURCES: + if match and path.exists() and path.is_dir(): + for file in path.glob(METADATA_PATTERN): + ret.extend(_load_authentication_file(file, repo)) + elif not match and path.exists() and path.is_file(): + ret.extend(_load_authentication_file(path, repo)) + + return ret + def make_id(meta): if 'monitored_instance' in meta: @@ -399,10 +457,7 @@ def make_id(meta): def make_edit_link(item): - if item['_repo'] == 'netdata/go.d.plugin': - item_path = item['_src_path'].relative_to(GO_REPO_PATH) - else: - item_path = item['_src_path'].relative_to(REPO_PATH) + item_path = item['_src_path'].relative_to(REPO_PATH) return f'https://github.com/{ item["_repo"] }/blob/master/{ item_path }' @@ -657,11 +712,54 @@ def render_notifications(categories, notifications, ids): return notifications, clean_notifications, ids +def render_authentications(categories, authentications, ids): + debug('Sorting authentications.') + + sort_integrations(authentications) + + debug('Checking authentication ids.') + + authentications, ids = dedupe_integrations(authentications, ids) + + clean_authentications = [] + + for item in authentications: + item['edit_link'] = make_edit_link(item) + + clean_item = deepcopy(item) + + for key in AUTHENTICATION_RENDER_KEYS: + + if key in item.keys(): + template = get_jinja_env().get_template(f'{ key }.md') + data = template.render(entry=item, clean=False) + clean_data = template.render(entry=item, clean=True) + + if 'variables' in item['meta']: + template = get_jinja_env().from_string(data) + data = template.render(variables=item['meta']['variables'], clean=False) + template = get_jinja_env().from_string(clean_data) + clean_data = template.render(variables=item['meta']['variables'], clean=True) + else: + data = '' + clean_data = '' + + item[key] = data + clean_item[key] = clean_data + + for k in ['_src_path', '_repo', '_index']: + del item[k], clean_item[k] + + clean_authentications.append(clean_item) + + return authentications, clean_authentications, ids + + def render_integrations(categories, integrations): template = get_jinja_env().get_template('integrations.js') data = template.render( - categories=json.dumps(categories), - integrations=json.dumps(integrations), + categories=json.dumps(categories, indent=4), + integrations=json.dumps(integrations, indent=4), ) OUTPUT_PATH.write_text(data) @@ -670,7 +768,7 @@ def render_json(categories, integrations): JSON_PATH.write_text(json.dumps({ 'categories': categories, 'integrations': integrations, - })) + }, indent=4)) def main(): @@ -680,16 +778,19 @@ def main(): deploy = load_deploy() exporters = load_exporters() notifications = load_notifications() + authentications = load_authentications() collectors, clean_collectors, ids = render_collectors(categories, collectors, dict()) deploy, clean_deploy, ids = render_deploy(distros, categories, deploy, ids) exporters, clean_exporters, ids = render_exporters(categories, exporters, ids) notifications, clean_notifications, ids = render_notifications(categories, notifications, ids) + authentications, clean_authentications, ids = render_authentications(categories, authentications, ids) + - integrations = collectors + deploy + exporters + notifications + integrations = collectors + deploy + exporters + notifications + authentications render_integrations(categories, integrations) - clean_integrations = clean_collectors + clean_deploy + clean_exporters + clean_notifications + clean_integrations = clean_collectors + clean_deploy + clean_exporters + clean_notifications + clean_authentications render_json(categories, clean_integrations) |