diff options
Diffstat (limited to 'integrations/gen_integrations.py')
| -rwxr-xr-x | integrations/gen_integrations.py | 129 |
1 files changed, 94 insertions, 35 deletions
diff --git a/integrations/gen_integrations.py b/integrations/gen_integrations.py index 6d439a52d..043c22a77 100755 --- a/integrations/gen_integrations.py +++ b/integrations/gen_integrations.py @@ -2,8 +2,10 @@ import json import os +import re import sys +from copy import deepcopy from pathlib import Path from jsonschema import Draft7Validator, ValidationError @@ -17,6 +19,7 @@ GO_REPO = 'netdata/go.d.plugin' INTEGRATIONS_PATH = Path(__file__).parent TEMPLATE_PATH = INTEGRATIONS_PATH / 'templates' OUTPUT_PATH = INTEGRATIONS_PATH / 'integrations.js' +JSON_PATH = INTEGRATIONS_PATH / 'integrations.json' CATEGORIES_FILE = INTEGRATIONS_PATH / 'categories.yaml' REPO_PATH = INTEGRATIONS_PATH.parent SCHEMA_PATH = INTEGRATIONS_PATH / 'schemas' @@ -65,6 +68,9 @@ NOTIFICATION_RENDER_KEYS = [ 'troubleshooting', ] +CUSTOM_TAG_PATTERN = re.compile('\\{% if .*?%\\}.*?\\{% /if %\\}|\\{%.*?%\\}', flags=re.DOTALL) +FIXUP_BLANK_PATTERN = re.compile('\\\\\\n *\\n') + GITHUB_ACTIONS = os.environ.get('GITHUB_ACTIONS', False) DEBUG = os.environ.get('DEBUG', False) @@ -146,11 +152,11 @@ def get_jinja_env(): def strfy(value): - if not isinstance(value, str): - return value - - return ' '.join([v.strip() for v in value.strip().split("\n") if v]).replace('|', '/') - + if isinstance(value, bool): + return "yes" if value else "no" + if isinstance(value, str): + return ' '.join([v.strip() for v in value.strip().split("\n") if v]).replace('|', '/') + return value def get_category_sets(categories): @@ -438,12 +444,17 @@ def render_collectors(categories, collectors, ids): debug('Removing duplicate collectors.') collectors, ids = dedupe_integrations(collectors, ids) + clean_collectors = [] idmap = {i['id']: i for i in collectors} for item in collectors: debug(f'Processing { item["id"] }.') + item['edit_link'] = make_edit_link(item) + + clean_item = deepcopy(item) + related = [] for res in item['meta']['related_resources']['integrations']['list']: @@ -487,23 +498,27 @@ def render_collectors(categories, collectors, ids): for key in COLLECTOR_RENDER_KEYS: if key in item.keys(): template = get_jinja_env().get_template(f'{ key }.md') - data = template.render(entry=item, related=related) + data = template.render(entry=item, related=related, clean=False) + clean_data = template.render(entry=item, related=related, clean=True) if 'variables' in item['meta']['monitored_instance']: template = get_jinja_env().from_string(data) data = template.render(variables=item['meta']['monitored_instance']['variables']) + template = get_jinja_env().from_string(clean_data) + clean_data = template.render(variables=item['meta']['monitored_instance']['variables']) else: data = '' + clean_data = '' item[key] = data + clean_item[key] = clean_data - item['edit_link'] = make_edit_link(item) + for k in ['_src_path', '_repo', '_index']: + del item[k], clean_item[k] - del item['_src_path'] - del item['_repo'] - del item['_index'] + clean_collectors.append(clean_item) - return collectors, ids + return collectors, clean_collectors, ids def render_deploy(distros, categories, deploy, ids): @@ -514,11 +529,14 @@ def render_deploy(distros, categories, deploy, ids): debug('Checking deployment ids.') deploy, ids = dedupe_integrations(deploy, ids) + clean_deploy = [] template = get_jinja_env().get_template('platform_info.md') for item in deploy: debug(f'Processing { item["id"] }.') + item['edit_link'] = make_edit_link(item) + clean_item = deepcopy(item) if item['platform_info']['group']: entries = [ @@ -532,16 +550,27 @@ def render_deploy(distros, categories, deploy, ids): else: entries = [] - data = template.render(entries=entries) + data = template.render(entries=entries, clean=False) + clean_data = template.render(entries=entries, clean=True) + + for method in clean_item['methods']: + for command in method['commands']: + command['command'] = CUSTOM_TAG_PATTERN.sub('', command['command']) + command['command'] = FIXUP_BLANK_PATTERN.sub('', command['command']) item['platform_info'] = data - item['edit_link'] = make_edit_link(item) + clean_item['platform_info'] = clean_data - del item['_src_path'] - del item['_repo'] - del item['_index'] + if 'clean_additional_info' in item: + clean_item['additional_info'] = item['clean_additional_info'] + del item['clean_additional_info'], clean_item['clean_additional_info'] - return deploy, ids + for k in ['_src_path', '_repo', '_index']: + del item[k], clean_item[k] + + clean_deploy.append(clean_item) + + return deploy, clean_deploy, ids def render_exporters(categories, exporters, ids): @@ -553,27 +582,37 @@ def render_exporters(categories, exporters, ids): exporters, ids = dedupe_integrations(exporters, ids) + clean_exporters = [] + for item in exporters: + item['edit_link'] = make_edit_link(item) + + clean_item = deepcopy(item) + for key in EXPORTER_RENDER_KEYS: if key in item.keys(): template = get_jinja_env().get_template(f'{ key }.md') - data = template.render(entry=item) + data = template.render(entry=item, clean=False) + clean_data = template.render(entry=item, clean=True) if 'variables' in item['meta']: template = get_jinja_env().from_string(data) - data = template.render(variables=item['meta']['variables']) + data = template.render(variables=item['meta']['variables'], clean=False) + template = get_jinja_env().from_string(clean_data) + clean_data = template.render(variables=item['meta']['variables'], clean=True) else: data = '' + clean_data = '' item[key] = data + clean_item[key] = clean_data - item['edit_link'] = make_edit_link(item) + for k in ['_src_path', '_repo', '_index']: + del item[k], clean_item[k] - del item['_src_path'] - del item['_repo'] - del item['_index'] + clean_exporters.append(clean_item) - return exporters, ids + return exporters, clean_exporters, ids def render_notifications(categories, notifications, ids): @@ -585,27 +624,37 @@ def render_notifications(categories, notifications, ids): notifications, ids = dedupe_integrations(notifications, ids) + clean_notifications = [] + for item in notifications: + item['edit_link'] = make_edit_link(item) + + clean_item = deepcopy(item) + for key in NOTIFICATION_RENDER_KEYS: if key in item.keys(): template = get_jinja_env().get_template(f'{ key }.md') - data = template.render(entry=item) + data = template.render(entry=item, clean=False) + clean_data = template.render(entry=item, clean=True) if 'variables' in item['meta']: template = get_jinja_env().from_string(data) - data = template.render(variables=item['meta']['variables']) + data = template.render(variables=item['meta']['variables'], clean=False) + template = get_jinja_env().from_string(clean_data) + clean_data = template.render(variables=item['meta']['variables'], clean=True) else: data = '' + clean_data = '' item[key] = data + clean_item[key] = clean_data - item['edit_link'] = make_edit_link(item) + for k in ['_src_path', '_repo', '_index']: + del item[k], clean_item[k] - del item['_src_path'] - del item['_repo'] - del item['_index'] + clean_notifications.append(clean_item) - return notifications, ids + return notifications, clean_notifications, ids def render_integrations(categories, integrations): @@ -617,6 +666,13 @@ def render_integrations(categories, integrations): OUTPUT_PATH.write_text(data) +def render_json(categories, integrations): + JSON_PATH.write_text(json.dumps({ + 'categories': categories, + 'integrations': integrations, + })) + + def main(): categories = load_categories() distros = load_yaml(DISTROS_FILE) @@ -625,14 +681,17 @@ def main(): exporters = load_exporters() notifications = load_notifications() - collectors, ids = render_collectors(categories, collectors, dict()) - deploy, ids = render_deploy(distros, categories, deploy, ids) - exporters, ids = render_exporters(categories, exporters, ids) - notifications, ids = render_notifications(categories, notifications, ids) + collectors, clean_collectors, ids = render_collectors(categories, collectors, dict()) + deploy, clean_deploy, ids = render_deploy(distros, categories, deploy, ids) + exporters, clean_exporters, ids = render_exporters(categories, exporters, ids) + notifications, clean_notifications, ids = render_notifications(categories, notifications, ids) integrations = collectors + deploy + exporters + notifications render_integrations(categories, integrations) + clean_integrations = clean_collectors + clean_deploy + clean_exporters + clean_notifications + render_json(categories, clean_integrations) + if __name__ == '__main__': sys.exit(main()) |