summaryrefslogtreecommitdiffstats
path: root/integrations/gen_integrations.py
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-03-09 13:19:48 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-03-09 13:20:02 +0000
commit58daab21cd043e1dc37024a7f99b396788372918 (patch)
tree96771e43bb69f7c1c2b0b4f7374cb74d7866d0cb /integrations/gen_integrations.py
parentReleasing debian version 1.43.2-1. (diff)
downloadnetdata-58daab21cd043e1dc37024a7f99b396788372918.tar.xz
netdata-58daab21cd043e1dc37024a7f99b396788372918.zip
Merging upstream version 1.44.3.
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'integrations/gen_integrations.py')
-rwxr-xr-xintegrations/gen_integrations.py129
1 files changed, 94 insertions, 35 deletions
diff --git a/integrations/gen_integrations.py b/integrations/gen_integrations.py
index 6d439a52d..043c22a77 100755
--- a/integrations/gen_integrations.py
+++ b/integrations/gen_integrations.py
@@ -2,8 +2,10 @@
import json
import os
+import re
import sys
+from copy import deepcopy
from pathlib import Path
from jsonschema import Draft7Validator, ValidationError
@@ -17,6 +19,7 @@ GO_REPO = 'netdata/go.d.plugin'
INTEGRATIONS_PATH = Path(__file__).parent
TEMPLATE_PATH = INTEGRATIONS_PATH / 'templates'
OUTPUT_PATH = INTEGRATIONS_PATH / 'integrations.js'
+JSON_PATH = INTEGRATIONS_PATH / 'integrations.json'
CATEGORIES_FILE = INTEGRATIONS_PATH / 'categories.yaml'
REPO_PATH = INTEGRATIONS_PATH.parent
SCHEMA_PATH = INTEGRATIONS_PATH / 'schemas'
@@ -65,6 +68,9 @@ NOTIFICATION_RENDER_KEYS = [
'troubleshooting',
]
+CUSTOM_TAG_PATTERN = re.compile('\\{% if .*?%\\}.*?\\{% /if %\\}|\\{%.*?%\\}', flags=re.DOTALL)
+FIXUP_BLANK_PATTERN = re.compile('\\\\\\n *\\n')
+
GITHUB_ACTIONS = os.environ.get('GITHUB_ACTIONS', False)
DEBUG = os.environ.get('DEBUG', False)
@@ -146,11 +152,11 @@ def get_jinja_env():
def strfy(value):
- if not isinstance(value, str):
- return value
-
- return ' '.join([v.strip() for v in value.strip().split("\n") if v]).replace('|', '/')
-
+ if isinstance(value, bool):
+ return "yes" if value else "no"
+ if isinstance(value, str):
+ return ' '.join([v.strip() for v in value.strip().split("\n") if v]).replace('|', '/')
+ return value
def get_category_sets(categories):
@@ -438,12 +444,17 @@ def render_collectors(categories, collectors, ids):
debug('Removing duplicate collectors.')
collectors, ids = dedupe_integrations(collectors, ids)
+ clean_collectors = []
idmap = {i['id']: i for i in collectors}
for item in collectors:
debug(f'Processing { item["id"] }.')
+ item['edit_link'] = make_edit_link(item)
+
+ clean_item = deepcopy(item)
+
related = []
for res in item['meta']['related_resources']['integrations']['list']:
@@ -487,23 +498,27 @@ def render_collectors(categories, collectors, ids):
for key in COLLECTOR_RENDER_KEYS:
if key in item.keys():
template = get_jinja_env().get_template(f'{ key }.md')
- data = template.render(entry=item, related=related)
+ data = template.render(entry=item, related=related, clean=False)
+ clean_data = template.render(entry=item, related=related, clean=True)
if 'variables' in item['meta']['monitored_instance']:
template = get_jinja_env().from_string(data)
data = template.render(variables=item['meta']['monitored_instance']['variables'])
+ template = get_jinja_env().from_string(clean_data)
+ clean_data = template.render(variables=item['meta']['monitored_instance']['variables'])
else:
data = ''
+ clean_data = ''
item[key] = data
+ clean_item[key] = clean_data
- item['edit_link'] = make_edit_link(item)
+ for k in ['_src_path', '_repo', '_index']:
+ del item[k], clean_item[k]
- del item['_src_path']
- del item['_repo']
- del item['_index']
+ clean_collectors.append(clean_item)
- return collectors, ids
+ return collectors, clean_collectors, ids
def render_deploy(distros, categories, deploy, ids):
@@ -514,11 +529,14 @@ def render_deploy(distros, categories, deploy, ids):
debug('Checking deployment ids.')
deploy, ids = dedupe_integrations(deploy, ids)
+ clean_deploy = []
template = get_jinja_env().get_template('platform_info.md')
for item in deploy:
debug(f'Processing { item["id"] }.')
+ item['edit_link'] = make_edit_link(item)
+ clean_item = deepcopy(item)
if item['platform_info']['group']:
entries = [
@@ -532,16 +550,27 @@ def render_deploy(distros, categories, deploy, ids):
else:
entries = []
- data = template.render(entries=entries)
+ data = template.render(entries=entries, clean=False)
+ clean_data = template.render(entries=entries, clean=True)
+
+ for method in clean_item['methods']:
+ for command in method['commands']:
+ command['command'] = CUSTOM_TAG_PATTERN.sub('', command['command'])
+ command['command'] = FIXUP_BLANK_PATTERN.sub('', command['command'])
item['platform_info'] = data
- item['edit_link'] = make_edit_link(item)
+ clean_item['platform_info'] = clean_data
- del item['_src_path']
- del item['_repo']
- del item['_index']
+ if 'clean_additional_info' in item:
+ clean_item['additional_info'] = item['clean_additional_info']
+ del item['clean_additional_info'], clean_item['clean_additional_info']
- return deploy, ids
+ for k in ['_src_path', '_repo', '_index']:
+ del item[k], clean_item[k]
+
+ clean_deploy.append(clean_item)
+
+ return deploy, clean_deploy, ids
def render_exporters(categories, exporters, ids):
@@ -553,27 +582,37 @@ def render_exporters(categories, exporters, ids):
exporters, ids = dedupe_integrations(exporters, ids)
+ clean_exporters = []
+
for item in exporters:
+ item['edit_link'] = make_edit_link(item)
+
+ clean_item = deepcopy(item)
+
for key in EXPORTER_RENDER_KEYS:
if key in item.keys():
template = get_jinja_env().get_template(f'{ key }.md')
- data = template.render(entry=item)
+ data = template.render(entry=item, clean=False)
+ clean_data = template.render(entry=item, clean=True)
if 'variables' in item['meta']:
template = get_jinja_env().from_string(data)
- data = template.render(variables=item['meta']['variables'])
+ data = template.render(variables=item['meta']['variables'], clean=False)
+ template = get_jinja_env().from_string(clean_data)
+ clean_data = template.render(variables=item['meta']['variables'], clean=True)
else:
data = ''
+ clean_data = ''
item[key] = data
+ clean_item[key] = clean_data
- item['edit_link'] = make_edit_link(item)
+ for k in ['_src_path', '_repo', '_index']:
+ del item[k], clean_item[k]
- del item['_src_path']
- del item['_repo']
- del item['_index']
+ clean_exporters.append(clean_item)
- return exporters, ids
+ return exporters, clean_exporters, ids
def render_notifications(categories, notifications, ids):
@@ -585,27 +624,37 @@ def render_notifications(categories, notifications, ids):
notifications, ids = dedupe_integrations(notifications, ids)
+ clean_notifications = []
+
for item in notifications:
+ item['edit_link'] = make_edit_link(item)
+
+ clean_item = deepcopy(item)
+
for key in NOTIFICATION_RENDER_KEYS:
if key in item.keys():
template = get_jinja_env().get_template(f'{ key }.md')
- data = template.render(entry=item)
+ data = template.render(entry=item, clean=False)
+ clean_data = template.render(entry=item, clean=True)
if 'variables' in item['meta']:
template = get_jinja_env().from_string(data)
- data = template.render(variables=item['meta']['variables'])
+ data = template.render(variables=item['meta']['variables'], clean=False)
+ template = get_jinja_env().from_string(clean_data)
+ clean_data = template.render(variables=item['meta']['variables'], clean=True)
else:
data = ''
+ clean_data = ''
item[key] = data
+ clean_item[key] = clean_data
- item['edit_link'] = make_edit_link(item)
+ for k in ['_src_path', '_repo', '_index']:
+ del item[k], clean_item[k]
- del item['_src_path']
- del item['_repo']
- del item['_index']
+ clean_notifications.append(clean_item)
- return notifications, ids
+ return notifications, clean_notifications, ids
def render_integrations(categories, integrations):
@@ -617,6 +666,13 @@ def render_integrations(categories, integrations):
OUTPUT_PATH.write_text(data)
+def render_json(categories, integrations):
+ JSON_PATH.write_text(json.dumps({
+ 'categories': categories,
+ 'integrations': integrations,
+ }))
+
+
def main():
categories = load_categories()
distros = load_yaml(DISTROS_FILE)
@@ -625,14 +681,17 @@ def main():
exporters = load_exporters()
notifications = load_notifications()
- collectors, ids = render_collectors(categories, collectors, dict())
- deploy, ids = render_deploy(distros, categories, deploy, ids)
- exporters, ids = render_exporters(categories, exporters, ids)
- notifications, ids = render_notifications(categories, notifications, ids)
+ collectors, clean_collectors, ids = render_collectors(categories, collectors, dict())
+ deploy, clean_deploy, ids = render_deploy(distros, categories, deploy, ids)
+ exporters, clean_exporters, ids = render_exporters(categories, exporters, ids)
+ notifications, clean_notifications, ids = render_notifications(categories, notifications, ids)
integrations = collectors + deploy + exporters + notifications
render_integrations(categories, integrations)
+ clean_integrations = clean_collectors + clean_deploy + clean_exporters + clean_notifications
+ render_json(categories, clean_integrations)
+
if __name__ == '__main__':
sys.exit(main())