Merging upstream version 1.46.3.

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 118 insertions, 17 deletions

diff --git a/integrations/gen_integrations.py b/integrations/gen_integrations.py
index 043c22a77..9510fdc7f 100755
--- a/integrations/gen_integrations.py
+++ b/integrations/gen_integrations.py
@@ -14,7 +14,6 @@ from referencing.jsonschema import DRAFT7
 from ruamel.yaml import YAML, YAMLError
 
 AGENT_REPO = 'netdata/netdata'
-GO_REPO = 'netdata/go.d.plugin'
 
 INTEGRATIONS_PATH = Path(__file__).parent
 TEMPLATE_PATH = INTEGRATIONS_PATH / 'templates'
@@ -23,15 +22,14 @@ JSON_PATH = INTEGRATIONS_PATH / 'integrations.json'
 CATEGORIES_FILE = INTEGRATIONS_PATH / 'categories.yaml'
 REPO_PATH = INTEGRATIONS_PATH.parent
 SCHEMA_PATH = INTEGRATIONS_PATH / 'schemas'
-GO_REPO_PATH = REPO_PATH / 'go.d.plugin'
 DISTROS_FILE = REPO_PATH / '.github' / 'data' / 'distros.yml'
 METADATA_PATTERN = '*/metadata.yaml'
 
 COLLECTOR_SOURCES = [
-    (AGENT_REPO, REPO_PATH / 'collectors', True),
-    (AGENT_REPO, REPO_PATH / 'collectors' / 'charts.d.plugin', True),
-    (AGENT_REPO, REPO_PATH / 'collectors' / 'python.d.plugin', True),
-    (GO_REPO, GO_REPO_PATH / 'modules', True),
+    (AGENT_REPO, REPO_PATH / 'src' / 'collectors', True),
+    (AGENT_REPO, REPO_PATH / 'src' / 'collectors' / 'charts.d.plugin', True),
+    (AGENT_REPO, REPO_PATH / 'src' / 'collectors' / 'python.d.plugin', True),
+    (AGENT_REPO, REPO_PATH / 'src' / 'go' / 'collectors' / 'go.d.plugin' / 'modules', True),
 ]
 
 DEPLOY_SOURCES = [
@@ -39,14 +37,18 @@ DEPLOY_SOURCES = [
 ]
 
 EXPORTER_SOURCES = [
-    (AGENT_REPO, REPO_PATH / 'exporting', True),
+    (AGENT_REPO, REPO_PATH / 'src' / 'exporting', True),
 ]
 
 NOTIFICATION_SOURCES = [
-    (AGENT_REPO, REPO_PATH / 'health' / 'notifications', True),
+    (AGENT_REPO, REPO_PATH / 'src' / 'health' / 'notifications', True),
     (AGENT_REPO, INTEGRATIONS_PATH / 'cloud-notifications' / 'metadata.yaml', False),
 ]
 
+AUTHENTICATION_SOURCES = [
+    (AGENT_REPO, INTEGRATIONS_PATH / 'cloud-authentication' / 'metadata.yaml', False),
+]
+
 COLLECTOR_RENDER_KEYS = [
     'alerts',
     'metrics',
@@ -68,6 +70,12 @@ NOTIFICATION_RENDER_KEYS = [
     'troubleshooting',
 ]
 
+AUTHENTICATION_RENDER_KEYS = [
+    'overview',
+    'setup',
+    'troubleshooting',
+]
+
 CUSTOM_TAG_PATTERN = re.compile('\\{% if .*?%\\}.*?\\{% /if %\\}|\\{%.*?%\\}', flags=re.DOTALL)
 FIXUP_BLANK_PATTERN = re.compile('\\\\\\n *\\n')
 
@@ -119,6 +127,11 @@ NOTIFICATION_VALIDATOR = Draft7Validator(
     registry=registry,
 )
 
+AUTHENTICATION_VALIDATOR = Draft7Validator(
+    {'$ref': './authentication.json#'},
+    registry=registry,
+)
+
 COLLECTOR_VALIDATOR = Draft7Validator(
     {'$ref': './collector.json#'},
     registry=registry,
@@ -386,6 +399,51 @@ def load_notifications():
 
     return ret
 
+def _load_authentication_file(file, repo):
+    debug(f'Loading { file }.')
+    data = load_yaml(file)
+
+    if not data:
+        return []
+
+    try:
+        AUTHENTICATION_VALIDATOR.validate(data)
+    except ValidationError:
+        warn(f'Failed to validate { file } against the schema.', file)
+        return []
+
+    if 'id' in data:
+        data['integration_type'] = 'authentication'
+        data['_src_path'] = file
+        data['_repo'] = repo
+        data['_index'] = 0
+
+        return [data]
+    else:
+        ret = []
+
+        for idx, item in enumerate(data):
+            item['integration_type'] = 'authentication'
+            item['_src_path'] = file
+            item['_repo'] = repo
+            item['_index'] = idx
+            ret.append(item)
+
+        return ret
+
+
+def load_authentications():
+    ret = []
+
+    for repo, path, match in AUTHENTICATION_SOURCES:
+        if match and path.exists() and path.is_dir():
+            for file in path.glob(METADATA_PATTERN):
+                ret.extend(_load_authentication_file(file, repo))
+        elif not match and path.exists() and path.is_file():
+            ret.extend(_load_authentication_file(path, repo))
+
+    return ret
+
 
 def make_id(meta):
     if 'monitored_instance' in meta:
@@ -399,10 +457,7 @@ def make_id(meta):
 
 
 def make_edit_link(item):
-    if item['_repo'] == 'netdata/go.d.plugin':
-        item_path = item['_src_path'].relative_to(GO_REPO_PATH)
-    else:
-        item_path = item['_src_path'].relative_to(REPO_PATH)
+    item_path = item['_src_path'].relative_to(REPO_PATH)
 
     return f'https://github.com/{ item["_repo"] }/blob/master/{ item_path }'
 
@@ -657,11 +712,54 @@ def render_notifications(categories, notifications, ids):
     return notifications, clean_notifications, ids
 
 
+def render_authentications(categories, authentications, ids):
+    debug('Sorting authentications.')
+
+    sort_integrations(authentications)
+
+    debug('Checking authentication ids.')
+
+    authentications, ids = dedupe_integrations(authentications, ids)
+
+    clean_authentications = []
+
+    for item in authentications:
+        item['edit_link'] = make_edit_link(item)
+
+        clean_item = deepcopy(item)
+
+        for key in AUTHENTICATION_RENDER_KEYS:
+            
+            if key in item.keys():
+                template = get_jinja_env().get_template(f'{ key }.md')
+                data = template.render(entry=item, clean=False)
+                clean_data = template.render(entry=item, clean=True)
+
+                if 'variables' in item['meta']:
+                    template = get_jinja_env().from_string(data)
+                    data = template.render(variables=item['meta']['variables'], clean=False)
+                    template = get_jinja_env().from_string(clean_data)
+                    clean_data = template.render(variables=item['meta']['variables'], clean=True)
+            else:
+                data = ''
+                clean_data = ''
+
+            item[key] = data
+            clean_item[key] = clean_data
+            
+        for k in ['_src_path', '_repo', '_index']:
+            del item[k], clean_item[k]
+
+        clean_authentications.append(clean_item)
+
+    return authentications, clean_authentications, ids
+
+
 def render_integrations(categories, integrations):
     template = get_jinja_env().get_template('integrations.js')
     data = template.render(
-        categories=json.dumps(categories),
-        integrations=json.dumps(integrations),
+        categories=json.dumps(categories, indent=4),
+        integrations=json.dumps(integrations, indent=4),
     )
     OUTPUT_PATH.write_text(data)
 
@@ -670,7 +768,7 @@ def render_json(categories, integrations):
     JSON_PATH.write_text(json.dumps({
         'categories': categories,
         'integrations': integrations,
-    }))
+    }, indent=4))
 
 
 def main():
@@ -680,16 +778,19 @@ def main():
     deploy = load_deploy()
     exporters = load_exporters()
     notifications = load_notifications()
+    authentications = load_authentications()
 
     collectors, clean_collectors, ids = render_collectors(categories, collectors, dict())
     deploy, clean_deploy, ids = render_deploy(distros, categories, deploy, ids)
     exporters, clean_exporters, ids = render_exporters(categories, exporters, ids)
     notifications, clean_notifications, ids = render_notifications(categories, notifications, ids)
+    authentications, clean_authentications, ids = render_authentications(categories, authentications, ids)
+
 
-    integrations = collectors + deploy + exporters + notifications
+    integrations = collectors + deploy + exporters + notifications + authentications
     render_integrations(categories, integrations)
 
-    clean_integrations = clean_collectors + clean_deploy + clean_exporters + clean_notifications
+    clean_integrations = clean_collectors + clean_deploy + clean_exporters + clean_notifications + clean_authentications
     render_json(categories, clean_integrations)