Merging upstream version 2.0.3+dfsg:

- does not include dygraphs anymore (Closes: #923993)
- does not include pako anymore (Closes: #1042533)
- does not include dashboard binaries anymore (Closes: #1045145)

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 154 insertions, 38 deletions

diff --git a/integrations/gen_docs_integrations.py b/integrations/gen_docs_integrations.py
index 51a59ed48..2dcbc7a1d 100644
--- a/integrations/gen_docs_integrations.py
+++ b/integrations/gen_docs_integrations.py
@@ -1,7 +1,7 @@
 import json
+import re
 import shutil
 from pathlib import Path
-import re
 
 # Dictionary responsible for making the symbolic links at the end of the script's run.
 symlink_dict = {}
@@ -25,10 +25,14 @@ def cleanup():
     for element in Path("integrations/cloud-notifications").glob('**/*/'):
         if "integrations" in str(element) and not "metadata.yaml" in str(element):
             shutil.rmtree(element)
+    for element in Path("integrations/logs").glob('**/*/'):
+        if "integrations" in str(element) and "metadata.yaml" not in str(element):
+            shutil.rmtree(element)
     for element in Path("integrations/cloud-authentication").glob('**/*/'):
         if "integrations" in str(element) and not "metadata.yaml" in str(element):
             shutil.rmtree(element)
 
+
 def generate_category_from_name(category_fragment, category_array):
     """
     Takes a category ID in splitted form ("." as delimiter) and the array of the categories, and returns the proper category name that Learn expects.
@@ -46,7 +50,7 @@ def generate_category_from_name(category_fragment, category_array):
                 try:
                     # print("equals")
                     # print(fragment, category_fragment[i+1])
-                    dummy_id = dummy_id + "." + category_fragment[i+1]
+                    dummy_id = dummy_id + "." + category_fragment[i + 1]
                     # print(dummy_id)
                 except IndexError:
                     return category_name.split("/", 1)[1]
@@ -75,19 +79,23 @@ def add_custom_edit_url(markdown_string, meta_yaml_link, sidebar_label_string, m
     """
 
     output = ""
+    path_to_md_file = ""
 
     if mode == 'default':
         path_to_md_file = f'{meta_yaml_link.replace("/metadata.yaml", "")}/integrations/{clean_string(sidebar_label_string)}'
 
-    elif mode == 'cloud-notifications':
+    elif mode == 'cloud-notification':
         path_to_md_file = meta_yaml_link.replace("metadata.yaml", f'integrations/{clean_string(sidebar_label_string)}')
 
-    elif mode == 'agent-notifications':
+    elif mode == 'agent-notification':
         path_to_md_file = meta_yaml_link.replace("metadata.yaml", "README")
 
     elif mode == 'cloud-authentication':
         path_to_md_file = meta_yaml_link.replace("metadata.yaml", f'integrations/{clean_string(sidebar_label_string)}')
 
+    elif mode == 'logs':
+        path_to_md_file = meta_yaml_link.replace("metadata.yaml", f'integrations/{clean_string(sidebar_label_string)}')
+
     output = markdown_string.replace(
         "<!--startmeta",
         f'<!--startmeta\ncustom_edit_url: \"{path_to_md_file}.md\"')
@@ -122,23 +130,29 @@ def read_integrations_js(path_to_file):
         print("Exception", e)
 
 
-def create_overview(integration, filename):
+def create_overview(integration, filename, overview_key_name="overview"):
+    # empty overview_key_name to have only image on overview
+    if not overview_key_name:
+        return f"""# {integration['meta']['name']}
 
-    split = re.split(r'(#.*\n)', integration['overview'], 1)
+<img src="https://netdata.cloud/img/{filename}" width="150"/>
+"""
+
+    split = re.split(r'(#.*\n)', integration[overview_key_name], 1)
 
     first_overview_part = split[1]
     rest_overview_part = split[2]
 
-    if len(filename) > 0:
-        return f"""{first_overview_part}
+    if not filename:
+        return f"""{first_overview_part}{rest_overview_part}
+"""
+
+    return f"""{first_overview_part}
 
 <img src="https://netdata.cloud/img/{filename}" width="150"/>
 
 {rest_overview_part}
 """
-    else:
-        return f"""{first_overview_part}{rest_overview_part}
-"""
 
 
 def build_readme_from_integration(integration, mode=''):
@@ -150,7 +164,8 @@ def build_readme_from_integration(integration, mode=''):
             meta_yaml = integration['edit_link'].replace("blob", "edit")
             sidebar_label = integration['meta']['monitored_instance']['name']
             learn_rel_path = generate_category_from_name(
-                integration['meta']['monitored_instance']['categories'][0].split("."), categories).replace("Data Collection", "Collecting Metrics")
+                integration['meta']['monitored_instance']['categories'][0].split("."), categories).replace(
+                "Data Collection", "Collecting Metrics")
             most_popular = integration['meta']['most_popular']
 
             # build the markdown string
@@ -221,7 +236,7 @@ endmeta-->
             print("Exception in exporter md construction", e, integration['id'])
 
     # NOTIFICATIONS
-    elif mode == 'notification':
+    elif mode == 'agent-notification':
         try:
             # initiate the variables for the notification method
             meta_yaml = integration['edit_link'].replace("blob", "edit")
@@ -238,7 +253,7 @@ learn_rel_path: "{learn_rel_path.replace("notifications", "Alerts & Notification
 message: "DO NOT EDIT THIS FILE DIRECTLY, IT IS GENERATED BY THE NOTIFICATION'S metadata.yaml FILE"
 endmeta-->
 
-{create_overview(integration, integration['meta']['icon_filename'])}"""
+{create_overview(integration, integration['meta']['icon_filename'], "overview")}"""
 
             if integration['setup']:
                 md += f"""
@@ -252,7 +267,67 @@ endmeta-->
 
         except Exception as e:
             print("Exception in notification md construction", e, integration['id'])
-    
+
+    elif mode == 'cloud-notification':
+        try:
+            # initiate the variables for the notification method
+            meta_yaml = integration['edit_link'].replace("blob", "edit")
+            sidebar_label = integration['meta']['name']
+            learn_rel_path = generate_category_from_name(integration['meta']['categories'][0].split("."), categories)
+
+            # build the markdown string
+            md = \
+                f"""<!--startmeta
+meta_yaml: "{meta_yaml}"
+sidebar_label: "{sidebar_label}"
+learn_status: "Published"
+learn_rel_path: "{learn_rel_path.replace("notifications", "Alerts & Notifications/Notifications")}"
+message: "DO NOT EDIT THIS FILE DIRECTLY, IT IS GENERATED BY THE NOTIFICATION'S metadata.yaml FILE"
+endmeta-->
+
+{create_overview(integration, integration['meta']['icon_filename'], "")}"""
+
+            if integration['setup']:
+                md += f"""
+{integration['setup']}
+"""
+
+            if integration['troubleshooting']:
+                md += f"""
+{integration['troubleshooting']}
+"""
+
+        except Exception as e:
+            print("Exception in notification md construction", e, integration['id'])
+
+    elif mode == 'logs':
+        try:
+            # initiate the variables for the logs integration
+            meta_yaml = integration['edit_link'].replace("blob", "edit")
+            sidebar_label = integration['meta']['name']
+            learn_rel_path = generate_category_from_name(integration['meta']['categories'][0].split("."), categories)
+
+            # build the markdown string
+            md = \
+                f"""<!--startmeta
+meta_yaml: "{meta_yaml}"
+sidebar_label: "{sidebar_label}"
+learn_status: "Published"
+learn_rel_path: "{learn_rel_path.replace("logs", "Logs")}"
+message: "DO NOT EDIT THIS FILE DIRECTLY, IT IS GENERATED BY THE LOGS' metadata.yaml FILE"
+endmeta-->
+
+{create_overview(integration, integration['meta']['icon_filename'])}"""
+
+            if integration['setup']:
+                md += f"""
+{integration['setup']}
+"""
+
+        except Exception as e:
+            print("Exception in logs md construction", e, integration['id'])
+
+
     # AUTHENTICATIONS
     elif mode == 'authentication':
         if True:
@@ -339,27 +414,35 @@ def write_to_file(path, md, meta_yaml, sidebar_label, community, mode='default')
                 except KeyError:
                     # We don't need to print something here.
                     pass
-    elif mode == 'notification':
+    elif mode == 'cloud-notification':
 
-        if "cloud-notifications" in path:
-            # for cloud notifications we generate them near their metadata.yaml
-            name = clean_string(integration['meta']['name'])
+        # for cloud notifications we generate them near their metadata.yaml
+        name = clean_string(integration['meta']['name'])
 
-            if not Path(f'{path}/integrations').exists():
-                Path(f'{path}/integrations').mkdir()
+        if not Path(f'{path}/integrations').exists():
+            Path(f'{path}/integrations').mkdir()
 
-            # proper_edit_name = meta_yaml.replace(
-            #     "metadata.yaml", f'integrations/{clean_string(sidebar_label)}.md\"')
+        # proper_edit_name = meta_yaml.replace(
+        #     "metadata.yaml", f'integrations/{clean_string(sidebar_label)}.md\"')
 
-            md = add_custom_edit_url(md, meta_yaml, sidebar_label, mode='cloud-notifications')
+        md = add_custom_edit_url(md, meta_yaml, sidebar_label, mode='cloud-notification')
 
-            finalpath = f'{path}/integrations/{name}.md'
-        else:
-            # add custom_edit_url as the md file, so we can have uniqueness in the ingest script
-            # afterwards the ingest will replace this metadata with meta_yaml
-            md = add_custom_edit_url(md, meta_yaml, sidebar_label, mode='agent-notifications')
+        finalpath = f'{path}/integrations/{name}.md'
 
-            finalpath = f'{path}/README.md'
+        try:
+            clean_and_write(
+                md,
+                Path(finalpath)
+            )
+        except FileNotFoundError as e:
+            print("Exception in writing to file", e)
+    elif mode == 'agent-notification':
+        # add custom_edit_url as the md file, so we can have uniqueness in the ingest script
+        # afterwards the ingest will replace this metadata with meta_yaml
+
+        md = add_custom_edit_url(md, meta_yaml, sidebar_label, mode='agent-notification')
+
+        finalpath = f'{path}/README.md'
 
         try:
             clean_and_write(
@@ -369,7 +452,29 @@ def write_to_file(path, md, meta_yaml, sidebar_label, community, mode='default')
 
         except FileNotFoundError as e:
             print("Exception in writing to file", e)
+    elif mode == 'logs':
+
+        # for logs we generate them near their metadata.yaml
 
+        name = clean_string(integration['meta']['name'])
+
+        if not Path(f'{path}/integrations').exists():
+            Path(f'{path}/integrations').mkdir()
+
+        # proper_edit_name = meta_yaml.replace(
+        #     "metadata.yaml", f'integrations/{clean_string(sidebar_label)}.md\"')
+
+        md = add_custom_edit_url(md, meta_yaml, sidebar_label, mode='logs')
+
+        finalpath = f'{path}/integrations/{name}.md'
+
+        try:
+            clean_and_write(
+                md,
+                Path(finalpath)
+            )
+        except FileNotFoundError as e:
+            print("Exception in writing to file", e)
     elif mode == 'authentication':
 
         name = clean_string(integration['meta']['name'])
@@ -383,7 +488,7 @@ def write_to_file(path, md, meta_yaml, sidebar_label, community, mode='default')
         md = add_custom_edit_url(md, meta_yaml, sidebar_label, mode='cloud-authentication')
 
         finalpath = f'{path}/integrations/{name}.md'
-        
+
         try:
             clean_and_write(
                 md,
@@ -422,7 +527,6 @@ cleanup()
 
 categories, integrations = read_integrations_js('integrations/integrations.js')
 
-
 # Iterate through every integration
 for integration in integrations:
 
@@ -442,20 +546,32 @@ for integration in integrations:
             path = build_path(meta_yaml)
             write_to_file(path, md, meta_yaml, sidebar_label, community)
 
-        # kind of specific if clause, so we can avoid running excessive code in the go repo
-        elif integration['integration_type'] == "notification":
+        elif integration['integration_type'] == "agent_notification":
+
+            meta_yaml, sidebar_label, learn_rel_path, md, community = build_readme_from_integration(
+                integration, mode='agent-notification')
+            path = build_path(meta_yaml)
+            write_to_file(path, md, meta_yaml, sidebar_label, community, mode='agent-notification')
+
+        elif integration['integration_type'] == "cloud_notification":
 
             meta_yaml, sidebar_label, learn_rel_path, md, community = build_readme_from_integration(
-                integration, mode='notification')
+                integration, mode='cloud-notification')
             path = build_path(meta_yaml)
-            write_to_file(path, md, meta_yaml, sidebar_label, community,  mode='notification')
+            write_to_file(path, md, meta_yaml, sidebar_label, community, mode='cloud-notification')
+
+        elif integration['integration_type'] == "logs":
+
+            meta_yaml, sidebar_label, learn_rel_path, md, community = build_readme_from_integration(
+                integration, mode='logs')
+            path = build_path(meta_yaml)
+            write_to_file(path, md, meta_yaml, sidebar_label, community, mode='logs')
 
         elif integration['integration_type'] == "authentication":
 
             meta_yaml, sidebar_label, learn_rel_path, md, community = build_readme_from_integration(
                 integration, mode='authentication')
             path = build_path(meta_yaml)
-            write_to_file(path, md, meta_yaml, sidebar_label, community,  mode='authentication')
-
+            write_to_file(path, md, meta_yaml, sidebar_label, community, mode='authentication')
 
 make_symlinks(symlink_dict)