From e55403ed71282d7bfd8b56df219de3c28a8af064 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Mon, 25 Nov 2024 15:45:37 +0100 Subject: Merging upstream version 2.0.3+dfsg: - does not include dygraphs anymore (Closes: #923993) - does not include pako anymore (Closes: #1042533) - does not include dashboard binaries anymore (Closes: #1045145) Signed-off-by: Daniel Baumann --- integrations/gen_docs_integrations.py | 192 +++++++++++++++++++++++++++------- 1 file changed, 154 insertions(+), 38 deletions(-) (limited to 'integrations/gen_docs_integrations.py') diff --git a/integrations/gen_docs_integrations.py b/integrations/gen_docs_integrations.py index 51a59ed48..2dcbc7a1d 100644 --- a/integrations/gen_docs_integrations.py +++ b/integrations/gen_docs_integrations.py @@ -1,7 +1,7 @@ import json +import re import shutil from pathlib import Path -import re # Dictionary responsible for making the symbolic links at the end of the script's run. symlink_dict = {} @@ -25,10 +25,14 @@ def cleanup(): for element in Path("integrations/cloud-notifications").glob('**/*/'): if "integrations" in str(element) and not "metadata.yaml" in str(element): shutil.rmtree(element) + for element in Path("integrations/logs").glob('**/*/'): + if "integrations" in str(element) and "metadata.yaml" not in str(element): + shutil.rmtree(element) for element in Path("integrations/cloud-authentication").glob('**/*/'): if "integrations" in str(element) and not "metadata.yaml" in str(element): shutil.rmtree(element) + def generate_category_from_name(category_fragment, category_array): """ Takes a category ID in splitted form ("." as delimiter) and the array of the categories, and returns the proper category name that Learn expects. @@ -46,7 +50,7 @@ def generate_category_from_name(category_fragment, category_array): try: # print("equals") # print(fragment, category_fragment[i+1]) - dummy_id = dummy_id + "." + category_fragment[i+1] + dummy_id = dummy_id + "." + category_fragment[i + 1] # print(dummy_id) except IndexError: return category_name.split("/", 1)[1] @@ -75,19 +79,23 @@ def add_custom_edit_url(markdown_string, meta_yaml_link, sidebar_label_string, m """ output = "" + path_to_md_file = "" if mode == 'default': path_to_md_file = f'{meta_yaml_link.replace("/metadata.yaml", "")}/integrations/{clean_string(sidebar_label_string)}' - elif mode == 'cloud-notifications': + elif mode == 'cloud-notification': path_to_md_file = meta_yaml_link.replace("metadata.yaml", f'integrations/{clean_string(sidebar_label_string)}') - elif mode == 'agent-notifications': + elif mode == 'agent-notification': path_to_md_file = meta_yaml_link.replace("metadata.yaml", "README") elif mode == 'cloud-authentication': path_to_md_file = meta_yaml_link.replace("metadata.yaml", f'integrations/{clean_string(sidebar_label_string)}') + elif mode == 'logs': + path_to_md_file = meta_yaml_link.replace("metadata.yaml", f'integrations/{clean_string(sidebar_label_string)}') + output = markdown_string.replace( " print("Exception in exporter md construction", e, integration['id']) # NOTIFICATIONS - elif mode == 'notification': + elif mode == 'agent-notification': try: # initiate the variables for the notification method meta_yaml = integration['edit_link'].replace("blob", "edit") @@ -238,7 +253,7 @@ learn_rel_path: "{learn_rel_path.replace("notifications", "Alerts & Notification message: "DO NOT EDIT THIS FILE DIRECTLY, IT IS GENERATED BY THE NOTIFICATION'S metadata.yaml FILE" endmeta--> -{create_overview(integration, integration['meta']['icon_filename'])}""" +{create_overview(integration, integration['meta']['icon_filename'], "overview")}""" if integration['setup']: md += f""" @@ -252,7 +267,67 @@ endmeta--> except Exception as e: print("Exception in notification md construction", e, integration['id']) - + + elif mode == 'cloud-notification': + try: + # initiate the variables for the notification method + meta_yaml = integration['edit_link'].replace("blob", "edit") + sidebar_label = integration['meta']['name'] + learn_rel_path = generate_category_from_name(integration['meta']['categories'][0].split("."), categories) + + # build the markdown string + md = \ + f""" + +{create_overview(integration, integration['meta']['icon_filename'], "")}""" + + if integration['setup']: + md += f""" +{integration['setup']} +""" + + if integration['troubleshooting']: + md += f""" +{integration['troubleshooting']} +""" + + except Exception as e: + print("Exception in notification md construction", e, integration['id']) + + elif mode == 'logs': + try: + # initiate the variables for the logs integration + meta_yaml = integration['edit_link'].replace("blob", "edit") + sidebar_label = integration['meta']['name'] + learn_rel_path = generate_category_from_name(integration['meta']['categories'][0].split("."), categories) + + # build the markdown string + md = \ + f""" + +{create_overview(integration, integration['meta']['icon_filename'])}""" + + if integration['setup']: + md += f""" +{integration['setup']} +""" + + except Exception as e: + print("Exception in logs md construction", e, integration['id']) + + # AUTHENTICATIONS elif mode == 'authentication': if True: @@ -339,27 +414,35 @@ def write_to_file(path, md, meta_yaml, sidebar_label, community, mode='default') except KeyError: # We don't need to print something here. pass - elif mode == 'notification': + elif mode == 'cloud-notification': - if "cloud-notifications" in path: - # for cloud notifications we generate them near their metadata.yaml - name = clean_string(integration['meta']['name']) + # for cloud notifications we generate them near their metadata.yaml + name = clean_string(integration['meta']['name']) - if not Path(f'{path}/integrations').exists(): - Path(f'{path}/integrations').mkdir() + if not Path(f'{path}/integrations').exists(): + Path(f'{path}/integrations').mkdir() - # proper_edit_name = meta_yaml.replace( - # "metadata.yaml", f'integrations/{clean_string(sidebar_label)}.md\"') + # proper_edit_name = meta_yaml.replace( + # "metadata.yaml", f'integrations/{clean_string(sidebar_label)}.md\"') - md = add_custom_edit_url(md, meta_yaml, sidebar_label, mode='cloud-notifications') + md = add_custom_edit_url(md, meta_yaml, sidebar_label, mode='cloud-notification') - finalpath = f'{path}/integrations/{name}.md' - else: - # add custom_edit_url as the md file, so we can have uniqueness in the ingest script - # afterwards the ingest will replace this metadata with meta_yaml - md = add_custom_edit_url(md, meta_yaml, sidebar_label, mode='agent-notifications') + finalpath = f'{path}/integrations/{name}.md' - finalpath = f'{path}/README.md' + try: + clean_and_write( + md, + Path(finalpath) + ) + except FileNotFoundError as e: + print("Exception in writing to file", e) + elif mode == 'agent-notification': + # add custom_edit_url as the md file, so we can have uniqueness in the ingest script + # afterwards the ingest will replace this metadata with meta_yaml + + md = add_custom_edit_url(md, meta_yaml, sidebar_label, mode='agent-notification') + + finalpath = f'{path}/README.md' try: clean_and_write( @@ -369,7 +452,29 @@ def write_to_file(path, md, meta_yaml, sidebar_label, community, mode='default') except FileNotFoundError as e: print("Exception in writing to file", e) + elif mode == 'logs': + + # for logs we generate them near their metadata.yaml + name = clean_string(integration['meta']['name']) + + if not Path(f'{path}/integrations').exists(): + Path(f'{path}/integrations').mkdir() + + # proper_edit_name = meta_yaml.replace( + # "metadata.yaml", f'integrations/{clean_string(sidebar_label)}.md\"') + + md = add_custom_edit_url(md, meta_yaml, sidebar_label, mode='logs') + + finalpath = f'{path}/integrations/{name}.md' + + try: + clean_and_write( + md, + Path(finalpath) + ) + except FileNotFoundError as e: + print("Exception in writing to file", e) elif mode == 'authentication': name = clean_string(integration['meta']['name']) @@ -383,7 +488,7 @@ def write_to_file(path, md, meta_yaml, sidebar_label, community, mode='default') md = add_custom_edit_url(md, meta_yaml, sidebar_label, mode='cloud-authentication') finalpath = f'{path}/integrations/{name}.md' - + try: clean_and_write( md, @@ -422,7 +527,6 @@ cleanup() categories, integrations = read_integrations_js('integrations/integrations.js') - # Iterate through every integration for integration in integrations: @@ -442,20 +546,32 @@ for integration in integrations: path = build_path(meta_yaml) write_to_file(path, md, meta_yaml, sidebar_label, community) - # kind of specific if clause, so we can avoid running excessive code in the go repo - elif integration['integration_type'] == "notification": + elif integration['integration_type'] == "agent_notification": + + meta_yaml, sidebar_label, learn_rel_path, md, community = build_readme_from_integration( + integration, mode='agent-notification') + path = build_path(meta_yaml) + write_to_file(path, md, meta_yaml, sidebar_label, community, mode='agent-notification') + + elif integration['integration_type'] == "cloud_notification": meta_yaml, sidebar_label, learn_rel_path, md, community = build_readme_from_integration( - integration, mode='notification') + integration, mode='cloud-notification') path = build_path(meta_yaml) - write_to_file(path, md, meta_yaml, sidebar_label, community, mode='notification') + write_to_file(path, md, meta_yaml, sidebar_label, community, mode='cloud-notification') + + elif integration['integration_type'] == "logs": + + meta_yaml, sidebar_label, learn_rel_path, md, community = build_readme_from_integration( + integration, mode='logs') + path = build_path(meta_yaml) + write_to_file(path, md, meta_yaml, sidebar_label, community, mode='logs') elif integration['integration_type'] == "authentication": meta_yaml, sidebar_label, learn_rel_path, md, community = build_readme_from_integration( integration, mode='authentication') path = build_path(meta_yaml) - write_to_file(path, md, meta_yaml, sidebar_label, community, mode='authentication') - + write_to_file(path, md, meta_yaml, sidebar_label, community, mode='authentication') make_symlinks(symlink_dict) -- cgit v1.2.3