import json
import shutil
from pathlib import Path
import re
# Dictionary responsible for making the symbolic links at the end of the script's run.
symlink_dict = {}
am_i_inside_go = "go.d.plugin" in str(Path.cwd())
def cleanup():
"""
clean directories that are either data collection or exporting integrations
"""
if am_i_inside_go:
for element in Path("modules").glob('**/*/'):
if "integrations" in str(element):
shutil.rmtree(element)
else:
for element in Path("collectors").glob('**/*/'):
if "integrations" in str(element):
shutil.rmtree(element)
for element in Path("exporting").glob('**/*/'):
if "integrations" in str(element):
shutil.rmtree(element)
for element in Path("integrations/cloud-notifications").glob('**/*/'):
if "integrations" in str(element) and not "metadata.yaml" in str(element):
shutil.rmtree(element)
def generate_category_from_name(category_fragment, category_array):
"""
Takes a category ID in splitted form ("." as delimiter) and the array of the categories, and returns the proper category name that Learn expects.
"""
category_name = ""
i = 0
dummy_id = category_fragment[0]
while i < len(category_fragment):
for category in category_array:
if dummy_id == category['id']:
category_name = category_name + "/" + category["name"]
try:
# print("equals")
# print(fragment, category_fragment[i+1])
dummy_id = dummy_id + "." + category_fragment[i+1]
# print(dummy_id)
except IndexError:
return category_name.split("/", 1)[1]
category_array = category['children']
break
i += 1
def clean_and_write(md, path):
"""
This function takes care of the special details element, and converts it to the equivalent that md expects.
Then it writes the buffer on the file provided.
"""
# clean first, replace
md = md.replace("{% details summary=\"", "").replace(
"\" %}", "
\n").replace("{% /details %}", " \n")
path.write_text(md)
def add_custom_edit_url(markdown_string, meta_yaml_link, sidebar_label_string, mode='default'):
"""
Takes a markdown string and adds a "custom_edit_url" metadata to the metadata field
"""
output = ""
if mode == 'default':
path_to_md_file = f'{meta_yaml_link.replace("/metadata.yaml", "")}/integrations/{clean_string(sidebar_label_string)}'
elif mode == 'cloud-notifications':
path_to_md_file = meta_yaml_link.replace("metadata.yaml", f'integrations/{clean_string(sidebar_label_string)}')
elif mode == 'agent-notifications':
path_to_md_file = meta_yaml_link.replace("metadata.yaml", "README")
output = markdown_string.replace(
"
{create_overview(integration, integration['meta']['monitored_instance']['icon_filename'])}"""
if integration['metrics']:
md += f"""
{integration['metrics']}
"""
if integration['alerts']:
md += f"""
{integration['alerts']}
"""
if integration['setup']:
md += f"""
{integration['setup']}
"""
if integration['troubleshooting']:
md += f"""
{integration['troubleshooting']}
"""
except Exception as e:
print("Exception in collector md construction", e, integration['id'])
# EXPORTERS
elif mode == 'exporter':
try:
# initiate the variables for the exporter
meta_yaml = integration['edit_link'].replace("blob", "edit")
sidebar_label = integration['meta']['name']
learn_rel_path = generate_category_from_name(integration['meta']['categories'][0].split("."), categories)
# build the markdown string
md = \
f"""
{create_overview(integration, integration['meta']['icon_filename'])}"""
if integration['setup']:
md += f"""
{integration['setup']}
"""
if integration['troubleshooting']:
md += f"""
{integration['troubleshooting']}
"""
except Exception as e:
print("Exception in exporter md construction", e, integration['id'])
# NOTIFICATIONS
elif mode == 'notification':
try:
# initiate the variables for the notification method
meta_yaml = integration['edit_link'].replace("blob", "edit")
sidebar_label = integration['meta']['name']
learn_rel_path = generate_category_from_name(integration['meta']['categories'][0].split("."), categories)
# build the markdown string
md = \
f"""
{create_overview(integration, integration['meta']['icon_filename'])}"""
if integration['setup']:
md += f"""
{integration['setup']}
"""
if integration['troubleshooting']:
md += f"""
{integration['troubleshooting']}
"""
except Exception as e:
print("Exception in notification md construction", e, integration['id'])
if "community" in integration['meta'].keys():
community = ""
else:
community = ""
return meta_yaml, sidebar_label, learn_rel_path, md, community
def build_path(meta_yaml_link):
"""
funtion that takes a metadata yaml file link, and makes it into a path that gets used to write to a file.
"""
return meta_yaml_link.replace("https://github.com/netdata/", "") \
.split("/", 1)[1] \
.replace("edit/master/", "") \
.replace("/metadata.yaml", "")
def write_to_file(path, md, meta_yaml, sidebar_label, community, mode='default'):
"""
takes the arguments needed to write the integration markdown to the proper file.
"""
upper, lower = md.split("##", 1)
md = upper + community + f"\n\n##{lower}"
if mode == 'default':
# Only if the path exists, this caters for running the same script on both the go and netdata repos.
if Path(path).exists():
if not Path(f'{path}/integrations').exists():
Path(f'{path}/integrations').mkdir()
try:
md = add_custom_edit_url(md, meta_yaml, sidebar_label)
clean_and_write(
md,
Path(f'{path}/integrations/{clean_string(sidebar_label)}.md')
)
except FileNotFoundError as e:
print("Exception in writing to file", e)
# If we only created one file inside the directory, add the entry to the symlink_dict, so we can make the symbolic link
if len(list(Path(f'{path}/integrations').iterdir())) == 1:
symlink_dict.update(
{path: f'integrations/{clean_string(sidebar_label)}.md'})
else:
try:
symlink_dict.pop(path)
except KeyError:
# We don't need to print something here.
pass
elif mode == 'notification':
if "cloud-notifications" in path:
# for cloud notifications we generate them near their metadata.yaml
name = clean_string(integration['meta']['name'])
if not Path(f'{path}/integrations').exists():
Path(f'{path}/integrations').mkdir()
# proper_edit_name = meta_yaml.replace(
# "metadata.yaml", f'integrations/{clean_string(sidebar_label)}.md\"')
md = add_custom_edit_url(md, meta_yaml, sidebar_label, mode='cloud-notifications')
finalpath = f'{path}/integrations/{name}.md'
else:
# add custom_edit_url as the md file, so we can have uniqueness in the ingest script
# afterwards the ingest will replace this metadata with meta_yaml
md = add_custom_edit_url(md, meta_yaml, sidebar_label, mode='agent-notifications')
finalpath = f'{path}/README.md'
try:
clean_and_write(
md,
Path(finalpath)
)
except FileNotFoundError as e:
print("Exception in writing to file", e)
def make_symlinks(symlink_dict):
"""
takes a dictionary with directories that have a 1:1 relationship between their README and the integration (only one) inside the "integrations" folder.
"""
for element in symlink_dict:
# Remove the README to prevent it being a normal file
Path(f'{element}/README.md').unlink()
# and then make a symlink to the actual markdown
Path(f'{element}/README.md').symlink_to(symlink_dict[element])
filepath = Path(f'{element}/{symlink_dict[element]}')
md = filepath.read_text()
# This preserves the custom_edit_url for most files as it was,
# so the existing links don't break, this is vital for link replacement afterwards
filepath.write_text(md.replace(
f'{element}/{symlink_dict[element]}', f'{element}/README.md'))
cleanup()
categories, integrations = read_integrations_js('integrations/integrations.js')
# Iterate through every integration
for integration in integrations:
if integration['integration_type'] == "collector":
meta_yaml, sidebar_label, learn_rel_path, md, community = build_readme_from_integration(
integration, mode='collector')
path = build_path(meta_yaml)
write_to_file(path, md, meta_yaml, sidebar_label, community)
elif not am_i_inside_go:
# kind of specific if clause, so we can avoid running excessive code in the go repo
if integration['integration_type'] == "exporter":
meta_yaml, sidebar_label, learn_rel_path, md, community = build_readme_from_integration(
integration, mode='exporter')
path = build_path(meta_yaml)
write_to_file(path, md, meta_yaml, sidebar_label, community)
# kind of specific if clause, so we can avoid running excessive code in the go repo
elif integration['integration_type'] == "notification":
meta_yaml, sidebar_label, learn_rel_path, md, community = build_readme_from_integration(
integration, mode='notification')
path = build_path(meta_yaml)
write_to_file(path, md, meta_yaml, sidebar_label, community, mode='notification')
make_symlinks(symlink_dict)