diff options
Diffstat (limited to 'compilerplugins/clang/pahole-all-classes.py')
| -rwxr-xr-x | compilerplugins/clang/pahole-all-classes.py | 156 |
1 files changed, 156 insertions, 0 deletions
diff --git a/compilerplugins/clang/pahole-all-classes.py b/compilerplugins/clang/pahole-all-classes.py new file mode 100755 index 000000000..6037287a8 --- /dev/null +++ b/compilerplugins/clang/pahole-all-classes.py @@ -0,0 +1,156 @@ +#!/usr/bin/python3 +# +# Find holes in structures, so that we can pack them and improve our memory density. +# +# In order to make this work, you need to +# (1) Be operating in a workspace where you have a __NON-DEBUG__ build of LibreOffice, but __WITH SYMBOLS__. +# (A debug build has different sizes for some things in the standard library.) +# (2) First run the unusedfields loplugin to generate a log file +# (3) Install the pahole stuff into your gdb, I used this one: +# https://github.com/PhilArmstrong/pahole-gdb +# (4) Run the script +# ./compilerplugins/clang/pahole-all-classes.py +# + +import _thread +import io +import os +import subprocess +import time +import re + +# search for all the class names in the file produced by the unusedfields loplugin +#a = subprocess.Popen("grep 'definition:' workdir/loplugin.unusedfields.log | sort -u", stdout=subprocess.PIPE, shell=True) +a = subprocess.Popen("cat n1", stdout=subprocess.PIPE, shell=True) + +classSet = set() +classSourceLocDict = dict() +locToClassDict = dict() +with a.stdout as txt: + for line in txt: + tokens = line.decode('utf8').strip().split("\t") + className = tokens[2].strip() + srcLoc = tokens[5].strip() + # ignore things like unions + if "anonymous" in className: continue + # ignore duplicates + if className in classSet: continue + classSet.add(className) + classSourceLocDict[className] = srcLoc + locToClassDict[srcLoc] = className +a.terminate() + +# Some of the pahole commands are going to fail, and I cannot read the error stream and the input stream +# together because python has no way of (easily) doing a non-blocking read. +# So I have to write the commands out using a background thread, and then read the entire resulting +# stream out below. +def write_pahole_commands(classes): + for className in classes: + stdin.write("echo " + className + " " + classSourceLocDict[className] + "\n") + stdin.write("pahole " + className + "\n") + stdin.flush() + stdin.write("echo all-done\n") + stdin.flush() + stdin.close() # only way to make it flush the last echo command + +# Use generator because lines often end up merged together in gdb's output, and we need +# to split them up, and that creates a mess in the parsing logic. +def read_generator(gdbOutput): + while True: + line = gdbOutput.readline(); + if line == "": return # end of file + line = line.decode('utf8').strip() + print("gdb: " + line) + for split in line.split("(gdb)"): + split = split.strip() + if len(split) == 0: continue + if "all-done" in split: return + yield split + +# build list of classes sorted by source location to increase the chances of +# processing stuff stored in the same DSO together +sortedLocs = sorted(locToClassDict.keys()) +classList = list() +for src in sortedLocs: + if "/inc/" in src or "include/" in src: + classList.append(locToClassDict[src]) + +with open("compilerplugins/clang/pahole.results", "wt") as f: + # Process 400 classes at a time, otherwise gdb's memory usage blows up and kills the machine + # This number is chosen to make gdb peak at around 8G. + while len(classList) > 0: + + currClassList = classList[0:500]; + classList = classList[500:] + + gdbProc = subprocess.Popen("gdb", stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True) + + stdin = io.TextIOWrapper(gdbProc.stdin, 'utf-8') + + # make gdb load all the debugging info + stdin.write("set confirm off\n") + # make gdb not wrap output and mess up my parsing + stdin.write("set width unlimited\n") + for filename in sorted(os.listdir('instdir/program')): + if filename.endswith(".so"): + stdin.write("add-symbol-file instdir/program/" + filename + "\n") + stdin.flush() + + + _thread.start_new_thread( write_pahole_commands, (currClassList,) ) + + firstLineRegex = re.compile(r"/\*\s+(\d+)\s+\*/ struct") # /* 16 */ struct Foo + fieldLineRegex = re.compile(r"/\*\s+(\d+)\s+(\d+)\s+\*/ ") # /* 12 8 */ class rtl::OUString aName + holeLineRegex = re.compile(r"/\* XXX (\d+) bit hole, try to pack \*/") + # sometimes pahole can't determine the size of a sub-struct, and then it returns bad data + bogusLineRegex = re.compile(r"/\*\s+\d+\s+0\s+\*/") + structLines = list() + foundHole = False + cumulativeHoleBits = 0 + alignedStructSize = 0 + foundBogusLine = False + # pahole doesn't report space at the end of the structure, so work it out myself + sizeOfStructWithoutPadding = 0 + for line in read_generator(gdbProc.stdout): + structLines.append(line) + firstLineMatch = firstLineRegex.match(line) + if firstLineMatch: + alignedStructSize = int(firstLineMatch.group(1)) + structLines.clear() + structLines.append(line) + holeLineMatch = holeLineRegex.match(line) + if holeLineMatch: + foundHole = True + cumulativeHoleBits += int(holeLineMatch.group(1)) + fieldLineMatch = fieldLineRegex.match(line) + if fieldLineMatch: + fieldPosInBytes = int(fieldLineMatch.group(1)) + fieldSizeInBytes = int(fieldLineMatch.group(2)) + sizeOfStructWithoutPadding = fieldPosInBytes + fieldSizeInBytes + if bogusLineRegex.match(line): + foundBogusLine = True + if line == "}": + # Ignore very large structs, packing those is not going to help much, and + # re-organising them can make them much less readable. + if foundHole and len(structLines) < 16 and alignedStructSize < 100 and not foundBogusLine: + # Verify that, after packing, and compiler alignment, the new structure will be actually smaller. + # Sometimes, we can save space, but the compiler will align the structure such that we don't + # actually save any space. + # TODO improve detection of the required alignment for a structure + holeAtEnd = alignedStructSize - sizeOfStructWithoutPadding + potentialSpace = (cumulativeHoleBits / 8) + holeAtEnd + if potentialSpace >= 8: + for line in structLines: + f.write(line + "\n") + if holeAtEnd > 0: + f.write("hole at end of struct: " + str(holeAtEnd) + "\n") + f.write("\n") + # reset state + structLines.clear() + foundHole = False + cumulativeHoleBits = 0 + structSize = 0 + foundBogusLine = False + actualStructSize = 0 + + gdbProc.terminate() |