summaryrefslogtreecommitdiffstats
path: root/compilerplugins/clang/pahole-all-classes.py
diff options
context:
space:
mode:
Diffstat (limited to 'compilerplugins/clang/pahole-all-classes.py')
-rwxr-xr-xcompilerplugins/clang/pahole-all-classes.py156
1 files changed, 156 insertions, 0 deletions
diff --git a/compilerplugins/clang/pahole-all-classes.py b/compilerplugins/clang/pahole-all-classes.py
new file mode 100755
index 000000000..6037287a8
--- /dev/null
+++ b/compilerplugins/clang/pahole-all-classes.py
@@ -0,0 +1,156 @@
+#!/usr/bin/python3
+#
+# Find holes in structures, so that we can pack them and improve our memory density.
+#
+# In order to make this work, you need to
+# (1) Be operating in a workspace where you have a __NON-DEBUG__ build of LibreOffice, but __WITH SYMBOLS__.
+# (A debug build has different sizes for some things in the standard library.)
+# (2) First run the unusedfields loplugin to generate a log file
+# (3) Install the pahole stuff into your gdb, I used this one:
+# https://github.com/PhilArmstrong/pahole-gdb
+# (4) Run the script
+# ./compilerplugins/clang/pahole-all-classes.py
+#
+
+import _thread
+import io
+import os
+import subprocess
+import time
+import re
+
+# search for all the class names in the file produced by the unusedfields loplugin
+#a = subprocess.Popen("grep 'definition:' workdir/loplugin.unusedfields.log | sort -u", stdout=subprocess.PIPE, shell=True)
+a = subprocess.Popen("cat n1", stdout=subprocess.PIPE, shell=True)
+
+classSet = set()
+classSourceLocDict = dict()
+locToClassDict = dict()
+with a.stdout as txt:
+ for line in txt:
+ tokens = line.decode('utf8').strip().split("\t")
+ className = tokens[2].strip()
+ srcLoc = tokens[5].strip()
+ # ignore things like unions
+ if "anonymous" in className: continue
+ # ignore duplicates
+ if className in classSet: continue
+ classSet.add(className)
+ classSourceLocDict[className] = srcLoc
+ locToClassDict[srcLoc] = className
+a.terminate()
+
+# Some of the pahole commands are going to fail, and I cannot read the error stream and the input stream
+# together because python has no way of (easily) doing a non-blocking read.
+# So I have to write the commands out using a background thread, and then read the entire resulting
+# stream out below.
+def write_pahole_commands(classes):
+ for className in classes:
+ stdin.write("echo " + className + " " + classSourceLocDict[className] + "\n")
+ stdin.write("pahole " + className + "\n")
+ stdin.flush()
+ stdin.write("echo all-done\n")
+ stdin.flush()
+ stdin.close() # only way to make it flush the last echo command
+
+# Use generator because lines often end up merged together in gdb's output, and we need
+# to split them up, and that creates a mess in the parsing logic.
+def read_generator(gdbOutput):
+ while True:
+ line = gdbOutput.readline();
+ if line == "": return # end of file
+ line = line.decode('utf8').strip()
+ print("gdb: " + line)
+ for split in line.split("(gdb)"):
+ split = split.strip()
+ if len(split) == 0: continue
+ if "all-done" in split: return
+ yield split
+
+# build list of classes sorted by source location to increase the chances of
+# processing stuff stored in the same DSO together
+sortedLocs = sorted(locToClassDict.keys())
+classList = list()
+for src in sortedLocs:
+ if "/inc/" in src or "include/" in src:
+ classList.append(locToClassDict[src])
+
+with open("compilerplugins/clang/pahole.results", "wt") as f:
+ # Process 400 classes at a time, otherwise gdb's memory usage blows up and kills the machine
+ # This number is chosen to make gdb peak at around 8G.
+ while len(classList) > 0:
+
+ currClassList = classList[0:500];
+ classList = classList[500:]
+
+ gdbProc = subprocess.Popen("gdb", stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True)
+
+ stdin = io.TextIOWrapper(gdbProc.stdin, 'utf-8')
+
+ # make gdb load all the debugging info
+ stdin.write("set confirm off\n")
+ # make gdb not wrap output and mess up my parsing
+ stdin.write("set width unlimited\n")
+ for filename in sorted(os.listdir('instdir/program')):
+ if filename.endswith(".so"):
+ stdin.write("add-symbol-file instdir/program/" + filename + "\n")
+ stdin.flush()
+
+
+ _thread.start_new_thread( write_pahole_commands, (currClassList,) )
+
+ firstLineRegex = re.compile(r"/\*\s+(\d+)\s+\*/ struct") # /* 16 */ struct Foo
+ fieldLineRegex = re.compile(r"/\*\s+(\d+)\s+(\d+)\s+\*/ ") # /* 12 8 */ class rtl::OUString aName
+ holeLineRegex = re.compile(r"/\* XXX (\d+) bit hole, try to pack \*/")
+ # sometimes pahole can't determine the size of a sub-struct, and then it returns bad data
+ bogusLineRegex = re.compile(r"/\*\s+\d+\s+0\s+\*/")
+ structLines = list()
+ foundHole = False
+ cumulativeHoleBits = 0
+ alignedStructSize = 0
+ foundBogusLine = False
+ # pahole doesn't report space at the end of the structure, so work it out myself
+ sizeOfStructWithoutPadding = 0
+ for line in read_generator(gdbProc.stdout):
+ structLines.append(line)
+ firstLineMatch = firstLineRegex.match(line)
+ if firstLineMatch:
+ alignedStructSize = int(firstLineMatch.group(1))
+ structLines.clear()
+ structLines.append(line)
+ holeLineMatch = holeLineRegex.match(line)
+ if holeLineMatch:
+ foundHole = True
+ cumulativeHoleBits += int(holeLineMatch.group(1))
+ fieldLineMatch = fieldLineRegex.match(line)
+ if fieldLineMatch:
+ fieldPosInBytes = int(fieldLineMatch.group(1))
+ fieldSizeInBytes = int(fieldLineMatch.group(2))
+ sizeOfStructWithoutPadding = fieldPosInBytes + fieldSizeInBytes
+ if bogusLineRegex.match(line):
+ foundBogusLine = True
+ if line == "}":
+ # Ignore very large structs, packing those is not going to help much, and
+ # re-organising them can make them much less readable.
+ if foundHole and len(structLines) < 16 and alignedStructSize < 100 and not foundBogusLine:
+ # Verify that, after packing, and compiler alignment, the new structure will be actually smaller.
+ # Sometimes, we can save space, but the compiler will align the structure such that we don't
+ # actually save any space.
+ # TODO improve detection of the required alignment for a structure
+ holeAtEnd = alignedStructSize - sizeOfStructWithoutPadding
+ potentialSpace = (cumulativeHoleBits / 8) + holeAtEnd
+ if potentialSpace >= 8:
+ for line in structLines:
+ f.write(line + "\n")
+ if holeAtEnd > 0:
+ f.write("hole at end of struct: " + str(holeAtEnd) + "\n")
+ f.write("\n")
+ # reset state
+ structLines.clear()
+ foundHole = False
+ cumulativeHoleBits = 0
+ structSize = 0
+ foundBogusLine = False
+ actualStructSize = 0
+
+ gdbProc.terminate()