aboutsummaryrefslogtreecommitdiff
path: root/scripts
diff options
context:
space:
mode:
Diffstat (limited to 'scripts')
-rw-r--r--scripts/README58
-rwxr-xr-xscripts/codegen/gen_linux_syscalls.pl37
-rwxr-xr-xscripts/codegen/gen_server_ciphers.py115
-rw-r--r--scripts/codegen/get_mozilla_ciphers.py210
-rwxr-xr-xscripts/maint/checkLogs.pl51
-rwxr-xr-xscripts/maint/checkOptionDocs.pl71
-rwxr-xr-xscripts/maint/checkSpace.pl138
-rwxr-xr-xscripts/maint/check_config_macros.pl20
-rwxr-xr-xscripts/maint/findMergedChanges.pl73
-rwxr-xr-xscripts/maint/format_changelog.py298
-rwxr-xr-xscripts/maint/redox.py228
-rwxr-xr-xscripts/maint/sortChanges.py40
-rwxr-xr-xscripts/maint/updateVersions.pl59
-rwxr-xr-xscripts/test/cov-blame48
-rwxr-xr-xscripts/test/cov-diff17
-rwxr-xr-xscripts/test/coverage46
-rw-r--r--scripts/test/scan-build.sh49
17 files changed, 1558 insertions, 0 deletions
diff --git a/scripts/README b/scripts/README
new file mode 100644
index 000000000..70c763923
--- /dev/null
+++ b/scripts/README
@@ -0,0 +1,58 @@
+The scripts directory holds tools for use in building, generating, testing,
+and maintaining the Tor source code. It is mainly for use by developers.
+
+Code maintenance scripts
+------------------------
+
+maint/checkLogs.pl -- Verify that Tor log statements are unique.
+
+maint/check_config_macros.pl -- Look for autoconf tests whose results are
+never used.
+
+maint/checkOptionDocs.pl -- Make sure that Tor options are documented in the
+manpage, and that the manpage only documents real Tor options.
+
+maint/checkSpaces.pl -- Style checker for the Tor source code. Mainly checks
+whitespace.
+
+maint/findMergedChanges.pl -- Find a set of changes/* files that have been
+merged into an upstream version.
+
+maint/format_changelog.py -- Flow the changelog into the proper format.
+
+maint/redox.py -- Find places that should have DOCDOC comments to indicate a
+need for doxygen comments, and put those comments there.
+
+maint/updateVersions.pl -- Update the version number in the .nsi and windows
+orconfig.h files.
+
+
+Testing scripts
+---------------
+
+test/cov-blame -- Mash up the results of gcov with git blame. Mainly useful
+to find out who has been writing untested code.
+
+test/cov-diff -- Compare two directories of gcov files to identify changed
+lines without coverage.
+
+test/coverage -- Generates a directory full of gcov files. You need to use
+this script instead of calling gcov directly because of our confusingly named
+object files.
+
+test/scan-build.sh -- Example script for invoking clang's scan-build
+static analysis tools.
+
+
+Code generation scripts
+-----------------------
+
+codegen/gen_linux_syscalls.pl -- Generate a table mapping linux syscall
+numbers to their names.
+
+codegen/gen_server_ciphers.py -- Generate a sorted list of TLS ciphersuites
+for servers to choose from.
+
+codegen/get_mozilla_ciphers.py -- Generate a list of TLS ciphersuites for
+clients to use in order to look like Firefox.
+
diff --git a/scripts/codegen/gen_linux_syscalls.pl b/scripts/codegen/gen_linux_syscalls.pl
new file mode 100755
index 000000000..f985bad6c
--- /dev/null
+++ b/scripts/codegen/gen_linux_syscalls.pl
@@ -0,0 +1,37 @@
+#!/usr/bin/perl -w
+
+use strict;
+my %syscalls = ();
+
+while (<>) {
+ if (/^#define (__NR_\w+) /) {
+ $syscalls{$1} = 1;
+ }
+}
+
+print <<EOL;
+/* Automatically generated with
+ gen_linux_syscalls.pl /usr/include/asm/unistd*.h
+ Do not edit.
+ */
+static const struct {
+ int syscall_num; const char *syscall_name;
+} SYSCALLS_BY_NUMBER[] = {
+EOL
+
+for my $k (sort keys %syscalls) {
+ my $name = $k;
+ $name =~ s/^__NR_//;
+ print <<EOL;
+#ifdef $k
+ { $k, "$name" },
+#endif
+EOL
+
+}
+
+print <<EOL
+ {0, NULL}
+};
+
+EOL
diff --git a/scripts/codegen/gen_server_ciphers.py b/scripts/codegen/gen_server_ciphers.py
new file mode 100755
index 000000000..97ed9d046
--- /dev/null
+++ b/scripts/codegen/gen_server_ciphers.py
@@ -0,0 +1,115 @@
+#!/usr/bin/python
+# Copyright 2014, The Tor Project, Inc
+# See LICENSE for licensing information
+
+# This script parses openssl headers to find ciphersuite names, determines
+# which ones we should be willing to use as a server, and sorts them according
+# to preference rules.
+#
+# Run it on all the files in your openssl include directory.
+
+import re
+import sys
+
+EPHEMERAL_INDICATORS = [ "_EDH_", "_DHE_", "_ECDHE_" ]
+BAD_STUFF = [ "_DES_40_", "MD5", "_RC4_", "_DES_64_",
+ "_SEED_", "_CAMELLIA_", "_NULL" ]
+
+# these never get #ifdeffed.
+MANDATORY = [
+ "TLS1_TXT_DHE_RSA_WITH_AES_256_SHA",
+ "TLS1_TXT_DHE_RSA_WITH_AES_128_SHA",
+ "SSL3_TXT_EDH_RSA_DES_192_CBC3_SHA",
+]
+
+def find_ciphers(filename):
+ with open(filename) as f:
+ for line in f:
+ m = re.search(r'(?:SSL3|TLS1)_TXT_\w+', line)
+ if m:
+ yield m.group(0)
+
+def usable_cipher(ciph):
+ ephemeral = False
+ for e in EPHEMERAL_INDICATORS:
+ if e in ciph:
+ ephemeral = True
+ if not ephemeral:
+ return False
+
+ if "_RSA_" not in ciph:
+ return False
+
+ for b in BAD_STUFF:
+ if b in ciph:
+ return False
+ return True
+
+# All fields we sort on, in order of priority.
+FIELDS = [ 'cipher', 'fwsec', 'mode', 'digest', 'bitlength' ]
+# Map from sorted fields to recognized value in descending order of goodness
+FIELD_VALS = { 'cipher' : [ 'AES', 'DES'],
+ 'fwsec' : [ 'ECDHE', 'DHE' ],
+ 'mode' : [ 'GCM', 'CBC' ],
+ 'digest' : [ 'SHA384', 'SHA256', 'SHA' ],
+ 'bitlength' : [ '256', '128', '192' ],
+}
+
+class Ciphersuite(object):
+ def __init__(self, name, fwsec, cipher, bitlength, mode, digest):
+ self.name = name
+ self.fwsec = fwsec
+ self.cipher = cipher
+ self.bitlength = bitlength
+ self.mode = mode
+ self.digest = digest
+
+ for f in FIELDS:
+ assert(getattr(self, f) in FIELD_VALS[f])
+
+ def sort_key(self):
+ return tuple(FIELD_VALS[f].index(getattr(self,f)) for f in FIELDS)
+
+
+def parse_cipher(ciph):
+ m = re.match('(?:TLS1|SSL3)_TXT_(EDH|DHE|ECDHE)_RSA(?:_WITH)?_(AES|DES)_(256|128|192)(|_CBC|_CBC3|_GCM)_(SHA|SHA256|SHA384)$', ciph)
+
+ if not m:
+ print "/* Couldn't parse %s ! */"%ciph
+ return None
+
+ fwsec, cipher, bits, mode, digest = m.groups()
+ if fwsec == 'EDH':
+ fwsec = 'DHE'
+
+ if mode in [ '_CBC3', '_CBC', '' ]:
+ mode = 'CBC'
+ elif mode == '_GCM':
+ mode = 'GCM'
+
+ return Ciphersuite(ciph, fwsec, cipher, bits, mode, digest)
+
+ALL_CIPHERS = []
+
+for fname in sys.argv[1:]:
+ ALL_CIPHERS += (parse_cipher(c)
+ for c in find_ciphers(fname)
+ if usable_cipher(c) )
+
+ALL_CIPHERS.sort(key=Ciphersuite.sort_key)
+
+for c in ALL_CIPHERS:
+ if c is ALL_CIPHERS[-1]:
+ colon = ';'
+ else:
+ colon = ' ":"'
+
+ if c.name in MANDATORY:
+ print " /* Required */"
+ print ' %s%s'%(c.name,colon)
+ else:
+ print "#ifdef %s"%c.name
+ print ' %s%s'%(c.name,colon)
+ print "#endif"
+
+
diff --git a/scripts/codegen/get_mozilla_ciphers.py b/scripts/codegen/get_mozilla_ciphers.py
new file mode 100644
index 000000000..0636eb365
--- /dev/null
+++ b/scripts/codegen/get_mozilla_ciphers.py
@@ -0,0 +1,210 @@
+#!/usr/bin/python
+# coding=utf-8
+# Copyright 2011, The Tor Project, Inc
+# original version by Arturo Filastò
+# See LICENSE for licensing information
+
+# This script parses Firefox and OpenSSL sources, and uses this information
+# to generate a ciphers.inc file.
+#
+# It takes two arguments: the location of a firefox source directory, and the
+# location of an openssl source directory.
+
+import os
+import re
+import sys
+
+if len(sys.argv) != 3:
+ print >>sys.stderr, "Syntax: get_mozilla_ciphers.py <firefox-source-dir> <openssl-source-dir>"
+ sys.exit(1)
+
+ff_root = sys.argv[1]
+ossl_root = sys.argv[2]
+
+def ff(s):
+ return os.path.join(ff_root, s)
+def ossl(s):
+ return os.path.join(ossl_root, s)
+
+#####
+# Read the cpp file to understand what Ciphers map to what name :
+# Make "ciphers" a map from name used in the javascript to a cipher macro name
+fileA = open(ff('security/manager/ssl/src/nsNSSComponent.cpp'),'r')
+
+# The input format is a file containing exactly one section of the form:
+# static CipherPref CipherPrefs[] = {
+# {"name", MACRO_NAME}, // comment
+# ...
+# {NULL, 0}
+# }
+
+inCipherSection = False
+cipherLines = []
+for line in fileA:
+ if line.startswith('static const CipherPref sCipherPrefs[]'):
+ # Get the starting boundary of the Cipher Preferences
+ inCipherSection = True
+ elif inCipherSection:
+ line = line.strip()
+ if line.startswith('{ nullptr, 0}'):
+ # At the ending boundary of the Cipher Prefs
+ break
+ else:
+ cipherLines.append(line)
+fileA.close()
+
+# Parse the lines and put them into a dict
+ciphers = {}
+cipher_pref = {}
+key_pending = None
+for line in cipherLines:
+ m = re.search(r'^{\s*\"([^\"]+)\",\s*(\S+)\s*(?:,\s*(true|false))?\s*}', line)
+ if m:
+ assert not key_pending
+ key,value,enabled = m.groups()
+ if enabled == 'true':
+ ciphers[key] = value
+ cipher_pref[value] = key
+ continue
+ m = re.search(r'^{\s*\"([^\"]+)\",', line)
+ if m:
+ assert not key_pending
+ key_pending = m.group(1)
+ continue
+ m = re.search(r'^\s*(\S+)(?:,\s*(true|false))?\s*}', line)
+ if m:
+ assert key_pending
+ key = key_pending
+ value,enabled = m.groups()
+ key_pending = None
+ if enabled == 'true':
+ ciphers[key] = value
+ cipher_pref[value] = key
+
+####
+# Now find the correct order for the ciphers
+fileC = open(ff('security/nss/lib/ssl/ssl3con.c'), 'r')
+firefox_ciphers = []
+inEnum=False
+for line in fileC:
+ if not inEnum:
+ if "ssl3CipherSuiteCfg cipherSuites[" in line:
+ inEnum = True
+ continue
+
+ if line.startswith("};"):
+ break
+
+ m = re.match(r'^\s*\{\s*([A-Z_0-9]+),', line)
+ if m:
+ firefox_ciphers.append(m.group(1))
+
+fileC.close()
+
+#####
+# Read the JS file to understand what ciphers are enabled. The format is
+# pref("name", true/false);
+# Build a map enabled_ciphers from javascript name to "true" or "false",
+# and an (unordered!) list of the macro names for those ciphers that are
+# enabled.
+fileB = open(ff('netwerk/base/public/security-prefs.js'), 'r')
+
+enabled_ciphers = {}
+for line in fileB:
+ m = re.match(r'pref\(\"([^\"]+)\"\s*,\s*(\S*)\s*\)', line)
+ if not m:
+ continue
+ key, val = m.groups()
+ if key.startswith("security.ssl3"):
+ enabled_ciphers[key] = val
+fileB.close()
+
+used_ciphers = []
+for k, v in enabled_ciphers.items():
+ if v == "true":
+ used_ciphers.append(ciphers[k])
+
+#oSSLinclude = ('/usr/include/openssl/ssl3.h', '/usr/include/openssl/ssl.h',
+# '/usr/include/openssl/ssl2.h', '/usr/include/openssl/ssl23.h',
+# '/usr/include/openssl/tls1.h')
+oSSLinclude = ('ssl/ssl3.h', 'ssl/ssl.h',
+ 'ssl/ssl2.h', 'ssl/ssl23.h',
+ 'ssl/tls1.h')
+
+#####
+# This reads the hex code for the ciphers that are used by firefox.
+# sslProtoD is set to a map from macro name to macro value in sslproto.h;
+# cipher_codes is set to an (unordered!) list of these hex values.
+sslProto = open(ff('security/nss/lib/ssl/sslproto.h'), 'r')
+sslProtoD = {}
+
+for line in sslProto:
+ m = re.match('#define\s+(\S+)\s+(\S+)', line)
+ if m:
+ key, value = m.groups()
+ sslProtoD[key] = value
+sslProto.close()
+
+cipher_codes = []
+for x in used_ciphers:
+ cipher_codes.append(sslProtoD[x].lower())
+
+####
+# Now read through all the openssl include files, and try to find the openssl
+# macro names for those files.
+openssl_macro_by_hex = {}
+all_openssl_macros = {}
+for fl in oSSLinclude:
+ fp = open(ossl(fl), 'r')
+ for line in fp.readlines():
+ m = re.match('#define\s+(\S+)\s+(\S+)', line)
+ if m:
+ value,key = m.groups()
+ if key.startswith('0x') and "_CK_" in value:
+ key = key.replace('0x0300','0x').lower()
+ #print "%s %s" % (key, value)
+ openssl_macro_by_hex[key] = value
+ all_openssl_macros[value]=key
+ fp.close()
+
+# Now generate the output.
+print """\
+/* This is an include file used to define the list of ciphers clients should
+ * advertise. Before including it, you should define the CIPHER and XCIPHER
+ * macros.
+ *
+ * This file was automatically generated by get_mozilla_ciphers.py.
+ */"""
+# Go in order by the order in CipherPrefs
+for firefox_macro in firefox_ciphers:
+
+ try:
+ js_cipher_name = cipher_pref[firefox_macro]
+ except KeyError:
+ # This one has no javascript preference.
+ continue
+
+ # The cipher needs to be enabled in security-prefs.js
+ if enabled_ciphers.get(js_cipher_name, 'false') != 'true':
+ continue
+
+ hexval = sslProtoD[firefox_macro].lower()
+
+ try:
+ openssl_macro = openssl_macro_by_hex[hexval.lower()]
+ openssl_macro = openssl_macro.replace("_CK_", "_TXT_")
+ if openssl_macro not in all_openssl_macros:
+ raise KeyError()
+ format = {'hex':hexval, 'macro':openssl_macro, 'note':""}
+ except KeyError:
+ # openssl doesn't have a macro for this.
+ format = {'hex':hexval, 'macro':firefox_macro,
+ 'note':"/* No openssl macro found for "+hexval+" */\n"}
+
+ res = """\
+%(note)s#ifdef %(macro)s
+ CIPHER(%(hex)s, %(macro)s)
+#else
+ XCIPHER(%(hex)s, %(macro)s)
+#endif""" % format
+ print res
diff --git a/scripts/maint/checkLogs.pl b/scripts/maint/checkLogs.pl
new file mode 100755
index 000000000..b00503e9a
--- /dev/null
+++ b/scripts/maint/checkLogs.pl
@@ -0,0 +1,51 @@
+#!/usr/bin/perl -w
+
+use strict;
+
+my %count = ();
+my $more = 0;
+my $last = "";
+
+while (<>) {
+ if ($more) {
+ if (/LD_BUG/) {
+ $more = 0;
+ next;
+ }
+ if (/\"((?:[^\"\\]+|\\.*)+)\"(.*)/) {
+ $last .= $1;
+ if ($2 !~ /[,\)]/) {
+ $more = 1;
+ } else {
+ $count{lc $last}++;
+ $more = 0;
+ }
+ } elsif (/[,\)]/) {
+ $count{lc $last}++;
+ $more = 0;
+ } elsif ($more == 2) {
+ print "SKIPPED more\n";
+ }
+ } elsif (/log_(?:warn|err|notice)\(\s*(LD_[A-Z_]*)\s*,\s*\"((?:[^\"\\]+|\\.)*)\"(.*)/) {
+ next if ($1 eq 'LD_BUG');
+ my $s = $2;
+ if ($3 =~ /[,\)]/ ) {
+ $count{lc $s}++;
+ } else {
+ $more = 1;
+ $last = $s;
+ }
+ } elsif (/log_(?:warn|err|notice)\(\s*((?:LD_[A-Z_]*)?)(.*)/) {
+ next if ($1 eq 'LD_BUG');
+ my $extra = $2;
+ chomp $extra;
+ $last = "";
+ $more = 2 if ($extra eq '');
+ }
+}
+
+while ((my $phrase, my $count) = each %count) {
+ if ($count > 1) {
+ print "$count\t$phrase\n";
+ }
+}
diff --git a/scripts/maint/checkOptionDocs.pl b/scripts/maint/checkOptionDocs.pl
new file mode 100755
index 000000000..94307c6ce
--- /dev/null
+++ b/scripts/maint/checkOptionDocs.pl
@@ -0,0 +1,71 @@
+#!/usr/bin/perl -w
+use strict;
+
+my %options = ();
+my %descOptions = ();
+my %torrcSampleOptions = ();
+my %manPageOptions = ();
+
+# Load the canonical list as actually accepted by Tor.
+open(F, "./src/or/tor --list-torrc-options |") or die;
+while (<F>) {
+ next if m!\[notice\] Tor v0\.!;
+ if (m!^([A-Za-z0-9_]+)!) {
+ $options{$1} = 1;
+ } else {
+ print "Unrecognized output> ";
+ print;
+ }
+}
+close F;
+
+# Load the contents of torrc.sample
+sub loadTorrc {
+ my ($fname, $options) = @_;
+ local *F;
+ open(F, "$fname") or die;
+ while (<F>) {
+ next if (m!##+!);
+ if (m!#([A-Za-z0-9_]+)!) {
+ $options->{$1} = 1;
+ }
+ }
+ close F;
+ 0;
+}
+
+loadTorrc("./src/config/torrc.sample.in", \%torrcSampleOptions);
+
+# Try to figure out what's in the man page.
+
+my $considerNextLine = 0;
+open(F, "./doc/tor.1.txt") or die;
+while (<F>) {
+ if (m!^(?:\[\[([A-za-z0-9_]+)\]\] *)?\*\*([A-Za-z0-9_]+)\*\*!) {
+ $manPageOptions{$2} = 1;
+ print "Missing an anchor: $2\n" unless (defined $1 or $2 eq 'tor');
+ }
+}
+close F;
+
+# Now, display differences:
+
+sub subtractHashes {
+ my ($s, $a, $b) = @_;
+ my @lst = ();
+ for my $k (keys %$a) {
+ push @lst, $k unless (exists $b->{$k});
+ }
+ print "$s: ", join(' ', sort @lst), "\n\n";
+ 0;
+}
+
+# subtractHashes("No online docs", \%options, \%descOptions);
+# subtractHashes("Orphaned online docs", \%descOptions, \%options);
+
+subtractHashes("Orphaned in torrc.sample.in", \%torrcSampleOptions, \%options);
+
+subtractHashes("Not in man page", \%options, \%manPageOptions);
+subtractHashes("Orphaned in man page", \%manPageOptions, \%options);
+
+
diff --git a/scripts/maint/checkSpace.pl b/scripts/maint/checkSpace.pl
new file mode 100755
index 000000000..682dbced0
--- /dev/null
+++ b/scripts/maint/checkSpace.pl
@@ -0,0 +1,138 @@
+#!/usr/bin/perl -w
+
+if ($ARGV[0] =~ /^-/) {
+ $lang = shift @ARGV;
+ $C = ($lang eq '-C');
+# $TXT = ($lang eq '-txt');
+}
+
+for $fn (@ARGV) {
+ open(F, "$fn");
+ $lastnil = 0;
+ $lastline = "";
+ $incomment = 0;
+ while (<F>) {
+ ## Warn about windows-style newlines.
+ if (/\r/) {
+ print " CR:$fn:$.\n";
+ }
+ ## Warn about tabs.
+ if (/\t/) {
+ print " TAB:$fn:$.\n";
+ }
+ ## Warn about markers that don't have a space in front of them
+ if (/^[a-zA-Z_][a-zA-Z_0-9]*:/) {
+ print "nosplabel:$fn:$.\n";
+ }
+ ## Warn about trailing whitespace.
+ if (/ +$/) {
+ print "Space\@EOL:$fn:$.\n";
+ }
+ ## Warn about control keywords without following space.
+ if ($C && /\s(?:if|while|for|switch)\(/) {
+ print " KW(:$fn:$.\n";
+ }
+ ## Warn about #else #if instead of #elif.
+ if (($lastline =~ /^\# *else/) and ($_ =~ /^\# *if/)) {
+ print " #else#if:$fn:$.\n";
+ }
+ ## Warn about some K&R violations
+ if (/^\s+\{/ and $lastline =~ /^\s*(if|while|for|else if)/ and
+ $lastline !~ /\{$/) {
+ print "non-K&R {:$fn:$.\n";
+ }
+ if (/^\s*else/ and $lastline =~ /\}$/) {
+ print " }\\nelse:$fn:$.\n";
+ }
+ $lastline = $_;
+ ## Warn about unnecessary empty lines.
+ if ($lastnil && /^\s*}\n/) {
+ print " UnnecNL:$fn:$.\n";
+ }
+ ## Warn about multiple empty lines.
+ if ($lastnil && /^$/) {
+ print " DoubleNL:$fn:$.\n";
+ } elsif (/^$/) {
+ $lastnil = 1;
+ } else {
+ $lastnil = 0;
+ }
+ ## Terminals are still 80 columns wide in my world. I refuse to
+ ## accept double-line lines.
+ if (/^.{80}/) {
+ print " Wide:$fn:$.\n";
+ }
+ ### Juju to skip over comments and strings, since the tests
+ ### we're about to do are okay there.
+ if ($C) {
+ if ($incomment) {
+ if (m!\*/!) {
+ s!.*?\*/!!;
+ $incomment = 0;
+ } else {
+ next;
+ }
+ }
+ if (m!/\*.*?\*/!) {
+ s!\s*/\*.*?\*/!!;
+ } elsif (m!/\*!) {
+ s!\s*/\*!!;
+ $incomment = 1;
+ next;
+ }
+ s!"(?:[^\"]+|\\.)*"!"X"!g;
+ next if /^\#/;
+ ## Warn about C++-style comments.
+ if (m!//!) {
+ # print " //:$fn:$.\n";
+ s!//.*!!;
+ }
+ ## Warn about unquoted braces preceded by non-space.
+ if (/([^\s'])\{/) {
+ print " $1\{:$fn:$.\n";
+ }
+ ## Warn about multiple internal spaces.
+ #if (/[^\s,:]\s{2,}[^\s\\=]/) {
+ # print " X X:$fn:$.\n";
+ #}
+ ## Warn about { with stuff after.
+ #s/\s+$//;
+ #if (/\{[^\}\\]+$/) {
+ # print " {X:$fn:$.\n";
+ #}
+ ## Warn about function calls with space before parens.
+ if (/(\w+)\s\(([A-Z]*)/) {
+ if ($1 ne "if" and $1 ne "while" and $1 ne "for" and
+ $1 ne "switch" and $1 ne "return" and $1 ne "int" and
+ $1 ne "elsif" and $1 ne "WINAPI" and $2 ne "WINAPI" and
+ $1 ne "void" and $1 ne "__attribute__" and $1 ne "op") {
+ print " fn ():$fn:$.\n";
+ }
+ }
+ ## Warn about functions not declared at start of line.
+ if ($in_func_head ||
+ ($fn !~ /\.h$/ && /^[a-zA-Z0-9_]/ &&
+ ! /^(?:const |static )*(?:typedef|struct|union)[^\(]*$/ &&
+ ! /= *\{$/ && ! /;$/)) {
+ if (/.\{$/){
+ print "fn() {:$fn:$.\n";
+ $in_func_head = 0;
+ } elsif (/^\S[^\(]* +\**[a-zA-Z0-9_]+\(/) {
+ $in_func_head = -1; # started with tp fn
+ } elsif (/;$/) {
+ $in_func_head = 0;
+ } elsif (/\{/) {
+ if ($in_func_head == -1) {
+ print "tp fn():$fn:$.\n";
+ }
+ $in_func_head = 0;
+ }
+ }
+ }
+ }
+ if (! $lastnil) {
+ print " EOL\@EOF:$fn:$.\n";
+ }
+ close(F);
+}
+
diff --git a/scripts/maint/check_config_macros.pl b/scripts/maint/check_config_macros.pl
new file mode 100755
index 000000000..bcde2becc
--- /dev/null
+++ b/scripts/maint/check_config_macros.pl
@@ -0,0 +1,20 @@
+#!/usr/bin/perl -w
+
+use strict;
+
+my @macros = ();
+
+open(F, 'orconfig.h.in');
+while(<F>) {
+ if (/^#undef +([A-Za-z0-9_]*)/) {
+ push @macros, $1;
+ }
+}
+close F;
+
+for my $m (@macros) {
+ my $s = `git grep '$m' src`;
+ if ($s eq '') {
+ print "Unused: $m\n";
+ }
+}
diff --git a/scripts/maint/findMergedChanges.pl b/scripts/maint/findMergedChanges.pl
new file mode 100755
index 000000000..d6c4105b7
--- /dev/null
+++ b/scripts/maint/findMergedChanges.pl
@@ -0,0 +1,73 @@
+#!/usr/bin/perl
+
+use warnings;
+use strict;
+
+sub nChanges {
+ my ($branches, $fname) = @_;
+ local *F;
+ # requires perl 5.8. Avoids shell issues if we ever get a changes
+ # file named by the parents of Little Johnny Tables.
+ open F, "-|", "git", "log", "--no-merges", "--pretty=format:%H", $branches, "--", $fname
+ or die "$!";
+ my @changes = <F>;
+ return scalar @changes
+}
+
+my $look_for_type = "merged";
+
+if (! @ARGV) {
+ print <<EOF
+Usage:
+ findMergedChanges.pl [--merged/--unmerged/--weird/--list] [--branch=<branchname] [--head=<branchname>] changes/*
+
+A change is "merged" if it has ever been merged to release-0.2.4 and it has had
+no subsequent changes in master.
+
+A change is "unmerged" if it has never been merged to release-0.2.4 and it
+has had changes in master.
+
+A change is "weird" if it has been merged to release-0.2.4 and it *has* had
+subsequent changes in master.
+
+Suggested application:
+ findMergedChanges.pl --merged changes/* | xargs -n 1 git rm
+
+EOF
+}
+
+my $target_branch = "origin/release-0.2.4";
+my $head = "origin/master";
+
+while (@ARGV and $ARGV[0] =~ /^--/) {
+ my $flag = shift @ARGV;
+ if ($flag =~ /^--(weird|merged|unmerged|list)/) {
+ $look_for_type = $1;
+ } elsif ($flag =~ /^--branch=(\S+)/) {
+ $target_branch = $1;
+ } elsif ($flag =~ /^--head=(\S+)/) {
+ $head = $1;
+ } else {
+ die "Unrecognized flag $flag";
+ }
+}
+
+for my $changefile (@ARGV) {
+ my $n_merged = nChanges($target_branch, $changefile);
+ my $n_postmerged = nChanges("${target_branch}..${head}", $changefile);
+ my $type;
+
+ if ($n_merged != 0 and $n_postmerged == 0) {
+ $type = "merged";
+ } elsif ($n_merged == 0 and $n_postmerged != 0) {
+ $type = "unmerged";
+ } else {
+ $type = "weird";
+ }
+
+ if ($type eq $look_for_type) {
+ print "$changefile\n";
+ } elsif ($look_for_type eq 'list') {
+ printf "% 8s: %s\n", $type, $changefile;
+ }
+}
diff --git a/scripts/maint/format_changelog.py b/scripts/maint/format_changelog.py
new file mode 100755
index 000000000..86f5c5039
--- /dev/null
+++ b/scripts/maint/format_changelog.py
@@ -0,0 +1,298 @@
+#!/usr/bin/python
+# Copyright (c) 2014, The Tor Project, Inc.
+# See LICENSE for licensing information
+#
+# This script reformats a section of the changelog to wrap everything to
+# the right width and put blank lines in the right places. Eventually,
+# it might include a linter.
+#
+# To run it, pipe a section of the changelog (starting with "Changes
+# in Tor 0.x.y.z-alpha" through the script.)
+
+import os
+import re
+import sys
+
+# ==============================
+# Oh, look! It's a cruddy approximation to Knuth's elegant text wrapping
+# algorithm, with totally ad hoc parameters!
+#
+# We're trying to minimize:
+# The total of the cubes of ragged space on underflowed intermediate lines,
+# PLUS
+# 100 * the fourth power of overflowed characters
+# PLUS
+# .1 * a bit more than the cube of ragged space on the last line.
+# PLUS
+# OPENPAREN_PENALTY for each line that starts with (
+#
+# We use an obvious dynamic programming algorithm to sorta approximate this.
+# It's not coded right or optimally, but it's fast enough for changelogs
+#
+# (Code found in an old directory of mine, lightly cleaned. -NM)
+
+NO_HYPHENATE=set("""
+pf-divert
+""".split())
+
+LASTLINE_UNDERFLOW_EXPONENT = 1
+LASTLINE_UNDERFLOW_PENALTY = 1
+
+UNDERFLOW_EXPONENT = 3
+UNDERFLOW_PENALTY = 1
+
+OVERFLOW_EXPONENT = 4
+OVERFLOW_PENALTY = 2000
+
+ORPHAN_PENALTY = 10000
+
+OPENPAREN_PENALTY = 200
+
+def generate_wrapping(words, divisions):
+ lines = []
+ last = 0
+ for i in divisions:
+ w = words[last:i]
+ last = i
+ line = " ".join(w).replace("\xff ","-").replace("\xff","-")
+ lines.append(line)
+ return lines
+
+def wrapping_quality(words, divisions, width1, width2):
+ total = 0.0
+
+ lines = generate_wrapping(words, divisions)
+ for line in lines:
+ length = len(line)
+ if line is lines[0]:
+ width = width1
+ else:
+ width = width2
+
+ if line[0:1] == '(':
+ total += OPENPAREN_PENALTY
+
+ if length > width:
+ total += OVERFLOW_PENALTY * (
+ (length - width) ** OVERFLOW_EXPONENT )
+ else:
+ if line is lines[-1]:
+ e,p = (LASTLINE_UNDERFLOW_EXPONENT, LASTLINE_UNDERFLOW_PENALTY)
+ if " " not in line:
+ total += ORPHAN_PENALTY
+ else:
+ e,p = (UNDERFLOW_EXPONENT, UNDERFLOW_PENALTY)
+
+ total += p * ((width - length) ** e)
+
+ return total
+
+def wrap_graf(words, prefix_len1=0, prefix_len2=0, width=72):
+ wrapping_after = [ (0,), ]
+
+ w1 = width - prefix_len1
+ w2 = width - prefix_len2
+
+ for i in range(1, len(words)+1):
+ best_so_far = None
+ best_score = 1e300
+ for j in range(i):
+ t = wrapping_after[j]
+ t1 = t[:-1] + (i,)
+ t2 = t + (i,)
+ wq1 = wrapping_quality(words, t1, w1, w2)
+ wq2 = wrapping_quality(words, t2, w1, w2)
+
+ if wq1 < best_score:
+ best_so_far = t1
+ best_score = wq1
+ if wq2 < best_score:
+ best_so_far = t2
+ best_score = wq2
+ wrapping_after.append( best_so_far )
+
+ lines = generate_wrapping(words, wrapping_after[-1])
+
+ return lines
+
+def hyphenateable(word):
+ if re.match(r'^[^\d\-]\D*-', word):
+ stripped = re.sub(r'^\W+','',word)
+ stripped = re.sub(r'\W+$','',word)
+ return stripped not in NO_HYPHENATE
+ else:
+ return False
+
+def split_paragraph(s):
+ "Split paragraph into words; tuned for Tor."
+
+ r = []
+ for word in s.split():
+ if hyphenateable(word):
+ while "-" in word:
+ a,word = word.split("-",1)
+ r.append(a+"\xff")
+ r.append(word)
+ return r
+
+def fill(text, width, initial_indent, subsequent_indent):
+ words = split_paragraph(text)
+ lines = wrap_graf(words, len(initial_indent), len(subsequent_indent),
+ width)
+ res = [ initial_indent, lines[0], "\n" ]
+ for line in lines[1:]:
+ res.append(subsequent_indent)
+ res.append(line)
+ res.append("\n")
+ return "".join(res)
+
+# ==============================
+
+
+TP_MAINHEAD = 0
+TP_HEADTEXT = 1
+TP_BLANK = 2
+TP_SECHEAD = 3
+TP_ITEMFIRST = 4
+TP_ITEMBODY = 5
+TP_END = 6
+
+def head_parser(line):
+ if re.match(r'^[A-Z]', line):
+ return TP_MAINHEAD
+ elif re.match(r'^ o ', line):
+ return TP_SECHEAD
+ elif re.match(r'^\s*$', line):
+ return TP_BLANK
+ else:
+ return TP_HEADTEXT
+
+def body_parser(line):
+ if re.match(r'^ o ', line):
+ return TP_SECHEAD
+ elif re.match(r'^ -',line):
+ return TP_ITEMFIRST
+ elif re.match(r'^ \S', line):
+ return TP_ITEMBODY
+ elif re.match(r'^\s*$', line):
+ return TP_BLANK
+ elif re.match(r'^Changes in', line):
+ return TP_END
+ else:
+ print "Weird line %r"%line
+
+class ChangeLog(object):
+ def __init__(self):
+ self.mainhead = None
+ self.headtext = []
+ self.curgraf = None
+ self.sections = []
+ self.cursection = None
+ self.lineno = 0
+
+ def addLine(self, tp, line):
+ self.lineno += 1
+
+ if tp == TP_MAINHEAD:
+ assert not self.mainhead
+ self.mainhead = line
+
+ elif tp == TP_HEADTEXT:
+ if self.curgraf is None:
+ self.curgraf = []
+ self.headtext.append(self.curgraf)
+ self.curgraf.append(line)
+
+ elif tp == TP_BLANK:
+ self.curgraf = None
+
+ elif tp == TP_SECHEAD:
+ self.cursection = [ self.lineno, line, [] ]
+ self.sections.append(self.cursection)
+
+ elif tp == TP_ITEMFIRST:
+ item = ( self.lineno, [ [line] ])
+ self.curgraf = item[1][0]
+ self.cursection[2].append(item)
+
+ elif tp == TP_ITEMBODY:
+ if self.curgraf is None:
+ self.curgraf = []
+ self.cursection[2][-1][1].append(self.curgraf)
+ self.curgraf.append(line)
+
+ else:
+ assert "This" is "unreachable"
+
+ def lint_head(self, line, head):
+ m = re.match(r'^ *o ([^\(]+)((?:\([^\)]+\))?):', head)
+ if not m:
+ print >>sys.stderr, "Weird header format on line %s"%line
+
+ def lint_item(self, line, grafs, head_type):
+ pass
+
+ def lint(self):
+ self.head_lines = {}
+ for sec_line, sec_head, items in self.sections:
+ head_type = self.lint_head(sec_line, sec_head)
+ for item_line, grafs in items:
+ self.lint_item(item_line, grafs, head_type)
+
+ def dumpGraf(self,par,indent1,indent2=-1):
+ if indent2 == -1:
+ indent2 = indent1
+ text = " ".join(re.sub(r'\s+', ' ', line.strip()) for line in par)
+
+ sys.stdout.write(fill(text,
+ width=72,
+ initial_indent=" "*indent1,
+ subsequent_indent=" "*indent2))
+
+ def dump(self):
+ print self.mainhead
+ for par in self.headtext:
+ self.dumpGraf(par, 2)
+ print
+ for _,head,items in self.sections:
+ if not head.endswith(':'):
+ print >>sys.stderr, "adding : to %r"%head
+ head = head + ":"
+ print head
+ for _,grafs in items:
+ self.dumpGraf(grafs[0],4,6)
+ for par in grafs[1:]:
+ print
+ self.dumpGraf(par,6,6)
+ print
+ print
+
+CL = ChangeLog()
+parser = head_parser
+
+sys.stdin = open('ChangeLog', 'r')
+
+for line in sys.stdin:
+ line = line.rstrip()
+ tp = parser(line)
+
+ if tp == TP_SECHEAD:
+ parser = body_parser
+ elif tp == TP_END:
+ nextline = line
+ break
+
+ CL.addLine(tp,line)
+
+CL.lint()
+
+sys.stdout = open('ChangeLog.new', 'w')
+
+CL.dump()
+
+print nextline
+
+for line in sys.stdin:
+ sys.stdout.write(line)
+
+os.rename('ChangeLog.new', 'ChangeLog')
diff --git a/scripts/maint/redox.py b/scripts/maint/redox.py
new file mode 100755
index 000000000..fa816a726
--- /dev/null
+++ b/scripts/maint/redox.py
@@ -0,0 +1,228 @@
+#!/usr/bin/python
+#
+# Copyright (c) 2008-2013, The Tor Project, Inc.
+# See LICENSE for licensing information.
+#
+# Hi!
+# I'm redox.py, the Tor redocumentation tool!
+# I am a horrible hack!
+# I read the output of doxygen from stderr, and add missing DOCDOC comments
+# to tell you where documentation should go!
+# To use me, edit the stuff below...
+# ...and run 'make doxygen 2>doxygen.stderr' ...
+# ...and run ./scripts/maint/redox.py < doxygen.stderr !
+# I'll make a bunch of new files by adding missing DOCDOC comments to your
+# source. Those files will have names like ./src/common/util.c.newdoc.
+# You will want to look over the changes by hand before checking them in.
+#
+# So, here's your workflow:
+#
+# 0. Make sure you're running a bourne shell for the redirects below.
+# 1. make doxygen 1>doxygen.stdout 2>doxygen.stderr.
+# 2. grep Warning doxygen.stderr | grep -v 'is not documented' | less
+# [This will tell you about all the bogus doxygen output you have]
+# 3. python ./scripts/maint/redox.py <doxygen.stderr
+# [This will make lots of .newdoc files with DOCDOC comments for
+# whatever was missing documentation.]
+# 4. Look over those .newdoc files, and see which docdoc comments you
+# want to merge into the main file. If it's all good, just run
+# "mv fname.c.newdoc fname.c". Otherwise, you'll need to merge
+# the parts you like by hand.
+
+# Which files should we ignore warning from? Mostly, these are external
+# files that we've snarfed in from somebody else, whose C we do no intend
+# to document for them.
+SKIP_FILES = [ "OpenBSD_malloc_Linux.c",
+ "eventdns.c",
+ "eventdns.h",
+ "strlcat.c",
+ "strlcpy.c",
+ "sha256.c",
+ "sha256.h",
+ "aes.c",
+ "aes.h" ]
+
+# What names of things never need javadoc
+SKIP_NAME_PATTERNS = [ r'^.*_c_id$',
+ r'^.*_H_ID$' ]
+
+# Which types of things should get DOCDOC comments added if they are
+# missing documentation? Recognized types are in KINDS below.
+ADD_DOCDOCS_TO_TYPES = [ 'function', 'type', 'typedef' ]
+ADD_DOCDOCS_TO_TYPES += [ 'variable', ]
+
+# ====================
+# The rest of this should not need hacking.
+
+import re
+import sys
+
+KINDS = [ "type", "field", "typedef", "define", "function", "variable",
+ "enumeration" ]
+
+NODOC_LINE_RE = re.compile(r'^([^:]+):(\d+): (\w+): (.*) is not documented\.$')
+
+THING_RE = re.compile(r'^Member ([a-zA-Z0-9_]+).*\((typedef|define|function|variable|enumeration|macro definition)\) of (file|class) ')
+
+SKIP_NAMES = [re.compile(s) for s in SKIP_NAME_PATTERNS]
+
+def parsething(thing):
+ """I figure out what 'foobar baz in quux quum is not documented' means,
+ and return: the name of the foobar, and the kind of the foobar.
+ """
+ if thing.startswith("Compound "):
+ tp, name = "type", thing.split()[1]
+ else:
+ m = THING_RE.match(thing)
+ if not m:
+ print thing, "???? Format didn't match."
+ return None, None
+ else:
+ name, tp, parent = m.groups()
+ if parent == 'class':
+ if tp == 'variable' or tp == 'function':
+ tp = 'field'
+
+ return name, tp
+
+def read():
+ """I snarf doxygen stderr from stdin, and parse all the "foo has no
+ documentation messages. I return a map from filename to lists
+ of tuples of (alleged line number, name of thing, kind of thing)
+ """
+ errs = {}
+ for line in sys.stdin:
+ m = NODOC_LINE_RE.match(line)
+ if m:
+ file, line, tp, thing = m.groups()
+ assert tp.lower() == 'warning'
+ name, kind = parsething(thing)
+ errs.setdefault(file, []).append((int(line), name, kind))
+
+ return errs
+
+def findline(lines, lineno, ident):
+ """Given a list of all the lines in the file (adjusted so 1-indexing works),
+ a line number that ident is alledgedly on, and ident, I figure out
+ the line where ident was really declared."""
+ lno = lineno
+ for lineno in xrange(lineno, 0, -1):
+ try:
+ if ident in lines[lineno]:
+ return lineno
+ except IndexError:
+ continue
+
+ return None
+
+FUNC_PAT = re.compile(r"^[A-Za-z0-9_]+\(")
+
+def hascomment(lines, lineno, kind):
+ """I return true if it looks like there's already a good comment about
+ the thing on lineno of lines of type kind. """
+ if "*/" in lines[lineno-1]:
+ return True
+ if kind == 'function' and FUNC_PAT.match(lines[lineno]):
+ if "*/" in lines[lineno-2]:
+ return True
+ return False
+
+def hasdocdoc(lines, lineno, kind):
+ """I return true if it looks like there's already a docdoc comment about
+ the thing on lineno of lines of type kind."""
+ try:
+ if "DOCDOC" in lines[lineno]:
+ return True
+ except IndexError:
+ pass
+ try:
+ if "DOCDOC" in lines[lineno-1]:
+ return True
+ except IndexError:
+ pass
+ if kind == 'function' and FUNC_PAT.match(lines[lineno]):
+ if "DOCDOC" in lines[lineno-2]:
+ return True
+ return False
+
+def checkf(fn, errs):
+ """I go through the output of read() for a single file, and build a list
+ of tuples of things that want DOCDOC comments. Each tuple has:
+ the line number where the comment goes; the kind of thing; its name.
+ """
+ for skip in SKIP_FILES:
+ if fn.endswith(skip):
+ print "Skipping",fn
+ return
+
+ comments = []
+ lines = [ None ]
+ try:
+ lines.extend( open(fn, 'r').readlines() )
+ except IOError:
+ return
+
+ for line, name, kind in errs:
+ if any(pat.match(name) for pat in SKIP_NAMES):
+ continue
+
+ if kind not in ADD_DOCDOCS_TO_TYPES:
+ continue
+
+ ln = findline(lines, line, name)
+ if ln == None:
+ print "Couldn't find the definition of %s allegedly on %s of %s"%(
+ name, line, fn)
+ else:
+ if hasdocdoc(lines, line, kind):
+# print "Has a DOCDOC"
+# print fn, line, name, kind
+# print "\t",lines[line-2],
+# print "\t",lines[line-1],
+# print "\t",lines[line],
+# print "-------"
+ pass
+ else:
+ if kind == 'function' and FUNC_PAT.match(lines[ln]):
+ ln = ln - 1
+
+ comments.append((ln, kind, name))
+
+ return comments
+
+def applyComments(fn, entries):
+ """I apply lots of comments to the file in fn, making a new .newdoc file.
+ """
+ N = 0
+
+ lines = [ None ]
+ try:
+ lines.extend( open(fn, 'r').readlines() )
+ except IOError:
+ return
+
+ # Process the comments in reverse order by line number, so that
+ # the line numbers for the ones we haven't added yet remain valid
+ # until we add them. Standard trick.
+ entries.sort()
+ entries.reverse()
+
+ for ln, kind, name in entries:
+
+ lines.insert(ln, "/* DOCDOC %s */\n"%name)
+ N += 1
+
+ outf = open(fn+".newdoc", 'w')
+ for line in lines[1:]:
+ outf.write(line)
+ outf.close()
+
+ print "Added %s DOCDOCs to %s" %(N, fn)
+
+e = read()
+
+for fn, errs in e.iteritems():
+ print `(fn, errs)`
+ comments = checkf(fn, errs)
+ if comments:
+ applyComments(fn, comments)
diff --git a/scripts/maint/sortChanges.py b/scripts/maint/sortChanges.py
new file mode 100755
index 000000000..f70490bad
--- /dev/null
+++ b/scripts/maint/sortChanges.py
@@ -0,0 +1,40 @@
+#!/usr/bin/python
+
+import re
+import sys
+
+def fetch(fn):
+ with open(fn) as f:
+ s = f.read()
+ s = "%s\n" % s.rstrip()
+ return s
+
+def score(s):
+ m = re.match(r'^ +o (.*)', s)
+ if not m:
+ print >>sys.stderr, "Can't score %r"%s
+ lw = m.group(1).lower()
+ if lw.startswith("major feature"):
+ score = 0
+ elif lw.startswith("major bug"):
+ score = 1
+ elif lw.startswith("major"):
+ score = 2
+ elif lw.startswith("minor feature"):
+ score = 10
+ elif lw.startswith("minor bug"):
+ score = 11
+ elif lw.startswith("minor"):
+ score = 12
+ else:
+ score = 100
+
+ return (score, lw, s)
+
+
+changes = [ score(fetch(fn)) for fn in sys.argv[1:] if not fn.endswith('~') ]
+
+changes.sort()
+
+for _, _, s in changes:
+ print s
diff --git a/scripts/maint/updateVersions.pl b/scripts/maint/updateVersions.pl
new file mode 100755
index 000000000..15c83b80a
--- /dev/null
+++ b/scripts/maint/updateVersions.pl
@@ -0,0 +1,59 @@
+#!/usr/bin/perl -w
+
+$CONFIGURE_IN = './configure.ac';
+$ORCONFIG_H = './src/win32/orconfig.h';
+$TOR_NSI = './contrib/win32build/tor-mingw.nsi.in';
+
+$quiet = 1;
+
+sub demand {
+ my $fn = shift;
+ die "Missing file $fn" unless (-f $fn);
+}
+
+demand($CONFIGURE_IN);
+demand($ORCONFIG_H);
+demand($TOR_NSI);
+
+# extract version from configure.ac
+
+open(F, $CONFIGURE_IN) or die "$!";
+$version = undef;
+while (<F>) {
+ if (/AC_INIT\(\[tor\],\s*\[([^\]]*)\]\)/) {
+ $version = $1;
+ last;
+ }
+}
+die "No version found" unless $version;
+print "Tor version is $version\n" unless $quiet;
+close F;
+
+sub correctversion {
+ my ($fn, $defchar) = @_;
+ undef $/;
+ open(F, $fn) or die "$!";
+ my $s = <F>;
+ close F;
+ if ($s =~ /^$defchar(?:)define\s+VERSION\s+\"([^\"]+)\"/m) {
+ $oldver = $1;
+ if ($oldver ne $version) {
+ print "Version mismatch in $fn: It thinks that the version is $oldver. I think it's $version. Fixing.\n";
+ $line = $defchar . "define VERSION \"$version\"";
+ open(F, ">$fn.bak");
+ print F $s;
+ close F;
+ $s =~ s/^$defchar(?:)define\s+VERSION.*?$/$line/m;
+ open(F, ">$fn");
+ print F $s;
+ close F;
+ } else {
+ print "$fn has the correct version. Good.\n" unless $quiet;
+ }
+ } else {
+ print "Didn't find a version line in $fn -- uh oh.\n";
+ }
+}
+
+correctversion($TOR_NSI, "!");
+correctversion($ORCONFIG_H, "#");
diff --git a/scripts/test/cov-blame b/scripts/test/cov-blame
new file mode 100755
index 000000000..601f21195
--- /dev/null
+++ b/scripts/test/cov-blame
@@ -0,0 +1,48 @@
+#!/usr/bin/python
+
+import os
+import re
+import subprocess
+import sys
+
+def handle_file(source_fname, cov_fname):
+
+ lines_blm = subprocess.Popen(["git", "blame", source_fname], stdout=subprocess.PIPE).stdout.readlines()
+ lines_cov = open(cov_fname).readlines()
+
+ # XXXX expensive!
+ while re.match(r'\s*-:\s*0:', lines_cov[0]):
+ del lines_cov[0]
+
+ if len(lines_blm) != len(lines_cov):
+ print >>sys.stderr, "MISMATCH IN NUMBER OF LINES in",source_fname
+
+ for b,c in zip(lines_blm, lines_cov):
+ m = re.match(r'\s*([^\s:]+):', c)
+ if not m:
+ print >>sys.stderr, "CONFUSING LINE %r"% c
+ cov = 'X'
+ elif m.group(1) == '-':
+ cov = '-'
+ elif m.group(1)[0] == '#':
+ cov = '#'
+ elif m.group(1)[0].isdigit():
+ cov = '1'
+ else:
+ print >>sys.stderr, "CONFUSING LINE %r"% c
+ cov = 'X'
+
+ print cov, b,
+
+COV_DIR = sys.argv[1]
+SOURCES = sys.argv[2:]
+
+for fn in SOURCES:
+ _, base = os.path.split(fn)
+ cfn = os.path.join(COV_DIR, base)
+ cfn += ".gcov"
+ if os.path.exists(cfn):
+ handle_file(fn, cfn)
+ else:
+ print >>sys.stderr, "NO FILE EXISTS CALLED ",cfn
+
diff --git a/scripts/test/cov-diff b/scripts/test/cov-diff
new file mode 100755
index 000000000..33a54802b
--- /dev/null
+++ b/scripts/test/cov-diff
@@ -0,0 +1,17 @@
+#!/bin/sh
+# Copyright 2013 The Tor Project, Inc.
+# See LICENSE for licensing information.
+
+# cov-diff -- compare two directories full of gcov files.
+
+DIRA="$1"
+DIRB="$2"
+
+for A in $DIRA/*; do
+ B=$DIRB/`basename $A`
+ perl -pe 's/^\s*\d+:/ 1:/; s/^([^:]+:)[\d\s]+:/$1/;' "$A" > "$A.tmp"
+ perl -pe 's/^\s*\d+:/ 1:/; s/^([^:]+:)[\d\s]+:/$1/;' "$B" > "$B.tmp"
+ diff -u "$A.tmp" "$B.tmp"
+ rm "$A.tmp" "$B.tmp"
+done
+
diff --git a/scripts/test/coverage b/scripts/test/coverage
new file mode 100755
index 000000000..f4ae47582
--- /dev/null
+++ b/scripts/test/coverage
@@ -0,0 +1,46 @@
+#!/bin/sh
+# Copyright 2013 The Tor Project, Inc.
+# See LICENSE for licensing information.
+
+# coverage -- run gcov on the appropriate set of object files to extract
+# coverage information.
+
+dst=$1
+
+for fn in src/or/*.c src/common/*.c; do
+ BN=`basename $fn`
+ DN=`dirname $fn`
+ F=`echo $BN | sed -e 's/\.c$//;'`
+ GC="${BN}.gcov"
+ # Figure out the object file names
+ ONS=`echo ${DN}/src_*-${F}.o`
+ ONS_WILDCARD_LITERAL="${DN}/src_*-${F}.o"
+ # If the wildcard didn't expand, no files
+ if [ "$ONS" != "${ONS_WILDCARD_LITERAL}" ]
+ then
+ for on in $ONS; do
+ # We should have a gcno file
+ GCNO=`echo $on | sed -e 's/\.o$/\.gcno/;'`
+ if [ -e $GCNO ]
+ then
+ # No need to test for gcda, since gcov assumes no execution
+ # if it's absent
+ rm -f $GC
+ gcov -o $on $fn
+ if [ -e $GC ]
+ then
+ if [ -n $dst ]
+ then
+ mv $GC $dst/$GC
+ fi
+ else
+ echo "gcov -o $on $fn didn't make a .gcov file"
+ fi
+ else
+ echo "Couldn't find gcno file for $on"
+ fi
+ done
+ else
+ echo "No object file found matching source file $fn"
+ fi
+done
diff --git a/scripts/test/scan-build.sh b/scripts/test/scan-build.sh
new file mode 100644
index 000000000..623b227fe
--- /dev/null
+++ b/scripts/test/scan-build.sh
@@ -0,0 +1,49 @@
+#!/bin/sh
+# Copyright 2014 The Tor Project, Inc
+# See LICENSE for licensing information
+#
+# This script is used for running a bunch of clang scan-build checkers
+# on Tor.
+#
+# It has hardwired paths for Nick's desktop at the moment.
+
+CHECKERS="\
+ --use-analyzer=/opt/clang-3.4/bin/clang \
+ -disable-checker deadcode.DeadStores \
+ -enable-checker alpha.core.CastSize \
+ -enable-checker alpha.core.CastToStruct \
+ -enable-checker alpha.core.IdenticalExpr \
+ -enable-checker alpha.core.SizeofPtr \
+ -enable-checker alpha.security.ArrayBoundV2 \
+ -enable-checker alpha.security.MallocOverflow \
+ -enable-checker alpha.security.ReturnPtrRange \
+ -enable-checker alpha.unix.SimpleStream
+ -enable-checker alpha.unix.cstring.BufferOverlap \
+ -enable-checker alpha.unix.cstring.NotNullTerminated \
+ -enable-checker alpha.unix.cstring.OutOfBounds \
+ -enable-checker alpha.core.FixedAddr \
+ -enable-checker security.insecureAPI.strcpy
+"
+
+/opt/clang-3.4/bin/scan-build/scan-build \
+ $CHECKERS \
+ --use-analyzer=/opt/clang-3.4/bin/clang \
+ ./configure
+
+/opt/clang-3.4/bin/scan-build/scan-build \
+ $CHECKERS \
+ --use-analyzer=/opt/clang-3.4/bin/clang \
+ make -j2
+
+
+# Haven't tried this yet.
+# -enable-checker alpha.unix.PthreadLock
+
+# This one gives a false positive on every strcmp.
+# -enable-checker alpha.core.PointerSub
+
+# This one hates it when we stick a nonzero const in a pointer.
+# -enable-checker alpha.core.FixedAddr
+
+# This one crashes sometimes for me.
+# -enable-checker alpha.deadcode.IdempotentOperations