"""How often does a published CVE record link a specific commit? Usage: python3 fixlinks.py cves.zip kev.json SECTION SECTION is one of: headline, years, cnas, hosts, shapes, kev, misses, pools, sample Reads the CNA container of every PUBLISHED record in a cvelistV5 bulk export and counts records whose references include a URL that names one specific revision in a version control system. Standard library only. """ import collections, json, random, re, sys, zipfile CUTOFF = "2026-09-01T00:00:00" # datePublished strictly before this KERNEL = "416baaa9-dc9f-4396-8d5f-8c081fb06d67" # Linux kernel CNA orgId SHA = r"[0-9a-f]{7,40}(?![0-9a-z])" GIT = re.compile("|".join([ r"github\.com/[^/]+/[^/]+/(?:pull/\d+/)?commits?/" + SHA, r"/-/commits?/" + SHA, # GitLab, any host r"gitlab[^/]*/.*/commit/" + SHA, r"git\.kernel\.org/.*(?:/c/|/linus/|[?&;]id=)" + SHA, r"kernel\.dance/#[0-9a-f]{12,40}", r"googlesource\.com/.*/\+/" + SHA, r"bitbucket\.org/.*/commits?/" + SHA, r"sourceforge\.net/.*/ci/" + SHA, ]), re.I) # cgit, gitweb, Gitea and similar: the word commit plus a hash parameter # or path segment. The hash must contain a letter so that numeric ids # such as ?id=1234567 are not mistaken for revisions. GENERIC_HASH = re.compile(r"[?&;/=](?:id=|h=)?(?=[0-9]*[a-f])" + SHA, re.I) NOT_VCS = re.compile(r"lists\.|mail|thread|bugzilla|issues?/", re.I) SVN = re.compile(r"/changeset/\d+|/changeset\?|[?&]rev=\d+(?!\d)", re.I) HG = re.compile(r"/rev/[0-9a-f]{12,40}(?![0-9a-z])", re.I) PULL = re.compile(r"github\.com/[^/]+/[^/]+/pull/\d+", re.I) # Used only by the misses section, which counts what the rules above get # wrong in either direction across the whole export. SKIPPED = { "gitweb h= with no 'commit'": r"\?p=[^;&]+\.git[;&]h=" + SHA + "$|[?&;]a=patch[;&].*h=" + SHA, "cgit patch or diff view": r"/(?:patch|diff)/[^?]*\?(?:.*&)?id=" + SHA, "Pagure commit": r"pagure\.io/.+/c/" + SHA, "commit in a PR file view": r"github\.com/[^/]+/[^/]+/pull/\d+/(?:files|changes)/" + SHA, "compare between two hashes": r"github\.com/[^/]+/[^/]+/compare/[0-9a-f]{7,40}\.\.\.?.*[0-9a-f]{7,40}", } WRONG = { "gist, user name contains commits": r"gist\.github\.com/", "Gerrit change number read as hash": r"googlesource\.com/.*/\+/[0-9]{7,40}(?![0-9a-z])", "gitweb search page": r"[?&;]a=search", } def is_git(url): u = url.replace("%3B", ";").replace("%3b", ";") if not re.match(r"https?://", u, re.I): return False if GIT.search(u): return True return bool(re.search("commit", u, re.I) and GENERIC_HASH.search(u) and not NOT_VCS.search(u)) def host(url): h = re.sub(r"^https?://(www\.)?", "", url.lower()).split("/")[0] if h in ("github.com", "gitlab.com", "git.kernel.org", "kernel.dance"): return "git.kernel.org" if h == "kernel.dance" else h if h.endswith("googlesource.com"): return "googlesource.com" if "/-/commit" in url: return "other GitLab" return "other" def records(path): with zipfile.ZipFile(path) as z: for name in sorted(z.namelist()): if name.endswith(".json") and "/CVE-" in name: yield json.loads(z.read(name)) def study(path, kev): states, rows, late = collections.Counter(), [], set() for d in records(path): meta = d.get("cveMetadata", {}) states[meta.get("state")] += 1 pub = meta.get("datePublished") or "" if meta.get("state") == "PUBLISHED" and pub[:19] >= CUTOFF: late.add(meta["cveId"]) if meta.get("state") != "PUBLISHED" or not pub or pub[:19] >= CUTOFF: continue cons = d.get("containers", {}) refs = [r for r in cons.get("cna", {}).get("references") or [] if isinstance(r, dict) and r.get("url")] adp = [r for a in cons.get("adp") or [] for r in a.get("references") or [] if isinstance(r, dict) and r.get("url")] git = [r for r in refs if is_git(r["url"])] rows.append(dict( id=meta["cveId"], year=pub[:4], cna=meta.get("assignerShortName") or "?", kernel=meta.get("assignerOrgId") == KERNEL, kev=meta["cveId"] in kev, urls=[r["url"] for r in refs], git=[r["url"] for r in git], adp_git=any(is_git(r["url"]) for r in adp), tagged=any("patch" in [t.lower() for t in r.get("tags") or []] for r in git))) return states, rows, late def pct(a, b): return f"{100 * a / b:.1f}%" if b else "-" def main(): path, kevpath, section = sys.argv[1:4] kev = {v["cveID"] for v in json.load(open(kevpath))["vulnerabilities"]} states, rows, late = study(path, kev) linked = [r for r in rows if r["git"]] nk = [r for r in rows if not r["kernel"]] nkl = [r for r in nk if r["git"]] if section == "headline": print("records in export ", dict(sorted(states.items(), key=str))) print(f"published before {CUTOFF[:10]} {len(rows):,}") print(f"link a git commit {len(linked):,} {pct(len(linked), len(rows))}") print(f" plus ADP only {sum(1 for r in rows if r['adp_git'] and not r['git']):,}") k = [r for r in rows if r["kernel"]] kl = [r for r in k if r["git"]] print(f"Linux kernel CNA {len(kl):,} of {len(k):,} {pct(len(kl), len(k))}" f" ({pct(len(kl), len(linked))} of all linking records)") print(f"everyone else {len(nkl):,} of {len(nk):,} {pct(len(nkl), len(nk))}") elif section == "years": print("year published linked % excl. kernel linked %") for y in sorted({r["year"] for r in rows}): a = [r for r in rows if r["year"] == y] b = [r for r in a if not r["kernel"]] al, bl = sum(1 for r in a if r["git"]), sum(1 for r in b if r["git"]) print(f"{y} {len(a):9,} {al:6,} {pct(al, len(a)):>5} {len(b):12,} {bl:6,} {pct(bl, len(b)):>5}") elif section == "cnas": n, c = collections.Counter(r["cna"] for r in rows), collections.Counter(r["cna"] for r in linked) print("CNA linked of records %") for cna, k in c.most_common(15): print(f"{cna[:24]:24} {k:7,} {n[cna]:11,} {pct(k, n[cna]):>5}") print() print("year top three CNAs by linking records") for y in sorted({r["year"] for r in linked if r["year"] >= "2010"}): top = collections.Counter(r["cna"] for r in linked if r["year"] == y).most_common(3) print(y, " ", ", ".join(f"{a} {b:,}" for a, b in top)) print() print("year top three CNAs by records published (of which linked)") for y in sorted({r["year"] for r in rows if r["year"] >= "2010"}): yr = [r for r in rows if r["year"] == y] yl = collections.Counter(r["cna"] for r in yr if r["git"]) top = collections.Counter(r["cna"] for r in yr).most_common(3) print(y, " ", ", ".join(f"{a} {b:,} ({yl[a]:,})" for a, b in top)) elif section == "hosts": by_rec, by_ref = collections.Counter(), collections.Counter() for r in linked: for h in {host(u) for u in r["git"]}: by_rec[h] += 1 for u in r["git"]: by_ref[host(u)] += 1 print("host records links") for h, k in by_rec.most_common(): print(f"{h:18} {k:7,} {by_ref[h]:6,}") elif section == "shapes": svn = [r for r in rows if any(SVN.search(u) for u in r["urls"])] print(f"SVN changeset, no git commit {sum(1 for r in svn if not r['git']):,}") print(f" of which trac.wordpress.org {sum(1 for r in svn if not r['git'] and any('trac.wordpress.org' in u for u in r['urls'])):,}") print(f"Mercurial revision, no git commit {sum(1 for r in rows if not r['git'] and any(HG.search(u) for u in r['urls'])):,}") print(f"GitHub pull request, no commit {sum(1 for r in rows if not r['git'] and any(PULL.search(u) for u in r['urls'])):,}") print(f"no references at all {sum(1 for r in rows if not r['urls']):,}") print(f"linking records, commit ref tagged 'patch' {sum(1 for r in linked if r['tagged']):,} of {len(linked):,}") print(f"linking records, 2+ distinct commits {sum(1 for r in linked if len(set(r['git'])) > 1):,}") elif section == "kev": kv = [r for r in rows if r["kev"]] kvl = [r for r in kv if r["git"]] print(f"KEV entries in scope {len(kv):,} (catalogue lists {len(kev):,})") print(f"link a git commit {len(kvl):,} {pct(len(kvl), len(kv))}") print(f" excluding kernel {sum(1 for r in kvl if not r['kernel']):,} of {sum(1 for r in kv if not r['kernel']):,}") print(f"KEV entries published on or after {CUTOFF[:10]} {len(kev & late):,}") elif section == "misses": dec = lambda u: u.replace("%3B", ";").replace("%3b", ";") none = [r for r in rows if not r["git"]] extra = set() print("git shapes the rules skip, in records with no counted link") for name, pat in SKIPPED.items(): ids = {r["id"] for r in none if any(re.search(pat, dec(u), re.I) for u in r["urls"])} extra |= ids print(f" {name:36} {len(ids):5,}") print(f" any of these {len(extra):5,}") wrong = set() print("records counted only through a shape that is not a commit") for name, pat in WRONG.items(): ids = {r["id"] for r in linked if all(re.search(pat, dec(u), re.I) for u in r["git"])} wrong |= ids print(f" {name:36} {len(ids):5,}") print(f" any of these {len(wrong):5,}") print(f"headline as printed {pct(len(linked), len(rows))}") print(f"with both corrections {pct(len(linked) + len(extra) - len(wrong), len(rows))}") elif section in ("sample", "pools"): rng = random.Random(20260927) refs = [(r["id"], u, is_git(u)) for r in rows for u in r["urls"]] hit = [x for x in refs if x[2]] near = [x for x in refs if not x[2] and re.search(r"commit|changeset|/rev/|[0-9a-f]{12,40}", x[1], re.I)] none = [r for r in rows if not r["git"]] if section == "pools": print(f"references counted as commit links {len(hit):,}") print(f"references that look like VCS, not counted {len(near):,}") print(f"records with no commit link {len(none):,}") return for tag, pick in (("MATCH", rng.sample(hit, 200)), ("NEAR", rng.sample(near, 200))): for cve, u, _ in pick: print(tag, cve, u, sep="\t") for r in rng.sample(none, 100): print("RECORD", r["id"], " ".join(r["urls"]) or "-", sep="\t") if __name__ == "__main__": main()