Skills › Scientific Agent Skills › citation-management › _common.py
_common.py
331 lines · skills/citation-management/scripts/_common.py at 5f2bc03
1#!/usr/bin/env python3
2"""Shared BibTeX parsing and rendering for the citation-management scripts.
4Standard library only.
6The parser here is brace-depth aware. That matters more than it sounds: a
7regular expression of the form ``\\{([^}]*)\\}`` stops at the first closing
8brace, so it truncates every title that protects a capitalised term --
9``{Highly accurate prediction with {AlphaFold}}`` -- and writing the truncated
10value back out produces a ``.bib`` file with unbalanced braces that no BibTeX
11engine will read. Capitalisation protection is routine in real bibliographies,
12and ``extract_metadata.py`` emits it deliberately, so the naive form corrupts
13this skill's own output.
15Every producer in this skill renders through :func:`render_entry` and keys
16through :func:`citation_key`, so entries from Crossref, PubMed, OpenAlex, and
17Scholar are directly comparable and deduplication works across sources.
18"""
20from __future__ import annotations
22import re
23import sys
24import unicodedata
25from collections import OrderedDict
26from typing import Dict, List, Optional
28# Field order used when rendering. Anything not listed keeps its input order
29# and is appended after these.
30FIELD_ORDER = [
31 'author', 'editor', 'title', 'booktitle', 'journal',
32 'year', 'month', 'volume', 'number', 'pages',
33 'publisher', 'address', 'edition', 'series',
34 'school', 'institution', 'organization',
35 'howpublished', 'doi', 'url', 'isbn', 'issn',
36 'note', 'abstract', 'keywords',
37]
39# @string, @comment and @preamble are not bibliography entries.
40_NON_ENTRY_TYPES = {'string', 'comment', 'preamble'}
42_FIELD_NAME = re.compile(r'\s*([A-Za-z][A-Za-z0-9_+:.-]*)\s*=\s*(.*)$', re.DOTALL)
45def _is_outer_braced(value: str) -> bool:
46 """True when the whole value is wrapped in one matched brace pair."""
47 if not (value.startswith('{') and value.endswith('}')):
48 return False
49 depth = 0
50 for index, char in enumerate(value):
51 if char == '{':
52 depth += 1
53 elif char == '}':
54 depth -= 1
55 if depth == 0:
56 # Only a wrapper if it closes at the very end; `{a} # {b}`
57 # and `{a}{b}` must be left alone.
58 return index == len(value) - 1
59 return False
62def _unwrap(value: str) -> str:
63 """Strip one layer of delimiters, preserving any nested braces inside."""
64 value = value.strip()
65 if _is_outer_braced(value):
66 return value[1:-1].strip()
67 if len(value) >= 2 and value[0] == '"' and value[-1] == '"':
68 return value[1:-1].strip()
69 return value
72def _split_top_level(body: str) -> List[str]:
73 """Split on commas that sit outside braces and quotes."""
74 parts: List[str] = []
75 buffer: List[str] = []
76 depth = 0
77 in_quote = False
79 for char in body:
80 if char == '"' and depth == 0:
81 in_quote = not in_quote
82 elif char == '{':
83 depth += 1
84 elif char == '}':
85 depth -= 1
87 if char == ',' and depth == 0 and not in_quote:
88 parts.append(''.join(buffer))
89 buffer = []
90 else:
91 buffer.append(char)
93 parts.append(''.join(buffer))
94 return parts
97def parse_bibtex(content: str) -> List[Dict]:
98 """Parse BibTeX source into entry dictionaries.
100 Each entry is ``{'type', 'key', 'fields', 'raw'}`` with ``fields`` an
101 ``OrderedDict`` of lowercased field names to their unwrapped values.
102 Malformed or unterminated entries are skipped rather than raised on.
103 """
104 entries: List[Dict] = []
105 length = len(content)
106 position = 0
108 while True:
109 at = content.find('@', position)
110 if at < 0:
111 break
113 cursor = at + 1
114 while cursor < length and content[cursor].isalpha():
115 cursor += 1
116 entry_type = content[at + 1:cursor].lower()
118 if not entry_type:
119 position = at + 1
120 continue
122 while cursor < length and content[cursor].isspace():
123 cursor += 1
125 if cursor >= length or content[cursor] not in '{(':
126 position = at + 1
127 continue
129 opener = content[cursor]
130 closer = '}' if opener == '{' else ')'
132 # Walk to the delimiter that matches the opener, counting depth.
133 depth = 0
134 end = -1
135 scan = cursor
136 while scan < length:
137 char = content[scan]
138 if char == opener:
139 depth += 1
140 elif char == closer:
141 depth -= 1
142 if depth == 0:
143 end = scan
144 break
145 scan += 1
147 if end < 0:
148 # Unterminated entry: nothing reliable left to read.
149 break
151 position = end + 1
153 if entry_type in _NON_ENTRY_TYPES:
154 continue
156 body = content[cursor + 1:end]
157 chunks = _split_top_level(body)
158 if not chunks:
159 continue
161 citation_key_text = chunks[0].strip()
162 if not citation_key_text or '=' in citation_key_text:
163 # No citation key -- not an entry we can work with.
164 continue
166 fields: "OrderedDict[str, str]" = OrderedDict()
167 for chunk in chunks[1:]:
168 if not chunk.strip():
169 continue
170 match = _FIELD_NAME.match(chunk)
171 if not match:
172 continue
173 fields[match.group(1).lower()] = _unwrap(match.group(2))
175 entries.append({
176 'type': entry_type,
177 'key': citation_key_text,
178 'fields': fields,
179 'raw': content[at:end + 1],
180 })
182 return entries
185def parse_bibtex_file(filepath: str) -> List[Dict]:
186 """Parse a BibTeX file, reporting read errors rather than raising."""
187 try:
188 with open(filepath, 'r', encoding='utf-8') as handle:
189 return parse_bibtex(handle.read())
190 except OSError as error:
191 print(f'Error reading file: {error}', file=sys.stderr)
192 return []
195def render_entry(entry_type: str, key: str, fields: Dict[str, str]) -> str:
196 """Render one entry with fields in a stable order and aligned values.
198 Empty values are dropped, so callers can pass a dictionary built from
199 metadata that may or may not have been populated.
200 """
201 present = OrderedDict()
202 for name in FIELD_ORDER:
203 value = fields.get(name)
204 if value:
205 present[name] = str(value)
206 for name, value in fields.items():
207 if name not in present and value:
208 present[name] = str(value)
210 if not present:
211 return f'@{entry_type}{{{key},\n}}'
213 width = max(len(name) for name in present)
214 lines = [f'@{entry_type}{{{key},']
215 rendered = [f' {name.ljust(width)} = {{{value}}}' for name, value in present.items()]
216 lines.append(',\n'.join(rendered))
217 lines.append('}')
218 return '\n'.join(lines)
221_PAGE_RANGE = re.compile(r'^(\d+)\s*(?:-{1,3}|–|—)\s*(\d+)$')
224def format_pages(raw: Optional[str]) -> str:
225 """Normalise a page field to a BibTeX en-dash range.
227 Handles the two mistakes the previous per-script implementations made:
228 a blanket ``replace('-', '--')`` turned an already-correct ``583--589``
229 into ``583----589``, and PubMed's abbreviated ``MedlinePgn`` ranges
230 (``1123-30``) need their end page expanded to ``1123--1130`` rather than
231 being left as a nonsensical ``1123--30``.
233 Values that are not ranges -- article numbers such as ``e0123456``,
234 or discontinuous lists -- are returned unchanged.
235 """
236 if not raw:
237 return ''
239 pages = re.sub(r'^\s*pp?\.\s*', '', str(raw).strip(), flags=re.IGNORECASE)
240 match = _PAGE_RANGE.match(pages)
241 if not match:
242 return pages
244 start, end = match.group(1), match.group(2)
245 if len(end) < len(start):
246 end = start[:len(start) - len(end)] + end
247 return f'{start}--{end}'
250def _ascii_fold(text: str) -> str:
251 """Reduce accented Latin characters to ASCII (Muller from Müller)."""
252 decomposed = unicodedata.normalize('NFKD', text)
253 return ''.join(char for char in decomposed if not unicodedata.combining(char))
256def sanitize_key(text: str) -> str:
257 """Reduce a string to characters that are safe in a BibTeX citation key.
259 Keys reach both LaTeX and, in some workflows, the filesystem, and their
260 source is a publisher-controlled metadata record. Restricting them to
261 ASCII letters and digits keeps `O'Brien` and `Müller` from producing an
262 entry that LaTeX rejects.
263 """
264 return re.sub(r'[^A-Za-z0-9]', '', _ascii_fold(text or ''))
267def _looks_like_initials(word: str) -> bool:
268 """True for `J`, `JA`, `J.A.` -- the initials half of a PubMed author."""
269 stripped = word.replace('.', '')
270 return bool(stripped) and len(stripped) <= 3 and stripped.isupper()
273def citation_key(authors: str, year: str, title: str) -> str:
274 """Build the citation key used by every producer in this skill.
276 ``<FirstAuthorSurname><Year><FirstSubstantiveTitleWord>``, ASCII-folded and
277 stripped of anything outside ``[A-Za-z0-9]``. Sharing one scheme across
278 Crossref, PubMed, OpenAlex, and Scholar output is what makes the
279 deduplication in ``format_bibtex.py`` and ``validate_citations.py`` able to
280 see the same paper twice.
281 """
282 surname = ''
283 if authors:
284 first = authors.split(' and ')[0].strip()
285 if ',' in first:
286 surname = first.split(',')[0]
287 else:
288 words = first.split()
289 if not words:
290 surname = ''
291 elif len(words) > 1 and _looks_like_initials(words[-1]):
292 # PubMed's comma-less form puts the surname first: `Smith JA`.
293 surname = words[0]
294 else:
295 # Western given-name-first order: `John Smith`.
296 surname = words[-1]
298 surname = sanitize_key(surname) or 'unknown'
300 digits = re.sub(r'[^0-9]', '', str(year or ''))
301 year_part = digits[:4] if digits else 'XXXX'
303 words = re.findall(r'[A-Za-z]{4,}', _ascii_fold(title or ''))
304 keyword = words[0].lower() if words else 'paper'
306 return f'{surname}{year_part}{keyword}'
309# Terms whose capitalisation BibTeX would otherwise flatten in a title.
310PROTECTED_TERMS = [
311 'AlphaFold', 'CRISPR', 'COVID', 'SARS', 'DNA', 'RNA', 'mRNA', 'HIV',
312 'AIDS', 'MRI', 'PCR', 'USA', 'UK', 'EU', 'GPU', 'CPU', 'AI', 'ML',
313]
316def protect_title(title: str) -> str:
317 """Brace terms whose capitalisation must survive BibTeX style processing.
319 Terms already inside braces are left alone, so running this twice is safe.
320 """
321 if not title:
322 return ''
324 for term in PROTECTED_TERMS:
325 # Skip occurrences already wrapped, so the function is idempotent.
326 title = re.sub(
327 rf'(?<!\{{)\b{re.escape(term)}\b(?!\}})',
328 f'{{{term}}}',
329 title,
330 )
331 return title