Skills › Scientific Agent Skills › citation-management › format_bibtex.py
format_bibtex.py
356 lines · skills/citation-management/scripts/format_bibtex.py at 5f2bc03
1#!/usr/bin/env python3
2"""
3BibTeX Formatter and Cleaner
4Format, clean, sort, and deduplicate BibTeX files.
6Writing is opt-in: pass --output to write elsewhere, or --in-place to
7overwrite the input. Neither flag prints the result to stdout and leaves the
8input untouched.
9"""
11from __future__ import annotations
13import argparse
14import re
15import sys
16from collections import OrderedDict
17from typing import Dict, List, Optional
19sys.path.insert(0, str(__import__("pathlib").Path(__file__).resolve().parent))
21from _common import ( # noqa: E402
22 FIELD_ORDER,
23 citation_key,
24 format_pages,
25 parse_bibtex_file,
26 render_entry,
27)
30class BibTeXFormatter:
31 """Format and clean BibTeX entries."""
33 def __init__(self):
34 # Standard field order for readability
35 self.field_order = list(FIELD_ORDER)
37 def parse_bibtex_file(self, filepath: str) -> List[Dict]:
38 """
39 Parse BibTeX file and extract entries.
41 Args:
42 filepath: Path to BibTeX file
44 Returns:
45 List of entry dictionaries
46 """
47 return parse_bibtex_file(filepath)
49 def format_entry(self, entry: Dict) -> str:
50 """
51 Format a single BibTeX entry.
53 Args:
54 entry: Entry dictionary
56 Returns:
57 Formatted BibTeX string
58 """
59 return render_entry(entry['type'], entry['key'], entry['fields'])
61 def fix_common_issues(self, entry: Dict) -> Dict:
62 """
63 Fix common formatting issues in entry.
65 Args:
66 entry: Entry dictionary
68 Returns:
69 Fixed entry dictionary
70 """
71 fixed = dict(entry)
72 fields = OrderedDict(entry['fields'])
74 # Normalise page ranges: strip `pp.`, use an en-dash range, and expand
75 # abbreviated end pages. Idempotent, unlike a blanket hyphen replace.
76 if 'pages' in fields:
77 fields['pages'] = format_pages(fields['pages'])
79 # Fix DOI (remove URL prefix if present)
80 if 'doi' in fields:
81 doi = fields['doi']
82 doi = doi.replace('https://doi.org/', '')
83 doi = doi.replace('http://doi.org/', '')
84 doi = doi.replace('https://dx.doi.org/', '')
85 doi = doi.replace('http://dx.doi.org/', '')
86 doi = doi.replace('doi:', '')
87 fields['doi'] = doi.strip()
89 # Fix author separators (semicolon or ampersand to 'and')
90 if 'author' in fields:
91 author = fields['author']
92 author = author.replace(';', ' and')
93 author = author.replace(' & ', ' and ')
94 # Clean up multiple 'and's
95 author = re.sub(r'\s+and\s+and\s+', ' and ', author)
96 author = re.sub(r'\s+', ' ', author).strip()
97 fields['author'] = author
99 fixed['fields'] = fields
100 return fixed
102 def deduplicate_entries(self, entries: List[Dict]) -> List[Dict]:
103 """
104 Remove duplicate entries based on DOI or citation key.
106 Args:
107 entries: List of entry dictionaries
109 Returns:
110 List of unique entries
111 """
112 seen_dois = set()
113 seen_keys = set()
114 unique_entries = []
116 for entry in entries:
117 doi = entry['fields'].get('doi', '').strip().lower()
118 key = entry['key']
120 # Check DOI first (more reliable)
121 if doi:
122 if doi in seen_dois:
123 print(f'Duplicate DOI found: {doi} (skipping {key})', file=sys.stderr)
124 continue
125 seen_dois.add(doi)
127 # Check citation key
128 if key in seen_keys:
129 print(f'Duplicate citation key found: {key} (skipping)', file=sys.stderr)
130 continue
131 seen_keys.add(key)
133 unique_entries.append(entry)
135 return unique_entries
137 def rekey_entries(self, entries: List[Dict]) -> List[Dict]:
138 """Rewrite citation keys to this skill's shared scheme.
140 Entries pulled from different sources arrive with different key
141 conventions, which hides duplicates. Collisions between genuinely
142 distinct papers get a letter suffix.
143 """
144 rekeyed = []
145 used = {}
147 for entry in entries:
148 fields = entry['fields']
149 base = citation_key(
150 fields.get('author', ''),
151 fields.get('year', ''),
152 fields.get('title', ''),
153 )
154 if base in used:
155 used[base] += 1
156 # 'a' is the original, so the second occurrence becomes 'b'.
157 new_key = f'{base}{chr(ord("a") + used[base])}'
158 else:
159 used[base] = 0
160 new_key = base
162 updated = dict(entry)
163 updated['key'] = new_key
164 rekeyed.append(updated)
166 return rekeyed
168 def sort_entries(self, entries: List[Dict], sort_by: str = 'key', descending: bool = False) -> List[Dict]:
169 """
170 Sort entries by specified field.
172 Args:
173 entries: List of entry dictionaries
174 sort_by: Field to sort by ('key', 'year', 'author', 'title')
175 descending: Sort in descending order
177 Returns:
178 Sorted list of entries
179 """
180 def get_sort_key(entry: Dict) -> str:
181 if sort_by == 'year':
182 # Zero-pad so string comparison orders numerically, and push
183 # undated entries to the end.
184 year = entry['fields'].get('year', '')
185 digits = re.sub(r'[^0-9]', '', year)
186 return digits.zfill(4) if digits else '9999'
187 if sort_by == 'author':
188 author = entry['fields'].get('author', '')
189 if not author:
190 return 'zzz'
191 first = author.split(' and ')[0]
192 if ',' in first:
193 return first.split(',')[0].strip().lower()
194 words = first.split()
195 return words[-1].lower() if words else 'zzz'
196 if sort_by == 'title':
197 return entry['fields'].get('title', '').lower()
198 return entry['key'].lower()
200 return sorted(entries, key=get_sort_key, reverse=descending)
202 def format_file(self, filepath: str, output: Optional[str] = None,
203 deduplicate: bool = False, sort_by: Optional[str] = None,
204 descending: bool = False, fix_issues: bool = True,
205 rekey: bool = False) -> Optional[str]:
206 """
207 Format entire BibTeX file.
209 Args:
210 filepath: Input BibTeX file
211 output: Output file, or None to return the formatted text
212 deduplicate: Remove duplicates
213 sort_by: Field to sort by
214 descending: Sort in descending order
215 fix_issues: Fix common formatting issues
216 rekey: Rewrite citation keys to the shared scheme
218 Returns:
219 The formatted BibTeX text, or None when nothing was parsed.
220 """
221 print(f'Parsing {filepath}...', file=sys.stderr)
222 entries = self.parse_bibtex_file(filepath)
224 if not entries:
225 print('No entries found', file=sys.stderr)
226 return None
228 print(f'Found {len(entries)} entries', file=sys.stderr)
230 # Fix common issues
231 if fix_issues:
232 print('Fixing common issues...', file=sys.stderr)
233 entries = [self.fix_common_issues(e) for e in entries]
235 # Rekey before deduplicating, so cross-source duplicates collapse.
236 if rekey:
237 print('Rewriting citation keys...', file=sys.stderr)
238 entries = self.rekey_entries(entries)
240 # Deduplicate
241 if deduplicate:
242 print('Removing duplicates...', file=sys.stderr)
243 original_count = len(entries)
244 entries = self.deduplicate_entries(entries)
245 removed = original_count - len(entries)
246 if removed > 0:
247 print(f'Removed {removed} duplicate(s)', file=sys.stderr)
249 # Sort
250 if sort_by:
251 print(f'Sorting by {sort_by}...', file=sys.stderr)
252 entries = self.sort_entries(entries, sort_by, descending)
254 # Format entries
255 print('Formatting entries...', file=sys.stderr)
256 formatted_entries = [self.format_entry(e) for e in entries]
257 output_content = '\n\n'.join(formatted_entries) + '\n'
259 if output is None:
260 return output_content
262 try:
263 with open(output, 'w', encoding='utf-8') as f:
264 f.write(output_content)
265 print(f'Successfully wrote {len(entries)} entries to {output}', file=sys.stderr)
266 except OSError as e:
267 print(f'Error writing file: {e}', file=sys.stderr)
268 sys.exit(1)
270 return output_content
273def main():
274 """Command-line interface."""
275 parser = argparse.ArgumentParser(
276 description='Format, clean, sort, and deduplicate BibTeX files',
277 epilog='Example: python format_bibtex.py references.bib -o clean.bib --deduplicate --sort year'
278 )
280 parser.add_argument(
281 'file',
282 help='BibTeX file to format'
283 )
285 parser.add_argument(
286 '-o', '--output',
287 help='Write the result to this file'
288 )
290 parser.add_argument(
291 '--in-place',
292 action='store_true',
293 help='Overwrite the input file (mutually exclusive with --output)'
294 )
296 parser.add_argument(
297 '--deduplicate', '--remove-duplicates',
298 dest='deduplicate',
299 action='store_true',
300 help='Remove duplicate entries'
301 )
303 parser.add_argument(
304 '--rekey',
305 action='store_true',
306 help="Rewrite citation keys to this skill's shared scheme before deduplicating"
307 )
309 parser.add_argument(
310 '--sort',
311 choices=['key', 'year', 'author', 'title'],
312 help='Sort entries by field'
313 )
315 parser.add_argument(
316 '--descending',
317 action='store_true',
318 help='Sort in descending order'
319 )
321 parser.add_argument(
322 '--no-fix',
323 action='store_true',
324 help='Do not fix common issues'
325 )
327 args = parser.parse_args()
329 if args.output and args.in_place:
330 parser.error('--output and --in-place are mutually exclusive')
332 destination = args.file if args.in_place else args.output
334 # Format file
335 formatter = BibTeXFormatter()
336 result = formatter.format_file(
337 args.file,
338 output=destination,
339 deduplicate=args.deduplicate,
340 sort_by=args.sort,
341 descending=args.descending,
342 fix_issues=not args.no_fix,
343 rekey=args.rekey,
344 )
346 if result is None:
347 sys.exit(1)
349 # Without a destination the input is left alone and the result goes to
350 # stdout, so a mistaken invocation cannot destroy a bibliography.
351 if destination is None:
352 print(result, end='')
355if __name__ == '__main__':
356 main()