Skills › Scientific Agent Skills › citation-management › extract_metadata.py
extract_metadata.py
744 lines · skills/citation-management/scripts/extract_metadata.py at 5f2bc03
1#!/usr/bin/env python3
2"""
3Metadata Extraction Tool
4Extract citation metadata from DOI, PMID, arXiv ID, or URL using various APIs.
5"""
7import sys
8import os
9import requests
10import argparse
11import time
12import re
13import json
14import xml.etree.ElementTree as ET
15from typing import Optional, Dict, List, Tuple
16from urllib.parse import urlparse, quote, unquote
18sys.path.insert(0, str(__import__("pathlib").Path(__file__).resolve().parent))
20from _common import ( # noqa: E402
21 citation_key,
22 format_pages,
23 protect_title,
24 render_entry,
25)
27class MetadataExtractor:
28 """Extract metadata from various sources and generate BibTeX."""
30 def __init__(self, email: Optional[str] = None):
31 """
32 Initialize extractor.
34 Args:
35 email: Email for Entrez API (recommended for PubMed)
36 """
37 self.session = requests.Session()
38 self.session.headers.update({
39 'User-Agent': 'MetadataExtractor/1.0 (Citation Management Tool)'
40 })
41 self.email = email or os.getenv('NCBI_EMAIL', '')
42 self._last_arxiv_request = None
44 def identify_type(self, identifier: str) -> Tuple[str, str]:
45 """
46 Identify the type of identifier.
48 Args:
49 identifier: DOI, PMID, arXiv ID, or URL
51 Returns:
52 Tuple of (type, cleaned_identifier)
53 """
54 identifier = identifier.strip()
56 # Check if URL
57 if identifier.startswith('http://') or identifier.startswith('https://'):
58 return self._parse_url(identifier)
60 # Check for DOI
61 identifier = re.sub(r'^doi:\s*', '', identifier, flags=re.IGNORECASE)
62 if identifier.startswith('10.'):
63 return ('doi', identifier)
65 # Check for arXiv ID
66 if re.match(r'^\d{4}\.\d{4,5}(v\d+)?$', identifier):
67 return ('arxiv', identifier)
68 if identifier.startswith('arXiv:'):
69 return ('arxiv', identifier.replace('arXiv:', ''))
70 if re.fullmatch(r'[a-z-]+(?:\.[A-Z]{2})?/\d{7}(?:v\d+)?', identifier):
71 return ('arxiv', identifier)
73 # Check for PMID (8-digit number typically)
74 if identifier.isdigit() and int(identifier) > 0:
75 return ('pmid', identifier)
77 # Check for PMCID
78 if identifier.upper().startswith('PMC') and identifier[3:].isdigit():
79 return ('pmcid', identifier.upper())
81 return ('unknown', identifier)
83 def _parse_url(self, url: str) -> Tuple[str, str]:
84 """Parse URL to extract identifier type and value."""
85 parsed = urlparse(url)
87 # DOI URLs
88 if parsed.hostname in ('doi.org', 'dx.doi.org'):
89 doi = unquote(parsed.path.lstrip('/'))
90 return ('doi', doi)
92 # PubMed URLs
93 if parsed.hostname == 'pubmed.ncbi.nlm.nih.gov' or (
94 parsed.hostname == 'www.ncbi.nlm.nih.gov' and parsed.path.startswith('/pubmed/')
95 ):
96 pmid = re.search(r'/(\d+)', parsed.path)
97 if pmid:
98 return ('pmid', pmid.group(1))
100 # arXiv URLs
101 if parsed.hostname in ('arxiv.org', 'www.arxiv.org', 'export.arxiv.org'):
102 arxiv_path = re.sub(r'^/(?:abs|pdf)/', '', parsed.path)
103 arxiv_path = re.sub(r'\.pdf$', '', arxiv_path)
104 kind, value = self.identify_type(arxiv_path)
105 if kind == 'arxiv':
106 return (kind, value)
108 if parsed.hostname in ('pmc.ncbi.nlm.nih.gov', 'www.ncbi.nlm.nih.gov'):
109 pmcid = re.search(r'/articles/(PMC\d+)', parsed.path, re.IGNORECASE)
110 if pmcid:
111 return ('pmcid', pmcid.group(1).upper())
113 # Nature, Science, Cell, etc. - try to extract DOI from URL
114 doi_match = re.search(r'10\.\d{4,}/[^\s]+', unquote(parsed.path))
115 if doi_match:
116 return ('doi', doi_match.group())
118 return ('url', url)
120 def extract_from_doi(self, doi: str) -> Optional[Dict]:
121 """
122 Extract metadata from DOI using CrossRef API.
124 Args:
125 doi: Digital Object Identifier
127 Returns:
128 Metadata dictionary or None
129 """
130 # A DOI is publisher-controlled text and may legitimately contain
131 # `#`, `?`, `<` or `>`, so it cannot be interpolated into a URL raw.
132 url = f'https://api.crossref.org/works/{quote(doi, safe="")}'
134 try:
135 response = self.session.get(url, timeout=15)
137 if response.status_code == 200:
138 data = response.json()
139 message = data.get('message', {})
141 container = message.get('container-title') or ['']
142 isbns = message.get('ISBN') or []
143 issns = message.get('ISSN') or []
145 metadata = {
146 'type': 'doi',
147 'entry_type': self._crossref_type_to_bibtex(message.get('type')),
148 'doi': doi,
149 'title': (message.get('title') or [''])[0],
150 'authors': self._format_authors_crossref(message.get('author', [])),
151 'editors': self._format_authors_crossref(message.get('editor', [])),
152 'year': self._extract_year_crossref(message),
153 'journal': container[0] if container else '',
154 'volume': str(message.get('volume', '')) if message.get('volume') else '',
155 'issue': str(message.get('issue', '')) if message.get('issue') else '',
156 'pages': message.get('page', ''),
157 'publisher': message.get('publisher', ''),
158 'isbn': isbns[0] if isbns else '',
159 'issn': issns[0] if issns else '',
160 'url': f'https://doi.org/{doi}'
161 }
163 return metadata
164 else:
165 print(f'Error: CrossRef API returned status {response.status_code} for DOI: {doi}', file=sys.stderr)
166 return None
168 except Exception as e:
169 print(f'Error extracting metadata from DOI {doi}: {e}', file=sys.stderr)
170 return None
172 def extract_from_pmid(self, pmid: str) -> Optional[Dict]:
173 """
174 Extract metadata from PMID using PubMed E-utilities.
176 Args:
177 pmid: PubMed ID
179 Returns:
180 Metadata dictionary or None
181 """
182 url = f'https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi'
183 params = {
184 'db': 'pubmed',
185 'id': pmid,
186 'retmode': 'xml',
187 'tool': 'citation-management',
188 }
190 if self.email:
191 params['email'] = self.email
193 api_key = os.getenv('NCBI_API_KEY')
194 if api_key:
195 params['api_key'] = api_key
197 try:
198 response = self.session.get(url, params=params, timeout=15)
200 if response.status_code == 200:
201 root = ET.fromstring(response.content)
202 article = root.find('.//PubmedArticle')
204 if article is None:
205 print(f'Error: No article found for PMID: {pmid}', file=sys.stderr)
206 return None
208 # Extract metadata from XML
209 medline_citation = article.find('.//MedlineCitation')
210 article_elem = medline_citation.find('.//Article')
211 journal = article_elem.find('.//Journal')
213 # Get DOI if available
214 doi = None
215 article_ids = article.findall('.//ArticleId')
216 for article_id in article_ids:
217 if article_id.get('IdType') == 'doi':
218 doi = article_id.text
219 break
221 metadata = {
222 'type': 'pmid',
223 'entry_type': 'article',
224 'pmid': pmid,
225 'title': self._pubmed_title(article_elem.find('.//ArticleTitle')),
226 'authors': self._format_authors_pubmed(article_elem.findall('.//Author')),
227 'year': self._extract_year_pubmed(article_elem),
228 'journal': journal.findtext('.//Title', ''),
229 'volume': journal.findtext('.//JournalIssue/Volume', ''),
230 'issue': journal.findtext('.//JournalIssue/Issue', ''),
231 'pages': article_elem.findtext('.//Pagination/MedlinePgn', ''),
232 'doi': doi
233 }
235 return metadata
236 else:
237 print(f'Error: PubMed API returned status {response.status_code} for PMID: {pmid}', file=sys.stderr)
238 return None
240 except Exception as e:
241 print(f'Error extracting metadata from PMID {pmid} ({type(e).__name__}); request details omitted', file=sys.stderr)
242 return None
244 @staticmethod
245 def _pubmed_title(element) -> str:
246 """Read a PubMed ArticleTitle as a citation title.
248 `itertext`, not `findtext`, because titles carry inline markup such as
249 <i> and <sup> and findtext returns only the leading run. PubMed also
250 terminates titles with a full stop that is not part of the title; a
251 trailing `?` or `!` is, so only the period is dropped.
252 """
253 if element is None:
254 return ''
255 text = ' '.join(''.join(element.itertext()).split())
256 return text[:-1] if text.endswith('.') else text
258 def extract_from_arxiv(self, arxiv_id: str) -> Optional[Dict]:
259 """
260 Extract metadata from arXiv ID using arXiv API.
262 Args:
263 arxiv_id: arXiv identifier
265 Returns:
266 Metadata dictionary or None
267 """
268 url = 'https://export.arxiv.org/api/query'
269 params = {
270 'id_list': arxiv_id,
271 'max_results': 1
272 }
274 try:
275 # arXiv requests must be at least three seconds apart, including
276 # callers using this class directly rather than the batch CLI.
277 if self._last_arxiv_request is not None:
278 time.sleep(max(0, 3.0 - (time.monotonic() - self._last_arxiv_request)))
279 self._last_arxiv_request = time.monotonic()
280 response = self.session.get(url, params=params, timeout=15)
282 if response.status_code == 200:
283 # Parse Atom XML
284 root = ET.fromstring(response.content)
285 ns = {'atom': 'http://www.w3.org/2005/Atom', 'arxiv': 'http://arxiv.org/schemas/atom'}
287 entry = root.find('atom:entry', ns)
288 if entry is None:
289 print(f'Error: No entry found for arXiv ID: {arxiv_id}', file=sys.stderr)
290 return None
292 if '/api/errors' in entry.findtext('atom:id', '', ns):
293 print(f'Error: arXiv returned an error entry for {arxiv_id}', file=sys.stderr)
294 return None
296 # Extract DOI if published
297 doi_elem = entry.find('arxiv:doi', ns)
298 doi = doi_elem.text if doi_elem is not None else None
300 # Extract journal reference if published
301 journal_ref_elem = entry.find('arxiv:journal_ref', ns)
302 journal_ref = journal_ref_elem.text if journal_ref_elem is not None else None
304 # Get publication date
305 published = entry.findtext('atom:published', '', ns)
306 year = published[:4] if published else ''
308 # Get authors
309 authors = []
310 for author in entry.findall('atom:author', ns):
311 name = author.findtext('atom:name', '', ns)
312 if name:
313 authors.append(name)
315 # Only call it an @article when there is a journal to name.
316 # A DOI alone is not enough -- arXiv mints DataCite DOIs for
317 # unpublished preprints -- and an @article without a `journal`
318 # fails the required-field check in validate_citations.py.
319 journal = self._journal_from_arxiv_ref(journal_ref)
321 metadata = {
322 'type': 'arxiv',
323 'entry_type': 'article' if journal else 'misc',
324 'arxiv_id': arxiv_id,
325 'title': entry.findtext('atom:title', '', ns).strip().replace('\n', ' '),
326 'authors': ' and '.join(authors),
327 'year': year,
328 'doi': doi,
329 'journal': journal,
330 'journal_ref': journal_ref,
331 'abstract': entry.findtext('atom:summary', '', ns).strip().replace('\n', ' '),
332 'url': f'https://arxiv.org/abs/{arxiv_id}'
333 }
335 return metadata
336 else:
337 print(f'Error: arXiv API returned status {response.status_code} for ID: {arxiv_id}', file=sys.stderr)
338 return None
340 except Exception as e:
341 print(f'Error extracting metadata from arXiv {arxiv_id}: {e}', file=sys.stderr)
342 return None
344 def metadata_to_bibtex(self, metadata: Dict, citation_key: Optional[str] = None) -> str:
345 """
346 Convert metadata dictionary to BibTeX format.
348 Args:
349 metadata: Metadata dictionary
350 citation_key: Optional custom citation key
352 Returns:
353 BibTeX string
354 """
355 if not citation_key:
356 citation_key = self._generate_citation_key(metadata)
358 entry_type = metadata.get('entry_type', 'misc')
360 fields = {
361 'author': metadata.get('authors', ''),
362 'editor': metadata.get('editors', ''),
363 'title': protect_title(metadata.get('title', '')),
364 'year': metadata.get('year', ''),
365 'volume': metadata.get('volume', ''),
366 'number': metadata.get('issue', ''),
367 'pages': format_pages(metadata.get('pages')),
368 'doi': metadata.get('doi') or '',
369 'isbn': metadata.get('isbn', ''),
370 'issn': metadata.get('issn', ''),
371 }
373 # The venue field's name depends on the entry type, and a book or
374 # report needs its publisher/institution or it fails validation.
375 if entry_type in ('inproceedings', 'incollection'):
376 fields['booktitle'] = metadata.get('journal', '')
377 elif entry_type != 'misc':
378 fields['journal'] = metadata.get('journal', '')
380 if entry_type in ('book', 'incollection', 'inproceedings'):
381 fields['publisher'] = metadata.get('publisher', '')
382 elif entry_type == 'techreport':
383 fields['institution'] = metadata.get('publisher', '')
385 if entry_type == 'misc' and metadata.get('type') == 'arxiv':
386 arxiv_id = metadata.get('arxiv_id', '')
387 fields['howpublished'] = f'arXiv:{arxiv_id}' if arxiv_id else 'arXiv'
389 # A DOI is the stable locator; only fall back to a URL without one.
390 if not fields['doi'] and metadata.get('url'):
391 fields['url'] = metadata['url']
393 # BibTeX takes one `note` per entry, so build it from all the parts
394 # that would otherwise each claim the field.
395 notes = []
396 if metadata.get('pmid'):
397 notes.append(f'PMID: {metadata["pmid"]}')
398 if metadata.get('type') == 'arxiv' and entry_type == 'misc':
399 notes.append('Preprint')
400 if notes:
401 fields['note'] = '. '.join(notes)
403 return render_entry(entry_type, citation_key, fields)
405 @staticmethod
406 def _journal_from_arxiv_ref(journal_ref: Optional[str]) -> str:
407 """Pull a journal name out of arXiv's free-text `journal_ref`.
409 The field has no schema -- `Nature 596, 583-589 (2021)` is typical --
410 so take the leading run of non-numeric text and leave the rest for the
411 enrichment pass rather than guessing at volume and pages.
412 """
413 if not journal_ref:
414 return ''
415 head = re.split(r'[,;]|\s\d', journal_ref.strip(), maxsplit=1)[0]
416 return head.strip(' .')
418 def _crossref_type_to_bibtex(self, crossref_type: str) -> str:
419 """Map CrossRef type to BibTeX entry type."""
420 type_map = {
421 'journal-article': 'article',
422 'book': 'book',
423 'book-chapter': 'incollection',
424 'proceedings-article': 'inproceedings',
425 'posted-content': 'misc',
426 'dataset': 'misc',
427 'report': 'techreport'
428 }
429 return type_map.get(crossref_type, 'misc')
431 def _format_authors_crossref(self, authors: List[Dict]) -> str:
432 """Format author list from CrossRef data."""
433 if not authors:
434 return ''
436 formatted = []
437 for author in authors:
438 given = author.get('given', '')
439 family = author.get('family', '')
440 if family:
441 if given:
442 formatted.append(f'{family}, {given}')
443 else:
444 formatted.append(family)
446 return ' and '.join(formatted)
448 def _format_authors_pubmed(self, authors: List) -> str:
449 """Format author list from PubMed XML."""
450 formatted = []
451 for author in authors:
452 last_name = author.findtext('.//LastName', '')
453 fore_name = author.findtext('.//ForeName', '')
454 if last_name:
455 if fore_name:
456 formatted.append(f'{last_name}, {fore_name}')
457 else:
458 formatted.append(last_name)
460 return ' and '.join(formatted)
462 def _extract_year_crossref(self, message: Dict) -> str:
463 """Extract year from CrossRef message."""
464 # Try published-print first, then published-online
465 date_parts = message.get('published-print', {}).get('date-parts', [[]])
466 if not date_parts or not date_parts[0]:
467 date_parts = message.get('published-online', {}).get('date-parts', [[]])
468 if not date_parts or not date_parts[0]:
469 date_parts = message.get('published', {}).get('date-parts', [[]])
470 if not date_parts or not date_parts[0]:
471 date_parts = message.get('issued', {}).get('date-parts', [[]])
473 if date_parts and date_parts[0]:
474 return str(date_parts[0][0])
475 return ''
477 def _extract_year_pubmed(self, article: ET.Element) -> str:
478 """Extract year from PubMed XML."""
479 year = article.findtext('.//Journal/JournalIssue/PubDate/Year', '')
480 if not year:
481 medline_date = article.findtext('.//Journal/JournalIssue/PubDate/MedlineDate', '')
482 if medline_date:
483 year_match = re.search(r'\d{4}', medline_date)
484 if year_match:
485 year = year_match.group()
486 return year
488 def _generate_citation_key(self, metadata: Dict) -> str:
489 """Generate a citation key from metadata.
491 Shared with every other producer in this skill so that entries from
492 different sources collide when -- and only when -- they are the same
493 paper. Values arrive verbatim from a publisher-controlled record, and
494 the key reaches both LaTeX and, in some workflows, a file path, so the
495 shared helper restricts it to ASCII letters and digits.
496 """
497 return citation_key(
498 metadata.get('authors', ''),
499 metadata.get('year', ''),
500 metadata.get('title', ''),
501 )
503 def _protect_title(self, title: str) -> str:
504 """Protect capitalization in title for BibTeX."""
505 return protect_title(title)
507 def extract_from_pmcid(self, pmcid: str) -> Optional[Dict]:
508 """Resolve a PMCID to a PMID via the NCBI ID converter, then extract."""
509 url = 'https://pmc.ncbi.nlm.nih.gov/tools/idconv/api/v1/articles/'
510 params = {'ids': pmcid, 'format': 'json', 'tool': 'citation-management'}
511 if self.email:
512 params['email'] = self.email
514 try:
515 response = self.session.get(url, params=params, timeout=15)
516 if response.status_code != 200:
517 print(f'Error: ID converter returned status {response.status_code} for {pmcid}', file=sys.stderr)
518 return None
520 records = response.json().get('records', [])
521 if not records or not records[0].get('pmid'):
522 if records and records[0].get('doi'):
523 return self.extract_from_doi(records[0]['doi'])
524 print(f'Error: Could not resolve {pmcid} to a PMID or DOI', file=sys.stderr)
525 return None
527 return self.extract_from_pmid(str(records[0]['pmid']))
529 except (requests.exceptions.RequestException, ValueError) as e:
530 print(f'Error resolving PMCID {pmcid} ({type(e).__name__}); request details omitted', file=sys.stderr)
531 return None
533 def extract_from_url(self, url: str) -> Optional[Dict]:
534 """Extract via a DOI advertised in the page's citation metadata.
536 Publishers embed `citation_doi` / `DC.Identifier` meta tags on article
537 pages. Reading the DOI and handing off to Crossref is far more reliable
538 than scraping the page itself, and it fails honestly when absent.
539 """
540 try:
541 response = self.session.get(url, timeout=15)
542 if response.status_code != 200:
543 print(f'Error: URL returned status {response.status_code}: {url}', file=sys.stderr)
544 return None
546 head = response.text[:200000]
547 patterns = [
548 r'<meta[^>]+name=["\'](?:citation_doi|DC\.Identifier|dc\.identifier)["\'][^>]+content=["\']([^"\']+)["\']',
549 r'<meta[^>]+content=["\']([^"\']+)["\'][^>]+name=["\'](?:citation_doi|DC\.Identifier|dc\.identifier)["\']',
550 ]
551 for pattern in patterns:
552 match = re.search(pattern, head, re.IGNORECASE)
553 if match:
554 doi = match.group(1).strip()
555 doi = re.sub(r'^(?:https?://(?:dx\.)?doi\.org/|doi:)', '', doi, flags=re.IGNORECASE)
556 if doi.startswith('10.'):
557 print(f'Found DOI in page metadata: {doi}', file=sys.stderr)
558 return self.extract_from_doi(doi)
560 print(f'Error: No DOI found in page metadata for {url}', file=sys.stderr)
561 return None
563 except requests.exceptions.RequestException as e:
564 print(f'Error fetching {url}: {e}', file=sys.stderr)
565 return None
568 def extract(self, identifier: str) -> Optional[str]:
569 """
570 Extract metadata and return BibTeX.
572 Args:
573 identifier: DOI, PMID, arXiv ID, or URL
575 Returns:
576 BibTeX string or None
577 """
578 id_type, clean_id = self.identify_type(identifier)
580 print(f'Identified as {id_type}: {clean_id}', file=sys.stderr)
582 metadata = None
584 if id_type == 'doi':
585 metadata = self.extract_from_doi(clean_id)
586 elif id_type == 'pmid':
587 metadata = self.extract_from_pmid(clean_id)
588 elif id_type == 'arxiv':
589 metadata = self.extract_from_arxiv(clean_id)
590 elif id_type == 'pmcid':
591 metadata = self.extract_from_pmcid(clean_id)
592 elif id_type == 'url':
593 metadata = self.extract_from_url(clean_id)
594 else:
595 print(f'Error: Unknown identifier type: {identifier}', file=sys.stderr)
596 return None
598 if metadata:
599 return self.metadata_to_bibtex(metadata)
600 else:
601 return None
603 def extract_record(self, identifier: str) -> Optional[Dict]:
604 """Extract metadata and return it with its rendered BibTeX attached."""
605 id_type, clean_id = self.identify_type(identifier)
607 print(f'Identified as {id_type}: {clean_id}', file=sys.stderr)
609 handlers = {
610 'doi': self.extract_from_doi,
611 'pmid': self.extract_from_pmid,
612 'arxiv': self.extract_from_arxiv,
613 'pmcid': self.extract_from_pmcid,
614 'url': self.extract_from_url,
615 }
617 handler = handlers.get(id_type)
618 if handler is None:
619 print(f'Error: Unknown identifier type: {identifier}', file=sys.stderr)
620 return None
622 metadata = handler(clean_id)
623 if not metadata:
624 return None
626 record = dict(metadata)
627 record['citation_key'] = self._generate_citation_key(metadata)
628 record['bibtex'] = self.metadata_to_bibtex(metadata, record['citation_key'])
629 return record
632def read_identifiers(filepath: str) -> List[str]:
633 """Read newline identifiers or the JSON results emitted by search scripts."""
634 with open(filepath, encoding='utf-8') as handle:
635 content = handle.read()
636 if not content.lstrip().startswith(('{', '[')):
637 return [line.strip() for line in content.splitlines() if line.strip()]
638 payload = json.loads(content)
639 rows = payload.get('results', payload.get('entries')) if isinstance(payload, dict) else payload
640 if not isinstance(rows, list):
641 raise ValueError('JSON input must be an array or contain results/entries array')
642 identifiers = []
643 for index, row in enumerate(rows, start=1):
644 if isinstance(row, str):
645 identifier = row.strip()
646 elif isinstance(row, dict):
647 identifier = next((str(row[key]).strip() for key in
648 ('doi', 'pmid', 'pmcid', 'arxiv_id', 'url') if row.get(key)), '')
649 else:
650 identifier = ''
651 if not identifier:
652 raise ValueError(f'JSON record {index} has no DOI, PMID, PMCID, arXiv ID, or URL')
653 identifiers.append(identifier)
654 return identifiers
657def main():
658 """Command-line interface."""
659 parser = argparse.ArgumentParser(
660 description='Extract citation metadata from DOI, PMID, arXiv ID, or URL',
661 epilog='Example: python extract_metadata.py --doi 10.1038/s41586-021-03819-2'
662 )
664 parser.add_argument('--doi', action='append', help='Digital Object Identifier (repeatable)')
665 parser.add_argument('--pmid', action='append', help='PubMed ID (repeatable)')
666 parser.add_argument('--pmcid', action='append', help='PubMed Central ID (repeatable)')
667 parser.add_argument('--arxiv', action='append', help='arXiv ID (repeatable)')
668 parser.add_argument('--url', action='append', help='URL to article (repeatable)')
669 parser.add_argument('-i', '--input', help='Newline identifiers or search-results JSON file')
670 parser.add_argument('-o', '--output', help='Output file for BibTeX (default: stdout)')
671 parser.add_argument('--format', choices=['bibtex', 'json'], default='bibtex', help='Output format')
672 parser.add_argument('--email', help='Email for NCBI E-utilities (recommended)')
674 args = parser.parse_args()
676 # Collect identifiers
677 identifiers = []
678 if args.doi:
679 identifiers.extend(args.doi)
680 if args.pmid:
681 identifiers.extend(args.pmid)
682 if args.pmcid:
683 identifiers.extend(args.pmcid)
684 if args.arxiv:
685 identifiers.extend(args.arxiv)
686 if args.url:
687 identifiers.extend(args.url)
689 if args.input:
690 try:
691 identifiers.extend(read_identifiers(args.input))
692 except Exception as e:
693 print(f'Error reading input file: {e}', file=sys.stderr)
694 sys.exit(1)
696 if not identifiers:
697 parser.print_help()
698 sys.exit(1)
700 # Extract metadata
701 extractor = MetadataExtractor(email=args.email)
702 records = []
704 for i, identifier in enumerate(identifiers):
705 print(f'\nProcessing {i+1}/{len(identifiers)}...', file=sys.stderr)
706 record = extractor.extract_record(identifier)
707 if record:
708 records.append(record)
710 # Rate limiting
711 if i < len(identifiers) - 1:
712 time.sleep(0.5)
714 if not records:
715 print('Error: No successful extractions', file=sys.stderr)
716 sys.exit(1)
718 bibtex_entries = [record['bibtex'] for record in records]
720 # Format output
721 if args.format == 'bibtex':
722 output = '\n\n'.join(bibtex_entries) + '\n'
723 else: # json
724 output = json.dumps({
725 'count': len(records),
726 'entries': records
727 }, indent=2)
729 # Write output
730 if args.output:
731 with open(args.output, 'w', encoding='utf-8') as f:
732 f.write(output)
733 print(f"\nSuccessfully wrote {len(records)} entries to {args.output}", file=sys.stderr)
734 else:
735 print(output)
737 print(f'\nExtracted {len(bibtex_entries)}/{len(identifiers)} entries', file=sys.stderr)
738 if len(records) < len(identifiers):
739 print('Error: partial extraction; review failed identifiers before using output', file=sys.stderr)
740 sys.exit(2)
743if __name__ == '__main__':
744 main()