Skills › Scientific Agent Skills › citation-management › search_google_scholar.py
search_google_scholar.py
266 lines · skills/citation-management/scripts/search_google_scholar.py at 5f2bc03
1#!/usr/bin/env python3
2"""
3Google Scholar Search Tool
4Search Google Scholar and export results.
6Note: This script requires the 'scholarly' library.
7Install with: uv pip install scholarly
8"""
10import sys
11import argparse
12import json
13import time
14import random
15from typing import List, Dict, Optional
16from itertools import islice
18sys.path.insert(0, str(__import__("pathlib").Path(__file__).resolve().parent))
20from _common import ( # noqa: E402
21 citation_key,
22 protect_title,
23 render_entry,
24)
26try:
27 from scholarly import scholarly, ProxyGenerator
28 SCHOLARLY_AVAILABLE = True
29except ImportError:
30 SCHOLARLY_AVAILABLE = False
31 print('Warning: scholarly library not installed. Install with: uv pip install scholarly', file=sys.stderr)
33class GoogleScholarSearcher:
34 """Search Google Scholar using scholarly library."""
36 def __init__(self, use_proxy: bool = False):
37 """
38 Initialize searcher.
40 Args:
41 use_proxy: Use an optional scholarly proxy provider (does not authorize bypassing blocks)
42 """
43 if not SCHOLARLY_AVAILABLE:
44 raise ImportError('scholarly library required. Install with: uv pip install scholarly')
46 # Setup proxy if requested
47 if use_proxy:
48 try:
49 pg = ProxyGenerator()
50 pg.FreeProxies()
51 scholarly.use_proxy(pg)
52 print('Using free proxy', file=sys.stderr)
53 except Exception as e:
54 print(f'Warning: Could not setup proxy: {e}', file=sys.stderr)
56 def search(self, query: str, max_results: int = 50,
57 year_start: Optional[int] = None, year_end: Optional[int] = None,
58 sort_by: str = 'relevance') -> List[Dict]:
59 """
60 Search Google Scholar.
62 Args:
63 query: Search query
64 max_results: Maximum number of results
65 year_start: Start year filter
66 year_end: End year filter
67 sort_by: Sort order ('relevance' or 'citations')
69 Returns:
70 List of result dictionaries
71 """
72 if not SCHOLARLY_AVAILABLE:
73 print('Error: scholarly library not installed', file=sys.stderr)
74 return []
76 print(f'Searching Google Scholar: {query}', file=sys.stderr)
77 print(f'Max results: {max_results}', file=sys.stderr)
79 results = []
81 try:
82 # Perform search
83 search_query = scholarly.search_pubs(query, year_low=year_start, year_high=year_end)
85 for i, result in enumerate(islice(search_query, max_results)):
87 print(f'Retrieved {i+1}/{max_results}', file=sys.stderr)
89 # Extract metadata
90 metadata = {
91 'title': result.get('bib', {}).get('title', ''),
92 'authors': ' and '.join(result.get('bib', {}).get('author', [])),
93 'year': result.get('bib', {}).get('pub_year', ''),
94 'venue': result.get('bib', {}).get('venue', ''),
95 'abstract': result.get('bib', {}).get('abstract', ''),
96 'citations': result.get('num_citations', 0),
97 'url': result.get('pub_url', ''),
98 'eprint_url': result.get('eprint_url', ''),
99 }
101 # Filter by year
102 if year_start or year_end:
103 try:
104 pub_year = int(metadata['year']) if metadata['year'] else 0
105 if year_start and pub_year < year_start:
106 continue
107 if year_end and pub_year > year_end:
108 continue
109 except ValueError:
110 pass
112 results.append(metadata)
114 # Rate limiting to avoid blocking
115 time.sleep(random.uniform(2, 5))
117 except Exception as e:
118 print(f'Error during search: {e}', file=sys.stderr)
119 return [] # A blocked or failed iterator is not a complete search.
121 # Sort if requested
122 if sort_by == 'citations' and results:
123 results.sort(key=lambda x: x.get('citations', 0), reverse=True)
125 return results
127 def metadata_to_bibtex(self, metadata: Dict) -> str:
128 """Convert metadata to BibTeX format.
130 Scholar records carry no DOI and an unstructured venue string, so an
131 entry built from one is a starting point: run it through the metadata
132 enrichment pass before citing it.
133 """
134 authors = metadata.get('authors', '')
136 key = citation_key(authors, metadata.get('year', ''), metadata.get('title', ''))
138 # Determine entry type (guess based on venue)
139 venue = metadata.get('venue', '').lower()
140 if 'proceedings' in venue or 'conference' in venue or 'symposium' in venue:
141 entry_type = 'inproceedings'
142 venue_field = 'booktitle'
143 else:
144 entry_type = 'article'
145 venue_field = 'journal'
147 fields = {
148 'author': authors,
149 'title': protect_title(metadata.get('title', '')),
150 venue_field: metadata.get('venue', ''),
151 'year': metadata.get('year', ''),
152 'url': metadata.get('url', ''),
153 }
155 # The citation count deliberately does not go in the `.bib`: it changes
156 # every week, and baking it into a bibliography makes the file wrong
157 # the moment it is written. It stays in the JSON output instead.
159 return render_entry(entry_type, key, fields)
162def main():
163 """Command-line interface."""
164 parser = argparse.ArgumentParser(
165 description='Search Google Scholar (requires scholarly library)',
166 epilog='Example: python search_google_scholar.py "machine learning" --limit 50'
167 )
169 parser.add_argument(
170 'query',
171 help='Search query'
172 )
174 parser.add_argument(
175 '--limit',
176 type=int,
177 default=50,
178 help='Maximum number of results (default: 50)'
179 )
181 parser.add_argument(
182 '--year-start',
183 type=int,
184 help='Start year for filtering'
185 )
187 parser.add_argument(
188 '--year-end',
189 type=int,
190 help='End year for filtering'
191 )
193 parser.add_argument(
194 '--sort-by',
195 choices=['relevance', 'citations'],
196 default='relevance',
197 help='Order within retrieved results; citations is a local sort (default: relevance)'
198 )
200 parser.add_argument(
201 '--use-proxy',
202 action='store_true',
203 help='Use scholarly free-proxy support; respect upstream access restrictions'
204 )
206 parser.add_argument(
207 '-o', '--output',
208 help='Output file (default: stdout)'
209 )
211 parser.add_argument(
212 '--format',
213 choices=['json', 'bibtex'],
214 default='json',
215 help='Output format (default: json)'
216 )
218 args = parser.parse_args()
219 if args.limit < 1:
220 parser.error('--limit must be positive')
222 if not SCHOLARLY_AVAILABLE:
223 print('\nError: scholarly library not installed', file=sys.stderr)
224 print('Install with: uv pip install scholarly', file=sys.stderr)
225 print('\nAlternatively, use PubMed search for biomedical literature:', file=sys.stderr)
226 print(' python search_pubmed.py "your query"', file=sys.stderr)
227 sys.exit(1)
229 # Search
230 searcher = GoogleScholarSearcher(use_proxy=args.use_proxy)
231 results = searcher.search(
232 args.query,
233 max_results=args.limit,
234 year_start=args.year_start,
235 year_end=args.year_end,
236 sort_by=args.sort_by
237 )
239 if not results:
240 print('No results found', file=sys.stderr)
241 sys.exit(1)
243 # Format output
244 if args.format == 'json':
245 output = json.dumps({
246 'query': args.query,
247 'count': len(results),
248 'results': results
249 }, indent=2)
250 else: # bibtex
251 bibtex_entries = [searcher.metadata_to_bibtex(r) for r in results]
252 output = '\n\n'.join(bibtex_entries) + '\n'
254 # Write output
255 if args.output:
256 with open(args.output, 'w', encoding='utf-8') as f:
257 f.write(output)
258 print(f'Wrote {len(results)} results to {args.output}', file=sys.stderr)
259 else:
260 print(output)
262 print(f'\nRetrieved {len(results)} results', file=sys.stderr)
265if __name__ == '__main__':
266 main()