Skills › Financial Services › ib-check-deck › extract_numbers.py
extract_numbers.py
305 lines · plugins/vertical-plugins/financial-analysis/skills/ib-check-deck/scripts/extract_numbers.py at 6056506
1#!/usr/bin/env python3
2"""
3Extract numerical values from presentation content for consistency checking.
5Usage:
6 python extract_numbers.py presentation-content.md
7 python extract_numbers.py presentation-content.md --output numbers.json
9This script parses markdown-formatted presentation content (from markitdown)
10and extracts all numerical values with their context and slide references.
11"""
13import argparse
14import json
15import re
16import sys
17from collections import defaultdict
18from dataclasses import dataclass, asdict
19from pathlib import Path
20from typing import Optional
23@dataclass
24class NumberInstance:
25 """A numerical value found in the presentation."""
26 value: str # Original string representation
27 normalized: float # Normalized numeric value
28 unit: str # Detected unit (M, B, K, %, bps, x, etc.)
29 slide: int # Slide number (0 if unknown)
30 context: str # Surrounding text for context
31 line_number: int # Line number in source file
32 category: str # Detected category (revenue, margin, multiple, etc.)
35def normalize_number(value_str: str, unit: str) -> float:
36 """Convert a number string with unit to a normalized float value."""
37 # Remove commas and spaces
38 clean = re.sub(r'[,\s]', '', value_str)
40 try:
41 base_value = float(clean)
42 except ValueError:
43 return 0.0
45 # Apply unit multipliers
46 multipliers = {
47 'T': 1e12,
48 'B': 1e9,
49 'bn': 1e9,
50 'billion': 1e9,
51 'M': 1e6,
52 'mm': 1e6,
53 'mn': 1e6,
54 'million': 1e6,
55 'K': 1e3,
56 'k': 1e3,
57 'thousand': 1e3,
58 }
60 for unit_key in sorted(multipliers.keys(), key=len, reverse=True):
61 if unit_key.lower() in unit.lower():
62 return base_value * multipliers[unit_key]
64 return base_value
67def detect_category(context: str, unit: str) -> str:
68 """Detect the category of a number based on context and unit."""
69 context_lower = context.lower()
71 # Revenue-related
72 if any(term in context_lower for term in ['revenue', 'sales', 'top line', 'topline']):
73 return 'revenue'
75 # EBITDA-related
76 if 'ebitda' in context_lower:
77 if any(term in context_lower for term in ['margin', '%', 'percent']):
78 return 'ebitda_margin'
79 return 'ebitda'
81 # Margin-related
82 if any(term in context_lower for term in ['margin', 'profit']):
83 return 'margin'
85 # Growth-related
86 if any(term in context_lower for term in ['growth', 'cagr', 'yoy', 'y/y']):
87 return 'growth'
89 # Valuation multiples
90 if any(term in context_lower for term in ['multiple', 'ev/', 'p/e', 'ev/ebitda', 'ev/revenue']):
91 return 'multiple'
93 # Enterprise value / market cap
94 if any(term in context_lower for term in ['enterprise value', 'ev ', 'market cap']):
95 return 'valuation'
97 # Percentage (generic)
98 if unit in ['%', 'bps', 'percent']:
99 return 'percentage'
101 # Multiple indicator
102 if unit == 'x':
103 return 'multiple'
105 return 'other'
108def extract_numbers(content: str) -> list[NumberInstance]:
109 """Extract all numbers from presentation content."""
110 numbers = []
111 current_slide = 0
113 # Pattern for slide markers (from markitdown format)
114 slide_pattern = re.compile(r'^#+\s*Slide\s*(\d+)|^<!-- Slide (\d+)')
116 # Pattern for numbers with various formats
117 # Matches: $500M, 500M, $500 million, 25%, 25.5%, 2.5x, 150bps, $1,234.56, etc.
118 number_pattern = re.compile(
119 r'(?P<currency>[$€£¥])?' # Optional currency symbol
120 r'(?P<number>[\d,]+(?:\.\d+)?)' # The number itself
121 r'\s*'
122 r'(?P<unit>%|bps|x|' # Common units
123 r'[Tt]rillion|[Bb]illion|[Mm]illion|[Tt]housand|' # Full words
124 r'[TBMKtbmk]n?|mm|MM)?' # Abbreviations
125 r'(?!\d)' # Negative lookahead to avoid partial matches
126 )
128 lines = content.split('\n')
130 for line_num, line in enumerate(lines, 1):
131 # Check for slide marker
132 slide_match = slide_pattern.match(line)
133 if slide_match:
134 current_slide = int(slide_match.group(1) or slide_match.group(2))
135 continue
137 # Find all numbers in the line
138 for match in number_pattern.finditer(line):
139 value_str = match.group('number')
140 currency = match.group('currency') or ''
141 unit = match.group('unit') or ''
143 # Skip very short numbers without context (likely not financial)
144 if len(value_str.replace(',', '').replace('.', '')) < 2 and not unit:
145 continue
147 # Skip year-like numbers (1900-2099) unless they have units
148 try:
149 num_val = float(value_str.replace(',', ''))
150 if 1900 <= num_val <= 2099 and not unit and not currency:
151 continue
152 except ValueError:
153 pass
155 # Build full value string
156 full_value = f"{currency}{value_str}{unit}"
158 # Get context (surrounding words)
159 start = max(0, match.start() - 50)
160 end = min(len(line), match.end() + 50)
161 context = line[start:end].strip()
163 # Normalize unit
164 if currency:
165 if not unit:
166 unit = 'USD' # Assume USD for $ without unit
167 else:
168 unit = f"USD_{unit}"
170 normalized = normalize_number(value_str, unit)
171 category = detect_category(context, unit)
173 numbers.append(NumberInstance(
174 value=full_value,
175 normalized=normalized,
176 unit=unit or 'none',
177 slide=current_slide,
178 context=context,
179 line_number=line_num,
180 category=category
181 ))
183 return numbers
186def find_inconsistencies(numbers: list[NumberInstance]) -> list[dict]:
187 """Find potential inconsistencies in extracted numbers."""
188 inconsistencies = []
190 # Group numbers by category
191 by_category = defaultdict(list)
192 for num in numbers:
193 if num.category != 'other':
194 by_category[num.category].append(num)
196 # Check each category for mismatches
197 for category, instances in by_category.items():
198 if len(instances) < 2:
199 continue
201 # Group by approximate value (within 5% tolerance)
202 value_groups = []
203 for inst in instances:
204 placed = False
205 for group in value_groups:
206 ref_value = group[0].normalized
207 if ref_value > 0:
208 diff_pct = abs(inst.normalized - ref_value) / ref_value
209 if diff_pct < 0.05: # 5% tolerance
210 group.append(inst)
211 placed = True
212 break
213 if not placed:
214 value_groups.append([inst])
216 # If we have multiple groups, there might be inconsistencies
217 if len(value_groups) > 1:
218 # Sort groups by size (largest first)
219 value_groups.sort(key=len, reverse=True)
221 # The largest group is likely "correct", others are potential issues
222 main_group = value_groups[0]
223 for other_group in value_groups[1:]:
224 inconsistencies.append({
225 'category': category,
226 'expected': {
227 'value': main_group[0].value,
228 'slides': sorted(set(n.slide for n in main_group)),
229 'count': len(main_group)
230 },
231 'found': {
232 'value': other_group[0].value,
233 'slides': sorted(set(n.slide for n in other_group)),
234 'count': len(other_group)
235 },
236 'severity': 'high' if category in ['revenue', 'ebitda', 'valuation'] else 'medium'
237 })
239 return inconsistencies
242def main():
243 parser = argparse.ArgumentParser(
244 description='Extract numbers from presentation content for consistency checking'
245 )
246 parser.add_argument('input_file', help='Markdown file with presentation content')
247 parser.add_argument('--output', '-o', help='Output JSON file (default: stdout)')
248 parser.add_argument('--check', '-c', action='store_true',
249 help='Check for inconsistencies and report')
251 args = parser.parse_args()
253 # Read input
254 input_path = Path(args.input_file)
255 if not input_path.exists():
256 print(f"Error: File not found: {args.input_file}", file=sys.stderr)
257 sys.exit(1)
259 content = input_path.read_text()
261 # Extract numbers
262 numbers = extract_numbers(content)
264 # Prepare output
265 output = {
266 'total_numbers': len(numbers),
267 'by_category': defaultdict(list),
268 'numbers': [asdict(n) for n in numbers]
269 }
271 for num in numbers:
272 output['by_category'][num.category].append({
273 'value': num.value,
274 'slide': num.slide,
275 'context': num.context[:100]
276 })
278 output['by_category'] = dict(output['by_category'])
280 # Check for inconsistencies if requested
281 if args.check:
282 inconsistencies = find_inconsistencies(numbers)
283 output['inconsistencies'] = inconsistencies
285 if inconsistencies:
286 print("\n=== POTENTIAL INCONSISTENCIES DETECTED ===\n", file=sys.stderr)
287 for inc in inconsistencies:
288 print(f"Category: {inc['category'].upper()}", file=sys.stderr)
289 print(f" Expected: {inc['expected']['value']} (Slides: {inc['expected']['slides']}, Count: {inc['expected']['count']})", file=sys.stderr)
290 print(f" Found: {inc['found']['value']} (Slides: {inc['found']['slides']}, Count: {inc['found']['count']})", file=sys.stderr)
291 print(f" Severity: {inc['severity']}", file=sys.stderr)
292 print(file=sys.stderr)
294 # Output results
295 json_output = json.dumps(output, indent=2)
297 if args.output:
298 Path(args.output).write_text(json_output)
299 print(f"Output written to {args.output}", file=sys.stderr)
300 else:
301 print(json_output)
304if __name__ == '__main__':
305 main()