Skills › Scientific Agent Skills › market-research-reports › generate_report_scaffold.py
generate_report_scaffold.py
453 lines · skills/market-research-reports/scripts/generate_report_scaffold.py at 5f2bc03
1#!/usr/bin/env python3
2"""Generate a bounded local evidence-first market-report workspace."""
4from __future__ import annotations
6import argparse
7import csv
8import json
9import re
10from pathlib import Path
11from typing import Any
13from _common import (
14 ValidationError,
15 error_exit,
16 parse_currency,
17 parse_iso_date,
18 parse_year,
19 read_json,
20 require_identifier,
21 require_list,
22 require_object,
23 require_text,
24 write_json_report,
25)
26from forecast_sensitivity import MAX_HORIZON
28PERIOD_RE = re.compile(r"^\d{4}-\d{4}$")
29SOURCE_FIELDS = (
30 "source_id",
31 "title",
32 "publisher",
33 "url",
34 "source_type",
35 "publication_date",
36 "retrieval_date",
37 "geography",
38 "currency",
39 "base_year",
40 "price_basis",
41 "measure_type",
42 "unit",
43 "taxonomy",
44 "taxonomy_version",
45 "revision_status",
46 "method",
47 "sample",
48 "limitations",
49 "license_or_terms",
50 "archive_path",
51)
52CLAIM_FIELDS = (
53 "claim_id",
54 "claim_text",
55 "claim_type",
56 "source_ids",
57 "calculation_id",
58 "assumption_ids",
59 "location",
60 "as_of_date",
61 "geography",
62 "currency",
63 "base_year",
64 "price_basis",
65 "measure_type",
66 "unit",
67 "taxonomy",
68 "taxonomy_version",
69 "revision_status",
70 "confidence",
71 "notes",
72)
73COMPETITOR_FIELDS = (
74 "competitor_id",
75 "competitor_name",
76 "feature_id",
77 "feature_name",
78 "status",
79 "evidence_source_ids",
80 "as_of_date",
81 "geography",
82 "product_scope",
83 "notes",
84)
85CONSISTENCY_FIELDS = (
86 "record_id",
87 "comparison_group",
88 "value",
89 "unit",
90 "currency",
91 "base_year",
92 "price_basis",
93 "measure_type",
94 "geography",
95 "period",
96 "taxonomy",
97 "taxonomy_version",
98 "denominator_id",
99 "source_id",
100)
101ASSUMPTION_FIELDS = (
102 "assumption_id",
103 "description",
104 "downside_value",
105 "base_value",
106 "upside_value",
107 "unit",
108 "source_ids",
109 "rationale",
110)
113def _string_list(value: Any, context: str) -> list[str]:
114 items = require_list(value, context, minimum=1, maximum=100)
115 return [
116 require_text(item, f"{context}[{index}]", maximum=500)
117 for index, item in enumerate(items)
118 ]
121def _period(value: Any, context: str) -> str:
122 text = require_text(value, context, maximum=9)
123 if not PERIOD_RE.fullmatch(text):
124 raise ValidationError(f"{context} must be YYYY-YYYY")
125 start, end = (int(part) for part in text.split("-"))
126 if start > end:
127 raise ValidationError(f"{context} start year must not exceed end year")
128 parse_year(start, context)
129 parse_year(end, context)
130 return text
133def validate_manifest(payload: dict[str, Any]) -> dict[str, Any]:
134 required = (
135 "schema_version",
136 "report_id",
137 "title",
138 "subtitle",
139 "prepared_for",
140 "prepared_by",
141 "classification",
142 "market_definition",
143 "inclusions",
144 "exclusions",
145 "geography",
146 "currency",
147 "base_year",
148 "price_basis",
149 "taxonomy",
150 "taxonomy_version",
151 "historical_period",
152 "forecast_period",
153 "retrieval_cutoff",
154 )
155 missing = [field for field in required if field not in payload]
156 if missing:
157 raise ValidationError(f"manifest is missing: {', '.join(missing)}")
158 if require_text(payload["schema_version"], "schema_version", maximum=10) != "1.0":
159 raise ValidationError("schema_version must be '1.0'")
160 classification = require_text(
161 payload["classification"], "classification", maximum=20
162 )
163 if classification not in {"public", "internal", "confidential"}:
164 raise ValidationError(
165 "classification must be public, internal, or confidential"
166 )
167 price_basis = require_text(payload["price_basis"], "price_basis", maximum=20)
168 if price_basis not in {"nominal", "real", "current", "constant"}:
169 raise ValidationError(
170 "price_basis must be nominal, real, current, or constant"
171 )
172 historical = _period(payload["historical_period"], "historical_period")
173 forecast = _period(payload["forecast_period"], "forecast_period")
174 historical_end = int(historical.split("-")[1])
175 forecast_start, forecast_end = (int(year) for year in forecast.split("-"))
176 if forecast_start != historical_end + 1:
177 raise ValidationError(
178 "forecast_period must begin the year after historical_period ends; "
179 "include any bridge estimates explicitly in the historical input"
180 )
181 if forecast_end - historical_end > MAX_HORIZON:
182 raise ValidationError(f"forecast horizon must not exceed {MAX_HORIZON} years")
183 return {
184 "schema_version": "1.0",
185 "report_id": require_identifier(payload["report_id"], "report_id"),
186 "title": require_text(payload["title"], "title", maximum=200),
187 "subtitle": require_text(payload["subtitle"], "subtitle", maximum=300),
188 "prepared_for": require_text(
189 payload["prepared_for"], "prepared_for", maximum=200
190 ),
191 "prepared_by": require_text(
192 payload["prepared_by"], "prepared_by", maximum=200
193 ),
194 "classification": classification,
195 "market_definition": require_text(
196 payload["market_definition"], "market_definition", maximum=2_000
197 ),
198 "inclusions": _string_list(payload["inclusions"], "inclusions"),
199 "exclusions": _string_list(payload["exclusions"], "exclusions"),
200 "geography": require_text(payload["geography"], "geography", maximum=200),
201 "currency": parse_currency(payload["currency"], "currency"),
202 "base_year": parse_year(payload["base_year"], "base_year"),
203 "price_basis": price_basis,
204 "taxonomy": require_text(payload["taxonomy"], "taxonomy", maximum=100),
205 "taxonomy_version": require_text(
206 payload["taxonomy_version"], "taxonomy_version", maximum=100
207 ),
208 "historical_period": historical,
209 "forecast_period": forecast,
210 "retrieval_cutoff": parse_iso_date(
211 payload["retrieval_cutoff"], "retrieval_cutoff"
212 ),
213 }
216def _safe_new_directory(raw_path: str | Path) -> Path:
217 path = Path(raw_path).expanduser()
218 if path.exists() or path.is_symlink():
219 raise ValidationError(f"output directory already exists: {path}")
220 try:
221 parent = path.parent.resolve(strict=True)
222 except FileNotFoundError as exc:
223 raise ValidationError(f"output parent does not exist: {path.parent}") from exc
224 if not parent.is_dir():
225 raise ValidationError(f"output parent is not a directory: {parent}")
226 if path.name in {"", ".", ".."}:
227 raise ValidationError("output directory name is invalid")
228 return parent / path.name
231def _write_csv_header(path: Path, fields: tuple[str, ...]) -> None:
232 with path.open("w", encoding="utf-8", newline="") as handle:
233 csv.writer(handle, lineterminator="\n").writerow(fields)
236def _one_line(value: str) -> str:
237 return " ".join(value.split())
240def generate(manifest: dict[str, Any], output_dir: str | Path) -> dict[str, Any]:
241 destination = _safe_new_directory(output_dir)
242 destination.mkdir(mode=0o755)
243 (destination / "data").mkdir()
244 (destination / "analysis").mkdir()
245 (destination / "sources").mkdir()
247 with (destination / "manifest.json").open("w", encoding="utf-8") as handle:
248 json.dump(manifest, handle, indent=2, sort_keys=True, ensure_ascii=False)
249 handle.write("\n")
251 inclusions = "\n".join(f"- {item}" for item in manifest["inclusions"])
252 exclusions = "\n".join(f"- {item}" for item in manifest["exclusions"])
253 report = f"""# {_one_line(manifest["title"])}
255{_one_line(manifest["subtitle"])}
257Prepared for: {_one_line(manifest["prepared_for"])}
258Prepared by: {_one_line(manifest["prepared_by"])}
259Retrieval cutoff: {manifest["retrieval_cutoff"]}
260Classification: {manifest["classification"]}
262> This report separates sourced facts, calculations, assumptions, forecasts, and
263> recommendations. Inline claim IDs map to `data/claims.csv`; source IDs map to
264> `data/source_ledger.csv`. It is not investment, legal, or financial advice.
266## Executive synopsis
268State only findings supported by claim IDs. Present market-size and forecast
269results as scenario ranges, not as a single certain value.
271## 1. Scope and market definition
273**Definition:** {manifest["market_definition"]}
275**Geography:** {manifest["geography"]}
276**Currency/base:** {manifest["currency"]}, {manifest["price_basis"]},
277base year {manifest["base_year"]}
278**Taxonomy:** {manifest["taxonomy"]} {manifest["taxonomy_version"]}
279**Historical period:** {manifest["historical_period"]}
280**Forecast period:** {manifest["forecast_period"]}
282### Included
283{inclusions}
285### Excluded
286{exclusions}
288## 2. Evidence and methodology
290Describe source hierarchy, retrieval dates, revisions, conflicts, conversions,
291survey methods, and limitations.
293## 3. Market sizing and reconciliation
295Report top-down and bottom-up TAM estimates separately. Show denominator,
296coverage keys, excluded categories, SAM filters, SOM capture assumptions,
297scenario range, sensitivity, and reconciliation gap.
299## 4. Demand and customer evidence
301Separate observed demand, survey estimates, interview themes, and analyst
302interpretation. Do not generalize qualitative interviews to a population.
304## 5. Competitive landscape
306Define product and geographic scope before calculating shares or concentration.
307Use the evidence-linked competitor matrix; label unknowns.
309## 6. Forecast scenarios
311Describe each conditional scenario, rate path, assumptions, drivers, inhibitors,
312and sensitivity. Do not label scenario ranges as confidence intervals.
314## 7. Regulation, risks, and uncertainties
316Distinguish enacted rules, proposed rules, analyst judgments, and legal advice.
318## 8. Implications and options
320Keep recommendations distinct from sourced findings. State dependencies,
321decision thresholds, and disconfirming evidence.
323## Limitations
325List missing data, source conflicts, taxonomy breaks, revisions, sampling and
326nonresponse issues, model limitations, and residual uncertainty.
328## References
330Render references from `data/source_ledger.csv`; do not cite unmapped sources.
331"""
332 (destination / "report.md").write_text(report, encoding="utf-8")
334 _write_csv_header(destination / "data" / "source_ledger.csv", SOURCE_FIELDS)
335 _write_csv_header(destination / "data" / "claims.csv", CLAIM_FIELDS)
336 _write_csv_header(
337 destination / "data" / "competitor_feature_matrix.csv", COMPETITOR_FIELDS
338 )
339 _write_csv_header(
340 destination / "data" / "consistency_input.csv", CONSISTENCY_FIELDS
341 )
342 _write_csv_header(destination / "data" / "assumptions.csv", ASSUMPTION_FIELDS)
344 sizing = {
345 "schema_version": "1.0",
346 "metadata": {
347 "market_id": manifest["report_id"],
348 "market_definition": manifest["market_definition"],
349 "geography": manifest["geography"],
350 "currency": manifest["currency"],
351 "base_year": manifest["base_year"],
352 "price_basis": manifest["price_basis"],
353 "unit": f"{manifest['currency']} per year",
354 "taxonomy": manifest["taxonomy"],
355 "taxonomy_version": manifest["taxonomy_version"],
356 "denominator_id": "define-denominator",
357 "as_of_date": manifest["retrieval_cutoff"],
358 },
359 "top_down": {"components": []},
360 "bottom_up": {"components": []},
361 "scenarios": [],
362 "reconciliation_tolerance_percent": 20.0,
363 }
364 forecast = {
365 "schema_version": "1.0",
366 "metadata": {
367 "series_id": "define-series",
368 "description": "Replace with the forecast measure",
369 "geography": manifest["geography"],
370 "currency": manifest["currency"],
371 "base_year": str(manifest["base_year"]),
372 "price_basis": manifest["price_basis"],
373 "unit": f"{manifest['currency']} per year",
374 "measure_type": "flow",
375 "as_of_date": manifest["retrieval_cutoff"],
376 "source_ids": [],
377 },
378 "start_year": int(manifest["historical_period"].split("-")[1]),
379 "start_value": 0,
380 "horizon_years": (
381 int(manifest["forecast_period"].split("-")[1])
382 - int(manifest["historical_period"].split("-")[1])
383 ),
384 "scenarios": [],
385 "sensitivity": {
386 "base_scenario_id": "base",
387 "growth_rate_shifts": [-0.02, 0.0, 0.02],
388 },
389 }
390 for name, payload in (
391 ("market_sizing.json", sizing),
392 ("forecast_sensitivity.json", forecast),
393 ):
394 with (destination / "analysis" / name).open("w", encoding="utf-8") as handle:
395 json.dump(payload, handle, indent=2, sort_keys=True, ensure_ascii=False)
396 handle.write("\n")
398 checklist = """# Review checklist
400- [ ] Scope, inclusions, exclusions, geography, period, taxonomy, and denominator are explicit.
401- [ ] Every factual or quantitative claim maps to source IDs.
402- [ ] Publication and retrieval dates, revisions, units, currencies, and base years are recorded.
403- [ ] Top-down and bottom-up sizing use compatible definitions and disjoint coverage keys.
404- [ ] TAM/SAM/SOM are presented as conditional scenarios with sensitivity.
405- [ ] Forecast ranges are labeled as scenarios, not confidence intervals.
406- [ ] Survey sample, frame, mode, field dates, weighting, response, and limitations are disclosed.
407- [ ] Competitor claims use public, lawful evidence and identify unknowns.
408- [ ] No PII, trade secrets, deceptive collection, or unsupported paid-market figures appear.
409- [ ] Findings, assumptions, calculations, forecasts, opinions, and recommendations remain distinct.
410- [ ] The report states that it is not investment, legal, or financial advice.
411"""
412 (destination / "review_checklist.md").write_text(checklist, encoding="utf-8")
414 created = sorted(
415 str(path.relative_to(destination))
416 for path in destination.rglob("*")
417 if path.is_file()
418 )
419 return {
420 "created": True,
421 "output_directory": str(destination),
422 "files": created,
423 "next_step": (
424 "Populate the empty ledgers and scenario files, then run the bundled "
425 "validators before drafting conclusions."
426 ),
427 }
430def build_parser() -> argparse.ArgumentParser:
431 parser = argparse.ArgumentParser(
432 description=(
433 "Generate a new local Markdown market-report scaffold and strict "
434 "evidence/analysis input files. Existing directories are never overwritten."
435 )
436 )
437 parser.add_argument("manifest", help="Local .json report manifest")
438 parser.add_argument("output_dir", help="New local output directory")
439 return parser
442def main() -> int:
443 args = build_parser().parse_args()
444 try:
445 manifest = validate_manifest(require_object(read_json(args.manifest), "root"))
446 write_json_report(generate(manifest, args.output_dir), None)
447 return 0
448 except ValidationError as exc:
449 return error_exit(exc)
452if __name__ == "__main__":
453 raise SystemExit(main())