diff --git a/.github/bump_version.py b/.github/bump_version.py new file mode 100644 index 00000000..bb0fd6dd --- /dev/null +++ b/.github/bump_version.py @@ -0,0 +1,79 @@ +"""Infer semver bump from towncrier fragment types and update version.""" + +import re +import sys +from pathlib import Path + + +def get_current_version(pyproject_path: Path) -> str: + text = pyproject_path.read_text() + match = re.search(r'^version\s*=\s*"(\d+\.\d+\.\d+)"', text, re.MULTILINE) + if not match: + print( + "Could not find version in pyproject.toml", + file=sys.stderr, + ) + sys.exit(1) + return match.group(1) + + +def infer_bump(changelog_dir: Path) -> str: + fragments = [ + f + for f in changelog_dir.iterdir() + if f.is_file() and f.name != ".gitkeep" + ] + if not fragments: + print("No changelog fragments found", file=sys.stderr) + sys.exit(1) + + categories = {f.suffix.lstrip(".") for f in fragments} + for f in fragments: + parts = f.stem.split(".") + if len(parts) >= 2: + categories.add(parts[-1]) + + if "breaking" in categories: + return "major" + if "added" in categories or "removed" in categories: + return "minor" + return "patch" + + +def bump_version(version: str, bump: str) -> str: + major, minor, patch = (int(x) for x in version.split(".")) + if bump == "major": + return f"{major + 1}.0.0" + elif bump == "minor": + return f"{major}.{minor + 1}.0" + else: + return f"{major}.{minor}.{patch + 1}" + + +def update_file(path: Path, old_version: str, new_version: str): + text = path.read_text() + updated = text.replace( + f'version = "{old_version}"', + f'version = "{new_version}"', + ) + if updated != text: + path.write_text(updated) + print(f" Updated {path}") + + +def main(): + root = Path(__file__).resolve().parent.parent + pyproject = root / "pyproject.toml" + changelog_dir = root / "changelog.d" + + current = get_current_version(pyproject) + bump = infer_bump(changelog_dir) + new = bump_version(current, bump) + + print(f"Version: {current} -> {new} ({bump})") + + update_file(pyproject, current, new) + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/changelog_entry.yaml b/.github/workflows/changelog_entry.yaml index d0eb8574..49ac82a9 100644 --- a/.github/workflows/changelog_entry.yaml +++ b/.github/workflows/changelog_entry.yaml @@ -1,29 +1,21 @@ -name: Versioning +name: Changelog entry on: pull_request: - branches: [ main ] + branches: [main] jobs: - check-changelog-entry: - name: Changelog entry check + check-changelog: + name: Check changelog fragment runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - - name: Check for changelog entry + - uses: actions/checkout@v4 + - name: Check for changelog fragment run: | - if [ ! -f "changelog_entry.yaml" ]; then - echo "Error: changelog_entry.yaml file is missing." - echo "Please add a changelog_entry.yaml file at the root of the repository." + FRAGMENTS=$(find changelog.d -type f ! -name '.gitkeep' | wc -l) + if [ "$FRAGMENTS" -eq 0 ]; then + echo "::error::No changelog fragment found in changelog.d/" + echo "Add one with: echo 'Description.' > changelog.d/\$(git branch --show-current)..md" + echo "Types: added, changed, fixed, removed, breaking" exit 1 fi - - # Check if the file is empty - if [ ! -s "changelog_entry.yaml" ]; then - echo "Error: changelog_entry.yaml file is empty." - echo "Please add content to the changelog_entry.yaml file." - exit 1 - fi - - echo "Changelog entry found and is not empty." \ No newline at end of file diff --git a/.github/workflows/versioning.yaml b/.github/workflows/versioning.yaml index b4e0edb9..73b99160 100644 --- a/.github/workflows/versioning.yaml +++ b/.github/workflows/versioning.yaml @@ -7,7 +7,7 @@ on: - main paths: - - changelog_entry.yaml + - changelog.d/** - "!pyproject.toml" jobs: @@ -19,19 +19,20 @@ jobs: - name: Checkout repo uses: actions/checkout@v4 with: - repository: ${{ github.event.pull_request.head.repo.full_name }} - ref: ${{ github.event.pull_request.head.ref }} token: ${{ secrets.POLICYENGINE_GITHUB }} + fetch-depth: 0 - name: Setup Python uses: actions/setup-python@v5 with: python-version: 3.12 - - name: Build changelog - run: pip install yaml-changelog && make changelog - - name: Preview changelog update - run: ".github/get-changelog-diff.sh" + - name: Install towncrier + run: pip install towncrier + - name: Bump version and build changelog + run: | + python .github/bump_version.py + towncrier build --yes --version $(python -c "import re; print(re.search(r'version = \"(.+?)\"', open('pyproject.toml').read()).group(1))") - name: Update changelog uses: EndBug/add-and-commit@v9 with: add: "." - message: Update package version \ No newline at end of file + message: Update package version diff --git a/Makefile b/Makefile index d9b90178..327895d3 100644 --- a/Makefile +++ b/Makefile @@ -30,8 +30,5 @@ publish: twine upload dist/* changelog: - build-changelog changelog.yaml --output changelog.yaml --update-last-date --start-from 1.0.0 --append-file changelog_entry.yaml - build-changelog changelog.yaml --org PolicyEngine --repo policyengine-us-data --output CHANGELOG.md --template .github/changelog_template.md - bump-version changelog.yaml pyproject.toml - rm changelog_entry.yaml || true - touch changelog_entry.yaml + python .github/bump_version.py + towncrier build --yes --version $$(python -c "import re; print(re.search(r'version = \"(.+?)\"', open('pyproject.toml').read()).group(1))") \ No newline at end of file diff --git a/changelog_entry.yaml b/changelog.d/.gitkeep similarity index 100% rename from changelog_entry.yaml rename to changelog.d/.gitkeep diff --git a/changelog.d/migrate-to-towncrier.changed.md b/changelog.d/migrate-to-towncrier.changed.md new file mode 100644 index 00000000..865484ad --- /dev/null +++ b/changelog.d/migrate-to-towncrier.changed.md @@ -0,0 +1 @@ +Migrated from changelog_entry.yaml to towncrier fragments to eliminate merge conflicts. diff --git a/changelog.yaml b/changelog.yaml deleted file mode 100644 index 3e80357b..00000000 --- a/changelog.yaml +++ /dev/null @@ -1,584 +0,0 @@ -- changes: - added: - - Initialized changelogging - date: 2024-09-09 17:29:10 - version: 1.0.0 -- bump: minor - changes: - changed: - - Lightened dependency list. - date: 2024-09-17 18:05:27 -- bump: minor - changes: - fixed: - - Compatibility with PolicyEngine UK. - date: 2024-09-18 00:32:05 -- bump: patch - changes: - fixed: - - Data download URLs. - date: 2024-09-18 10:05:45 -- bump: patch - changes: - removed: - - Survey Enhance is no longer used as a top-level import. - date: 2024-09-18 11:38:21 -- bump: patch - changes: - fixed: - - Bug causing the Extended FRS to not generate. - date: 2024-09-18 12:57:28 -- bump: patch - changes: - removed: - - Torch as a dependency. - date: 2024-09-18 13:26:36 -- bump: patch - changes: - fixed: - - GH actions naming. - - Bug causing the Extended FRS to error. - date: 2024-09-18 13:57:40 -- bump: minor - changes: - added: - - Re-run calibration with more epochs. - date: 2024-10-16 17:02:56 -- bump: minor - changes: - added: - - Missing changelog entry. - date: 2024-10-16 17:05:39 -- bump: minor - changes: - added: - - Moved epoch count to 10k per year. - date: 2024-10-16 17:05:58 -- bump: minor - changes: - added: - - Future year income targeting. - - Random takeup variable values. - date: 2024-10-18 16:05:10 -- bump: minor - changes: - added: - - Calibration for private school students. - date: 2024-10-21 17:03:50 -- bump: minor - changes: - changed: - - Data URLs updated. - date: 2024-10-22 08:30:52 -- bump: minor - changes: - fixed: - - Bug removing capital gains. - date: 2024-10-22 11:18:48 -- bump: patch - changes: - added: - - Automatic calibration. - date: 2024-11-27 19:28:29 -- bump: patch - changes: - fixed: - - Constituency weights are in A-Z order. - date: 2024-11-30 13:23:17 -- bump: minor - changes: - added: - - Target uprating for constituencies. - date: 2024-12-03 11:21:54 -- bump: patch - changes: - added: - - Dropout in constituency calibration. - date: 2024-12-03 17:25:35 -- bump: minor - changes: - added: - - Local authority weights. - date: 2024-12-09 11:51:05 -- bump: patch - changes: - fixed: - - Documentation errors. - date: 2024-12-10 10:46:07 -- bump: patch - changes: - fixed: - - GCP uploads to buckets happen alongside HF uploads. - - Versioning in data uploads. - date: 2025-05-26 21:38:53 -- bump: patch - changes: - fixed: - - Typo in huggingface repo location. - date: 2025-05-26 21:44:28 -- bump: patch - changes: - fixed: - - Added missing HF token - date: 2025-05-26 22:10:54 -- bump: patch - changes: - added: - - Child limit affected household calibration. - date: 2025-05-27 13:30:04 -- bump: patch - changes: - fixed: - - Uprating in child limit target. - date: 2025-05-27 14:40:08 -- bump: minor - changes: - fixed: - - Inconsistent local area targets removed. - date: 2025-06-09 20:25:41 -- bump: minor - changes: - changed: - - Tax-benefit targets updated from new DWP forecasts. - date: 2025-06-10 12:37:20 -- bump: patch - changes: - fixed: - - Documentation deployment. - date: 2025-06-10 12:41:53 -- bump: patch - changes: - fixed: - - Documentation publishes. - - Local authority calibration consistent with constituency calibration. - - Domestic rates are nonzero. - date: 2025-06-13 14:51:39 -- bump: patch - changes: - fixed: - - Documentation used 2022 datasets rather than 2025. - date: 2025-06-16 15:00:47 -- bump: minor - changes: - added: - - Public service imputations. - date: 2025-06-19 21:08:19 -- bump: patch - changes: - added: - - Calibration log exporting. - date: 2025-06-24 10:48:32 -- bump: patch - changes: - added: - - Calibration improvements. - date: 2025-06-24 11:18:42 -- bump: patch - changes: - fixed: - - Name corrected in calibration build artifact. - date: 2025-06-24 11:39:46 -- bump: patch - changes: - fixed: - - Missing columns in the calibration log. - date: 2025-06-24 13:14:48 -- bump: minor - changes: - added: - - PIP calibration. - date: 2025-06-27 09:15:13 -- bump: patch - changes: - added: - - Free school meals - date: 2025-07-09 14:19:38 -- bump: patch - changes: - fixed: - - Free school meals magnitude error. - date: 2025-07-09 15:25:53 -- bump: patch - changes: - fixed: - - SSMG uprating. - date: 2025-07-10 12:41:13 -- bump: minor - changes: - added: - - Structural insurance payments - - External child payments - - Healthy Start payments - date: 2025-07-10 16:01:46 -- bump: patch - changes: - fixed: - - Added calibrated weights from 2022. - date: 2025-07-14 15:21:27 -- bump: patch - changes: - added: - - Council Tax calibration. - date: 2025-07-17 11:44:26 -- bump: minor - changes: - added: - - New multi-year dataset format for FRS and Enhanced FRS. - date: 2025-07-21 13:53:43 -- bump: patch - changes: - fixed: - - Bug in new multi-year dataset. - date: 2025-07-22 08:53:18 -- bump: patch - changes: - changed: - - Updated policyengine-uk to 2.40.2 (pin). - date: 2025-07-22 09:41:46 -- bump: patch - changes: - fixed: - - UK model actually bumped. - date: 2025-07-22 11:17:39 -- bump: patch - changes: - fixed: - - Migrated to more efficient dataset version. - date: 2025-08-04 15:08:06 -- bump: patch - changes: - added: - - Added index.html file to docs folder to fix GitHub Pages 404 error - date: 2025-08-05 11:15:17 -- bump: patch - changes: - added: - - Add index.yaml to fix GitHub Pages 404 error - date: 2025-08-05 11:47:08 -- bump: patch - changes: - changed: - - Moved to functional, simplified architecture. - date: 2025-08-06 10:08:10 -- bump: patch - changes: - fixed: - - Test result. - date: 2025-08-06 10:28:59 -- bump: patch - changes: - fixed: - - Imputation model syntax. - date: 2025-08-20 10:32:56 -- bump: patch - changes: - added: - - Calibration to benefit cap statistics. - date: 2025-09-01 09:33:20 -- bump: patch - changes: - fixed: - - Add is_married to FRS benefit unit dataset - date: 2025-09-11 16:14:02 -- bump: minor - changes: - changed: - - Upgraded documentation to Jupyter Book 2.0 (MyST-based) - fixed: - - Jupyter Book deployment to GitHub Pages by adding docs workflow and fixing branch - reference - date: 2025-09-30 13:58:18 -- bump: minor - changes: - fixed: - - Re-add dividends to calibration target set. - date: 2025-10-02 14:29:16 -- bump: patch - changes: - changed: - - Remove birth_year from FRS dataset generation to allow dynamic calculation - date: 2025-10-02 15:18:04 -- bump: patch - changes: - changed: - - Relaxed childcare test tolerance to allow ratios up to 1.6 - date: 2025-10-02 16:12:23 -- bump: patch - changes: - changed: - - Relaxed childcare test tolerance to allow ratios within 100% of target (0 to - 2.0) - date: 2025-10-02 16:46:59 -- bump: patch - changes: - added: - - Regional and country labels for UK constituencies. - date: 2025-10-07 16:32:10 -- bump: patch - changes: - fixed: - - Bump patch version to try and get HF upload passing. - date: 2025-10-20 16:37:14 -- bump: patch - changes: - changed: - - Refactored income imputation to selectively impute only dividend income on the - main dataset. - - Removed winter fuel allowance from loss calculations. - date: 2025-10-21 10:18:38 -- bump: minor - changes: - added: - - Universal Credit calibration at national level by award amount and family type, - and at constituency level in total. - date: 2025-10-21 12:09:14 -- bump: minor - changes: - added: - - Add pension_contributions_via_salary_sacrifice variable from FRS SPNAMT field - date: 2025-11-20 13:08:22 -- bump: minor - changes: - added: - - Salary sacrifice imputation using FRS SALSAC routing question to impute ~30% - employee participation per HMRC survey data. - date: 2025-11-26 22:46:41 -- bump: minor - changes: - added: - - "Add salary sacrifice NI relief as calibration targets (employee \xA31.2bn,\ - \ employer \xA32.9bn from SPP)" - date: 2025-11-27 15:30:00 -- bump: patch - changes: - fixed: - - Hallucinated calibration targets. - date: 2025-11-27 16:12:15 -- bump: patch - changes: - added: - - SS HMRC calibration targets. - date: 2025-11-27 16:29:01 -- bump: patch - changes: - fixed: - - Updated personal allowance reform test expected value after calibration fix. - date: 2025-11-27 17:11:09 -- bump: minor - changes: - added: - - rail_usage variable derived from rail_subsidy_spending / fare_index at survey - year, enabling fare reforms to modify prices independently of usage quantity - date: 2025-11-27 18:33:27 -- bump: patch - changes: - changed: - - Calibrate savings income from ONS National Accounts D.41g household interest - data instead of SPI (fixes underestimation from ~3bn to ~55bn) - date: 2025-11-28 16:14:35 -- bump: patch - changes: - fixed: - - Fix changelog encoding test to skip when changelog_entry.yaml is empty after - versioning - date: 2025-11-28 17:23:26 -- bump: minor - changes: - added: - - Student loan plan imputation based on age and reported repayments - date: 2025-11-29 00:33:46 -- bump: minor - changes: - fixed: - - LA calibration now consistent with constituency calibration. - date: 2025-12-01 11:20:56 -- bump: patch - changes: - fixed: - - Made sure LA calibration actually runs. - date: 2025-12-02 00:22:31 -- bump: minor - changes: - added: - - Add vehicle ownership calibration and imputation - date: 2025-12-03 17:02:06 -- bump: patch - changes: - changed: - - Updated Node.js from 22 to 24 LTS in docs workflow - date: 2025-12-03 22:49:46 -- bump: minor - changes: - added: - - Use num_vehicles as predictor for fuel spending imputation - date: 2025-12-03 23:01:14 -- bump: minor - changes: - added: - - Housing cost calibration to LAs. - date: 2025-12-04 17:51:20 -- bump: patch - changes: - fixed: - - Zero out fuel spending for households without fuel consumption - date: 2025-12-07 09:17:37 -- bump: patch - changes: - fixed: - - Added mortgage and private rent targets. - date: 2025-12-08 17:47:38 -- bump: patch - changes: - fixed: - - A bug hard-coded to 2023, which meant that custom FRS datasets defined after - 2024 fail on BRMA imputation. - date: 2025-12-09 10:06:31 -- bump: patch - changes: - changed: - - Reduced test runtime by using 32 epochs instead of 512 when TESTING environment - variable is set - date: 2025-12-09 14:39:12 -- bump: minor - changes: - added: - - Tests for property_purchased rate and SDLT total validation against official - HMRC figures. - - Tests for low-income decile sanity checks to prevent negative net incomes and - impossible tax rates. - fixed: - - Set property_purchased stochastically at 3.85% based on HMRC housing transaction - data, fixing unrealistic SDLT charges that caused 224% tax rates in the first - income decile. - date: 2025-12-19 11:21:28 -- bump: patch - changes: - fixed: - - Widened VAT reform test tolerance to accommodate calibration variance between - test and production datasets. - date: 2025-12-19 14:41:54 -- bump: patch - changes: - fixed: - - Clip negative values to zero for employment, self-employment, savings interest, - tax-free savings, and dividend income variables. - date: 2026-01-06 09:55:01 -- bump: minor - changes: - added: - - Move all randomness to data package for deterministic country package. Take-up - decisions are now generated stochastically during dataset creation using take-up - rates from YAML parameter files. - date: 2026-01-06 10:13:56 -- bump: minor - changes: - added: - - Scotland-specific calibration targets for children under 16 (NRS mid-year estimates) - and households with 3+ children (Census 2022) to improve accuracy of Scottish - policy analysis. - date: 2026-01-13 14:24:26 -- bump: minor - changes: - added: - - Add calibration target for babies under 1 in Scotland (~46k from NRS Vital Events) - date: 2026-01-15 16:03:56 -- bump: minor - changes: - added: - - Add would_claim_scp variable generation for Scottish Child Payment deterministic - takeup - date: 2026-01-17 02:11:31 -- bump: minor - changes: - added: - - Add Scottish Child Payment as a calibration target using Scottish Budget figures - date: 2026-01-17 05:09:55 -- bump: patch - changes: - changed: - - Improved calibration weight initialization to use country-aware divisors. Households - now start with weights divided by the number of areas in their country rather - than total areas, improving convergence for devolved nation targets. - date: 2026-01-17 16:56:01 -- bump: minor - changes: - added: - - Add calibration target for UC households in Scotland with youngest child under - 1 (~14k from DWP Stat-Xplore) - date: 2026-01-21 14:22:42 -- bump: patch - changes: - fixed: - - Fix black formatting for 5 files to comply with black 26.1.0 - date: 2026-01-21 15:14:09 -- bump: patch - changes: - fixed: - - Use region.values for Scotland comparisons in loss function to ensure consistent - behavior with StringArray types - date: 2026-01-21 19:06:42 -- bump: minor - changes: - added: - - Calibrate salary sacrifice population to HMRC/ASHE headcount targets (7.7mn - total, 3.3mn above 2k cap, 4.3mn below 2k cap). Two-stage imputation in salary_sacrifice.py - converts pension contributors to below-cap SS users, and three new headcount - calibration targets in loss.py. - date: 2026-02-16 10:16:06 -- bump: patch - changes: - fixed: - - Fix salary sacrifice headcount regression caused by uprating mismatch. The calibrator - classified records above/below the 2k cap at uprated 2025 prices, but the saved - dataset stores values at 2023 prices where the classification differs. Now evaluates - SS amounts at base-year prices before applying the threshold, so calibration - matches what end-users see. Stage 2 imputation now moves the full employee pension - amount to SS (instead of capping at 2k) so both above-cap and below-cap records - are created for the calibrator. - - Remove xfail markers from salary sacrifice headcount tests. - date: 2026-02-17 16:05:41 -- bump: minor - changes: - changed: - - Replaced ad-hoc calibration targets with structured registry and source modules. - date: 2026-02-18 14:08:40 -- bump: minor - changes: - added: - - highest_education variable derived from FRS EDUCQUAL field. - date: 2026-02-19 08:39:08 -- bump: patch - changes: - added: - - Test for highest_education in enhanced FRS dataset. - date: 2026-02-19 11:54:03 -- bump: patch - changes: - added: - - UC households by children count (0, 1, 2, 3+) as constituency calibration targets. - date: 2026-02-19 13:58:30 -- bump: patch - changes: - fixed: - - Widened UC taper rate reform test tolerance to 15bn to account for calibration - variance. - date: 2026-02-19 16:15:46 -- bump: minor - changes: - added: - - SLC student loan calibration targets for Plan 2 and Plan 5 England borrowers - earning above repayment threshold (2025-2030), fetched live from Explore Education - Statistics. - date: 2026-02-20 13:50:59 -- bump: patch - changes: - fixed: - - Upload datasets to public HuggingFace repo (policyengine/policyengine-uk-data) - in addition to private repo, so policyengine-uk gets the latest data. - date: 2026-02-23 13:26:29 -- bump: patch - changes: - fixed: - - Revert public HuggingFace upload that would have violated UK Data Service licence - terms. - date: 2026-02-23 17:00:30 diff --git a/pyproject.toml b/pyproject.toml index 3ccfd29c..a894a060 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,7 +43,8 @@ dev = [ "yaml-changelog>=0.1.7", "itables", "quantile-forest", - "build", + "build", "towncrier>=24.8.0", + ] [tool.setuptools] @@ -85,4 +86,37 @@ extend-exclude = ''' | build | dist )/ -''' +''' + +[tool.towncrier] +package = "policyengine_uk_data" +directory = "changelog.d" +filename = "CHANGELOG.md" +title_format = "## [{version}] - {project_date}" +issue_format = "" +underlines = ["", "", ""] + +[[tool.towncrier.type]] +directory = "breaking" +name = "Breaking changes" +showcontent = true + +[[tool.towncrier.type]] +directory = "added" +name = "Added" +showcontent = true + +[[tool.towncrier.type]] +directory = "changed" +name = "Changed" +showcontent = true + +[[tool.towncrier.type]] +directory = "fixed" +name = "Fixed" +showcontent = true + +[[tool.towncrier.type]] +directory = "removed" +name = "Removed" +showcontent = true