Add calibration package checkpointing, target config, and hyperparameter CLI

.github/bump_version.py

@@ -0,0 +1,79 @@
+"""Infer semver bump from towncrier fragment types and update version."""
+
+import re
+import sys
+from pathlib import Path
+
+
+def get_current_version(pyproject_path: Path) -> str:
+    text = pyproject_path.read_text()
+    match = re.search(r'^version\s*=\s*"(\d+\.\d+\.\d+)"', text, re.MULTILINE)
+    if not match:
+        print(
+            "Could not find version in pyproject.toml",
+            file=sys.stderr,
+        )
+        sys.exit(1)
+    return match.group(1)
+
+
+def infer_bump(changelog_dir: Path) -> str:
+    fragments = [
+        f
+        for f in changelog_dir.iterdir()
+        if f.is_file() and f.name != ".gitkeep"
+    ]
+    if not fragments:
+        print("No changelog fragments found", file=sys.stderr)
+        sys.exit(1)
+
+    categories = {f.suffix.lstrip(".") for f in fragments}
+    for f in fragments:
+        parts = f.stem.split(".")
+        if len(parts) >= 2:
+            categories.add(parts[-1])
+
+    if "breaking" in categories:
+        return "major"
+    if "added" in categories or "removed" in categories:
+        return "minor"
+    return "patch"
+
+
+def bump_version(version: str, bump: str) -> str:
+    major, minor, patch = (int(x) for x in version.split("."))
+    if bump == "major":
+        return f"{major + 1}.0.0"
+    elif bump == "minor":
+        return f"{major}.{minor + 1}.0"
+    else:
+        return f"{major}.{minor}.{patch + 1}"
+
+
+def update_file(path: Path, old_version: str, new_version: str):
+    text = path.read_text()
+    updated = text.replace(
+        f'version = "{old_version}"',
+        f'version = "{new_version}"',
+    )
+    if updated != text:
+        path.write_text(updated)
+        print(f"  Updated {path}")
+
+
+def main():
+    root = Path(__file__).resolve().parent.parent
+    pyproject = root / "pyproject.toml"
+    changelog_dir = root / "changelog.d"
+
+    current = get_current_version(pyproject)
+    bump = infer_bump(changelog_dir)
+    new = bump_version(current, bump)
+
+    print(f"Version: {current} -> {new} ({bump})")
+
+    update_file(pyproject, current, new)
+
+
+if __name__ == "__main__":
+    main()

.github/workflows/local_area_publish.yaml

@@ -4,7 +4,7 @@ on:
   push:
     branches: [main]
     paths:
-      - 'policyengine_us_data/datasets/cps/local_area_calibration/**'
+      - 'policyengine_us_data/calibration/**'
       - '.github/workflows/local_area_publish.yaml'
       - 'modal_app/**'
   repository_dispatch:
@@ -23,7 +23,7 @@ on:
         type: boolean
 
 # Trigger strategy:
-# 1. Automatic: Code changes to local_area_calibration/ pushed to main
+# 1. Automatic: Code changes to calibration/ pushed to main
 # 2. repository_dispatch: Calibration workflow triggers after uploading new weights
 # 3. workflow_dispatch: Manual trigger with optional parameters
 
@@ -55,7 +55,7 @@ jobs:
           SKIP_UPLOAD="${{ github.event.inputs.skip_upload || 'false' }}"
           BRANCH="${{ github.head_ref || github.ref_name }}"
 
-          CMD="modal run modal_app/local_area.py --branch=${BRANCH} --num-workers=${NUM_WORKERS}"
+          CMD="modal run modal_app/local_area.py::main --branch=${BRANCH} --num-workers=${NUM_WORKERS}"
 
           if [ "$SKIP_UPLOAD" = "true" ]; then
             CMD="${CMD} --skip-upload"
@@ -71,5 +71,60 @@ jobs:
           echo "" >> $GITHUB_STEP_SUMMARY
           echo "Files have been uploaded to GCS and staged on HuggingFace." >> $GITHUB_STEP_SUMMARY
           echo "" >> $GITHUB_STEP_SUMMARY
-          echo "### Next step: Promote to production" >> $GITHUB_STEP_SUMMARY
-          echo "Trigger the **Promote Local Area H5 Files** workflow with the version from the build output." >> $GITHUB_STEP_SUMMARY
+          echo "### Next step: Validation runs automatically" >> $GITHUB_STEP_SUMMARY
+          echo "The validate-staging job will now check all staged H5s." >> $GITHUB_STEP_SUMMARY
+
+  validate-staging:
+    needs: publish-local-area
+    runs-on: ubuntu-latest
+    env:
+      HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }}
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.13'
+
+      - name: Set up uv
+        uses: astral-sh/setup-uv@v5
+
+      - name: Install dependencies
+        run: uv sync
+
+      - name: Validate staged H5s
+        run: |
+          uv run python -m policyengine_us_data.calibration.validate_staging \
+            --area-type states --output validation_results.csv
+
+      - name: Upload validation results to HF
+        run: |
+          uv run python -c "
+          from policyengine_us_data.utils.huggingface import upload
+          upload('validation_results.csv',
+                 'policyengine/policyengine-us-data',
+                 'calibration/logs/validation_results.csv')
+          "
+
+      - name: Post validation summary
+        if: always()
+        run: |
+          echo "## Validation Results" >> $GITHUB_STEP_SUMMARY
+          if [ -f validation_results.csv ]; then
+            TOTAL=$(tail -n +2 validation_results.csv | wc -l)
+            FAILS=$(grep -c ',FAIL,' validation_results.csv || true)
+            echo "- **${TOTAL}** targets validated" >> $GITHUB_STEP_SUMMARY
+            echo "- **${FAILS}** sanity failures" >> $GITHUB_STEP_SUMMARY
+            echo "" >> $GITHUB_STEP_SUMMARY
+            echo "Review in dashboard, then trigger **Promote** workflow." >> $GITHUB_STEP_SUMMARY
+          else
+            echo "Validation did not produce output." >> $GITHUB_STEP_SUMMARY
+          fi
+
+      - name: Upload validation artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: validation-results
+          path: validation_results.csv

.github/workflows/pr_changelog.yaml

@@ -1,30 +1,21 @@
 name: Changelog entry
+
 on:
   pull_request:
     branches: [main]
 
 jobs:
-  check-fork:
+  check-changelog:
+    name: Check changelog fragment
     runs-on: ubuntu-latest
     steps:
-      - name: Check if PR is from fork
+      - uses: actions/checkout@v4
+      - name: Check for changelog fragment
         run: |
-          if [ "${{ github.event.pull_request.head.repo.full_name }}" != "${{ github.repository }}" ]; then
-            echo "❌ ERROR: This PR is from a fork repository."
-            echo "PRs must be created from branches in the main PolicyEngine/policyengine-us-data repository."
-            echo "Please close this PR and create a new one following these steps:"
-            echo "1. git checkout main"
-            echo "2. git pull upstream main"
-            echo "3. git checkout -b your-branch-name"
-            echo "4. git push -u upstream your-branch-name"
-            echo "5. Create PR from the upstream branch"
+          FRAGMENTS=$(find changelog.d -type f ! -name '.gitkeep' | wc -l)
+          if [ "$FRAGMENTS" -eq 0 ]; then
+            echo "::error::No changelog fragment found in changelog.d/"
+            echo "Add one with: echo 'Description.' > changelog.d/\$(git branch --show-current).<type>.md"
+            echo "Types: added, changed, fixed, removed, breaking"
             exit 1
           fi
-          echo "✅ PR is from the correct repository"
-
-  require-entry:
-    needs: check-fork
-    uses: ./.github/workflows/reusable_changelog_check.yaml
-    with:
-      require_entry: true
-      validate_format: true

.github/workflows/versioning.yaml

@@ -7,7 +7,7 @@ on:
           - main
 
       paths:
-        - changelog_entry.yaml
+        - "changelog.d/**"
         - "!pyproject.toml"
 
 jobs:
@@ -19,20 +19,23 @@ jobs:
             uses: actions/checkout@v4
             with:
               token: ${{ secrets.POLICYENGINE_GITHUB }}
+              fetch-depth: 0
           - name: Setup Python
             uses: actions/setup-python@v5
             with:
               python-version: 3.12
           - name: Install uv
             uses: astral-sh/setup-uv@v5
-          - name: Build changelog
-            run: pip install yaml-changelog && make changelog
+          - name: Install towncrier
+            run: pip install towncrier
+          - name: Bump version and build changelog
+            run: |
+              python .github/bump_version.py
+              towncrier build --yes --version $(python -c "import re; print(re.search(r'version = \"(.+?)\"', open('pyproject.toml').read()).group(1))")
           - name: Update lockfile
             run: uv lock
-          - name: Preview changelog update
-            run: ".github/get-changelog-diff.sh"
           - name: Update changelog
             uses: EndBug/add-and-commit@v9
             with:
               add: "."
-              message: Update package version
+              message: Update package version

.gitignore

@@ -30,12 +30,12 @@ docs/.ipynb_checkpoints/
 ## ACA PTC state-level uprating factors
 !policyengine_us_data/storage/aca_ptc_multipliers_2022_2024.csv
 
-## Raw input cache for database pipeline
-policyengine_us_data/storage/calibration/raw_inputs/
+## Calibration run outputs (weights, diagnostics, packages, config)
+policyengine_us_data/storage/calibration/
 
 ## Batch processing checkpoints
 completed_*.txt
 
 ## Test fixtures
-!policyengine_us_data/tests/test_local_area_calibration/test_fixture_50hh.h5
+!policyengine_us_data/tests/test_calibration/test_fixture_50hh.h5
 oregon_ctc_analysis.py

CHANGELOG.md

@@ -1,3 +1,17 @@
+## [1.70.0] - 2026-02-26
+
+### Added
+
+- Add end-to-end test for calibration database build pipeline.
+
+
+## [1.69.4] - 2026-02-24
+
+### Changed
+
+- Migrated from changelog_entry.yaml to towncrier fragments to eliminate merge conflicts.
+
+
 # Changelog
 
 All notable changes to this project will be documented in this file.