From 5a4f8dc60adbc550dca6093265b373484ef77ae4 Mon Sep 17 00:00:00 2001 From: grumo35 Date: Mon, 31 Jan 2022 11:24:04 +0100 Subject: [PATCH 001/146] Update __init__.py Added basic proxy support from environment variables, need to add this option in parameter of patchman cli. --- util/__init__.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/util/__init__.py b/util/__init__.py index ddbe63eb..bc669c1e 100644 --- a/util/__init__.py +++ b/util/__init__.py @@ -14,6 +14,7 @@ # # You should have received a copy of the GNU General Public License # along with Patchman. If not, see +from os import getenv import sys import requests @@ -33,6 +34,11 @@ from progressbar import Bar, ETA, Percentage, ProgressBar from patchman.signals import error_message +http_proxy=getenv('http_proxy') +proxies = { + 'http': http_proxy, + 'https': http_proxy, +} if ProgressBar.__dict__.get('maxval'): pbar2 = False @@ -130,7 +136,7 @@ def get_url(url): """ res = None try: - res = requests.get(url, stream=True) + res = requests.get(url,proxies=proxies, stream=True) except requests.exceptions.Timeout: error_message.send(sender=None, text='Timeout - {0!s}'.format(url)) except requests.exceptions.TooManyRedirects: From c62ccb2cc9769d2dd20b0807daa342368b424df0 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Sat, 22 Mar 2025 00:48:33 -0400 Subject: [PATCH 002/146] Update __init__.py --- util/__init__.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/util/__init__.py b/util/__init__.py index f27e232a..457e6782 100644 --- a/util/__init__.py +++ b/util/__init__.py @@ -29,24 +29,23 @@ from time import time from tqdm import tqdm -from patchman.signals import error_message, info_message, debug_message - -http_proxy = getenv('http_proxy') -http_proxy = getenv('https_proxy') -proxies = { - 'http': http_proxy, - 'https': https_proxy, -} - from django.utils.timezone import make_aware from django.utils.dateparse import parse_datetime from django.conf import settings +from patchman.signals import error_message, info_message, debug_message pbar = None verbose = None Checksum = Enum('Checksum', 'md5 sha sha1 sha256 sha512') +http_proxy = getenv('http_proxy') +https_proxy = getenv('https_proxy') +proxies = { + 'http': http_proxy, + 'https': https_proxy, +} + def get_verbosity(): """ Get the global verbosity level From 959d3cbe89697f95dc82ce7d91d5a92075b13234 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Sat, 22 Mar 2025 00:49:38 -0400 Subject: [PATCH 003/146] Update __init__.py --- util/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/util/__init__.py b/util/__init__.py index 457e6782..10ee21f7 100644 --- a/util/__init__.py +++ b/util/__init__.py @@ -14,13 +14,13 @@ # # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -from os import getenv import requests import bz2 import magic import zlib import lzma +import os from datetime import datetime, timezone from enum import Enum from hashlib import md5, sha1, sha256, sha512 @@ -39,8 +39,8 @@ verbose = None Checksum = Enum('Checksum', 'md5 sha sha1 sha256 sha512') -http_proxy = getenv('http_proxy') -https_proxy = getenv('https_proxy') +http_proxy = os.getenv('http_proxy') +https_proxy = os.getenv('https_proxy') proxies = { 'http': http_proxy, 'https': https_proxy, @@ -122,7 +122,7 @@ def get_url(url, headers={}, params={}): response = None try: debug_message.send(sender=None, text=f'Trying {url} headers:{headers} params:{params}') - response = requests.get(url, headers=headers, params=params, stream=True, proxies = proxies, timeout=30) + response = requests.get(url, headers=headers, params=params, stream=True, proxies=proxies, timeout=30) debug_message.send(sender=None, text=f'{response.status_code}: {response.headers}') if response.status_code in [403, 404]: return response From 50d46d215ee2292f3d0c68a24d7e3171f82b4ab8 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Wed, 23 Apr 2025 17:35:03 -0400 Subject: [PATCH 004/146] handle duplicate CVSSes better --- security/models.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/security/models.py b/security/models.py index a847ab02..b60acb47 100644 --- a/security/models.py +++ b/security/models.py @@ -125,19 +125,20 @@ def add_cvss_score(self, vector_string, score=None, severity=None, version=None) score = cvss_score.base_score if not severity: severity = cvss_score.severities()[0] - existing = self.cvss_scores.filter(version=version, vector_string=vector_string) - if existing: - cvss = existing.first() - else: + try: cvss, created = CVSS.objects.get_or_create( version=version, vector_string=vector_string, score=score, severity=severity, ) - cvss.score = score - cvss.severity = severity - cvss.save() + except CVSS.MultipleObjectsReturned: + matching_cvsses = CVSS.objects.filter( + version=version, + vector_string=vector_string, + ) + cvss = matching_cvsses.first() + matching_cvsses.exclude(id=cvss.id).delete() self.cvss_scores.add(cvss) def fetch_cve_data(self, fetch_nist_data=False, sleep_secs=6): From 51201364a391f5c2bb102442a5562cd31eb756fa Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Tue, 8 Apr 2025 15:49:31 -0400 Subject: [PATCH 005/146] reduce max charfield length for mysql --- security/migrations/0001_initial.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/security/migrations/0001_initial.py b/security/migrations/0001_initial.py index 5655f8b0..c22d1727 100644 --- a/security/migrations/0001_initial.py +++ b/security/migrations/0001_initial.py @@ -27,7 +27,7 @@ class Migration(migrations.Migration): ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('cwe_id', models.CharField(max_length=255, unique=True)), ('name', models.CharField(blank=True, max_length=255, null=True)), - ('description', models.CharField(blank=True, max_length=65535, null=True)), + ('description', models.CharField(blank=True, max_length=21844, null=True)), ], ), migrations.CreateModel( @@ -36,7 +36,7 @@ class Migration(migrations.Migration): ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('cve_id', models.CharField(max_length=255, unique=True)), ('title', models.CharField(blank=True, max_length=255, null=True)), - ('description', models.CharField(max_length=65535)), + ('description', models.CharField(max_length=21844)), ('reserved_date', models.DateTimeField(blank=True, null=True)), ('published_date', models.DateTimeField(blank=True, null=True)), ('rejected_date', models.DateTimeField(blank=True, null=True)), From 662d02498d1fbb59ff4eff2dc5e7b8f5f1268061 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Fri, 18 Apr 2025 00:05:34 -0400 Subject: [PATCH 006/146] further reduce charfield size for mysql --- security/migrations/0001_initial.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/security/migrations/0001_initial.py b/security/migrations/0001_initial.py index c22d1727..5f922c9a 100644 --- a/security/migrations/0001_initial.py +++ b/security/migrations/0001_initial.py @@ -27,7 +27,7 @@ class Migration(migrations.Migration): ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('cwe_id', models.CharField(max_length=255, unique=True)), ('name', models.CharField(blank=True, max_length=255, null=True)), - ('description', models.CharField(blank=True, max_length=21844, null=True)), + ('description', models.CharField(blank=True, max_length=255, null=True)), ], ), migrations.CreateModel( @@ -36,7 +36,7 @@ class Migration(migrations.Migration): ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('cve_id', models.CharField(max_length=255, unique=True)), ('title', models.CharField(blank=True, max_length=255, null=True)), - ('description', models.CharField(max_length=21844)), + ('description', models.CharField(max_length=255)), ('reserved_date', models.DateTimeField(blank=True, null=True)), ('published_date', models.DateTimeField(blank=True, null=True)), ('rejected_date', models.DateTimeField(blank=True, null=True)), From 7cff020620dd1d398f48074325f4fa931e90b941 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Tue, 29 Apr 2025 16:10:52 -0400 Subject: [PATCH 007/146] reduce URLField max_length to 765 --- errata/migrations/0001_initial.py | 2 +- security/migrations/0005_reference_cve_references.py | 2 +- security/models.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/errata/migrations/0001_initial.py b/errata/migrations/0001_initial.py index 85fe88b4..d02a7dc8 100644 --- a/errata/migrations/0001_initial.py +++ b/errata/migrations/0001_initial.py @@ -19,7 +19,7 @@ class Migration(migrations.Migration): fields=[ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('er_type', models.CharField(max_length=255)), - ('url', models.URLField(max_length=2000)), + ('url', models.URLField(max_length=765)), ], ), migrations.CreateModel( diff --git a/security/migrations/0005_reference_cve_references.py b/security/migrations/0005_reference_cve_references.py index 97251add..f94cf7d5 100644 --- a/security/migrations/0005_reference_cve_references.py +++ b/security/migrations/0005_reference_cve_references.py @@ -15,7 +15,7 @@ class Migration(migrations.Migration): fields=[ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('ref_type', models.CharField(max_length=255)), - ('url', models.URLField(max_length=2000)), + ('url', models.URLField(max_length=765)), ], options={ 'unique_together': {('ref_type', 'url')}, diff --git a/security/models.py b/security/models.py index b60acb47..9c097eed 100644 --- a/security/models.py +++ b/security/models.py @@ -29,7 +29,7 @@ class Reference(models.Model): ref_type = models.CharField(max_length=255) - url = models.URLField(max_length=2000) + url = models.URLField(max_length=765) class Meta: unique_together = ['ref_type', 'url'] From aeddb403c3efc407d098899343d009cee608e118 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 8 May 2025 15:45:31 +0000 Subject: [PATCH 008/146] Bump django from 4.2.20 to 4.2.21 Bumps [django](https://github.com/django/django) from 4.2.20 to 4.2.21. - [Commits](https://github.com/django/django/compare/4.2.20...4.2.21) --- updated-dependencies: - dependency-name: django dependency-version: 4.2.21 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index dca4fe03..2f72fae5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -Django==4.2.20 +Django==4.2.21 django-taggit==4.0.0 django-extensions==3.2.3 django-bootstrap3==23.1 From f1b92c2b22d587f964de36a5d38635150f161d07 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 6 Jun 2025 23:28:39 +0000 Subject: [PATCH 009/146] Bump django from 4.2.21 to 4.2.22 Bumps [django](https://github.com/django/django) from 4.2.21 to 4.2.22. - [Commits](https://github.com/django/django/compare/4.2.21...4.2.22) --- updated-dependencies: - dependency-name: django dependency-version: 4.2.22 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 2f72fae5..3418b5d0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -Django==4.2.21 +Django==4.2.22 django-taggit==4.0.0 django-extensions==3.2.3 django-bootstrap3==23.1 From 489dfeca351468edef8e3e22662a42d4600af08f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 10 Jun 2025 08:47:24 +0000 Subject: [PATCH 010/146] Bump requests from 2.32.3 to 2.32.4 Bumps [requests](https://github.com/psf/requests) from 2.32.3 to 2.32.4. - [Release notes](https://github.com/psf/requests/releases) - [Changelog](https://github.com/psf/requests/blob/main/HISTORY.md) - [Commits](https://github.com/psf/requests/compare/v2.32.3...v2.32.4) --- updated-dependencies: - dependency-name: requests dependency-version: 2.32.4 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 2f72fae5..955c7bbb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ python-debian==1.0.1 defusedxml==0.7.1 PyYAML==6.0.2 chardet==5.2.0 -requests==2.32.3 +requests==2.32.4 colorama==0.4.6 djangorestframework==3.15.2 django-filter==25.1 From 3559762a23704e4ca04d1dcfc5c5f60f8a206aa2 Mon Sep 17 00:00:00 2001 From: vtalos Date: Thu, 17 Jul 2025 19:33:36 +0300 Subject: [PATCH 011/146] Remove unused dependency 'chardet' from requirements.txt --- requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 088f0870..a39eb8cc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,6 @@ django-bootstrap3==23.1 python-debian==1.0.1 defusedxml==0.7.1 PyYAML==6.0.2 -chardet==5.2.0 requests==2.32.4 colorama==0.4.6 djangorestframework==3.15.2 From 08e37c67a843916362f0af341da23fa63ff4b7d3 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Tue, 5 Aug 2025 20:15:16 -0400 Subject: [PATCH 012/146] get_or_create_module only returns module --- modules/utils.py | 4 ++-- repos/repo_types/yum.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/utils.py b/modules/utils.py index 817a610c..f56a0f62 100644 --- a/modules/utils.py +++ b/modules/utils.py @@ -23,7 +23,7 @@ def get_or_create_module(name, stream, version, context, arch, repo): """ Get or create a module object - Returns the module and a boolean for created + Returns the module """ created = False m_arch, c = PackageArchitecture.objects.get_or_create(name=arch) @@ -46,7 +46,7 @@ def get_or_create_module(name, stream, version, context, arch, repo): arch=m_arch, repo=repo, ) - return module, created + return module def get_matching_modules(name, stream, version, context, arch): diff --git a/repos/repo_types/yum.py b/repos/repo_types/yum.py index d08c7393..7ac85816 100644 --- a/repos/repo_types/yum.py +++ b/repos/repo_types/yum.py @@ -91,7 +91,7 @@ def extract_module_metadata(data, url, repo): packages.add(package) from modules.utils import get_or_create_module - module, created = get_or_create_module(m_name, m_stream, m_version, m_context, arch, repo) + module = get_or_create_module(m_name, m_stream, m_version, m_context, arch, repo) package_ids = [] for package in packages: From 82087ec7f41f5fb0908f1ea56dc03f08902778bd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 10 Sep 2025 02:19:45 +0000 Subject: [PATCH 013/146] Bump django from 4.2.22 to 4.2.24 Bumps [django](https://github.com/django/django) from 4.2.22 to 4.2.24. - [Commits](https://github.com/django/django/compare/4.2.22...4.2.24) --- updated-dependencies: - dependency-name: django dependency-version: 4.2.24 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a39eb8cc..9d2baa9e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -Django==4.2.22 +Django==4.2.24 django-taggit==4.0.0 django-extensions==3.2.3 django-bootstrap3==23.1 From 36decd54c55b614c25c6b9c8fbba9be5ee7926f3 Mon Sep 17 00:00:00 2001 From: Will Furnell Date: Fri, 12 Sep 2025 13:33:45 +0100 Subject: [PATCH 014/146] Package types are in the Package class --- packages/models.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/models.py b/packages/models.py index f4c9c59e..74a83c0c 100644 --- a/packages/models.py +++ b/packages/models.py @@ -195,11 +195,11 @@ def __str__(self): rel = f'-{self.release}' else: rel = '' - if self.packagetype == self.GENTOO: + if self.packagetype == Package.GENTOO: return f'{self.category}/{self.name}-{epo}{self.version}{rel}-{self.arch}.{self.get_packagetype_display()}' - elif self.packagetype in [self.DEB, self.ARCH]: + elif self.packagetype in [Package.DEB, Package.ARCH]: return f'{self.name}_{epo}{self.version}{rel}_{self.arch}.{self.get_packagetype_display()}' - elif self.packagetype == self.RPM: + elif self.packagetype == Package.RPM: return f'{self.name}-{epo}{self.version}{rel}-{self.arch}.{self.get_packagetype_display()}' else: return f'{self.name}-{epo}{self.version}{rel}-{self.arch}.{self.get_packagetype_display()}' From c7e54c55fc3b1f54482fb2c232ea2e961bf8ac8d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 1 Oct 2025 23:00:50 +0000 Subject: [PATCH 015/146] Bump django from 4.2.24 to 4.2.25 Bumps [django](https://github.com/django/django) from 4.2.24 to 4.2.25. - [Commits](https://github.com/django/django/compare/4.2.24...4.2.25) --- updated-dependencies: - dependency-name: django dependency-version: 4.2.25 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 9d2baa9e..9d277d38 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -Django==4.2.24 +Django==4.2.25 django-taggit==4.0.0 django-extensions==3.2.3 django-bootstrap3==23.1 From d52251e37517f67d1d4e969511dda562fa8e7f85 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Fri, 3 Oct 2025 10:54:33 -0400 Subject: [PATCH 016/146] bump redis --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 9d277d38..2f264c9b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,7 +15,7 @@ python-magic==0.4.27 gitpython==3.1.44 tenacity==8.2.3 celery==5.4.0 -redis==5.2.1 +redis==6.4.0 django-celery-beat==2.7.0 tqdm==4.67.1 cvss==3.4 From c3697d897d6ecf493c70f02dd215db48dabe175d Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Fri, 3 Oct 2025 10:59:40 -0400 Subject: [PATCH 017/146] Update license in common.py --- util/templatetags/common.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/util/templatetags/common.py b/util/templatetags/common.py index 6737c438..2aea1e5e 100644 --- a/util/templatetags/common.py +++ b/util/templatetags/common.py @@ -1,12 +1,10 @@ -# Copyright 2010 VPAC -# Copyright 2013-2021 Marcus Furlong +# Copyright 2013-2025 Marcus Furlong # # This file is part of Patchman. # # Patchman is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. +# the Free Software Foundation, version 3 only. # # Patchman is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of @@ -14,7 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with Patchman If not, see . +# along with Patchman. If not, see import re From 80a2417c5bf3616b43d0e45dd33a2173bfdc3bf4 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Fri, 3 Oct 2025 11:11:25 -0400 Subject: [PATCH 018/146] fix licenses --- hosts/templatetags/report_alert.py | 7 +++---- setup.py | 2 +- util/filterspecs.py | 7 +++---- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/hosts/templatetags/report_alert.py b/hosts/templatetags/report_alert.py index 3a3e3a9a..a28c5058 100644 --- a/hosts/templatetags/report_alert.py +++ b/hosts/templatetags/report_alert.py @@ -1,11 +1,10 @@ -# Copyright 2016-2021 Marcus Furlong +# Copyright 2016-2025 Marcus Furlong # # This file is part of Patchman. # # Patchman is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. +# the Free Software Foundation, version 3 only. # # Patchman is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of @@ -13,7 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with Patchman If not, see . +# along with Patchman. If not, see from datetime import timedelta diff --git a/setup.py b/setup.py index d8249a67..6ec6d974 100755 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright 2013-2021 Marcus Furlong +# Copyright 2013-2025 Marcus Furlong # # This file is part of Patchman. # diff --git a/util/filterspecs.py b/util/filterspecs.py index 1c845ff3..722b45df 100644 --- a/util/filterspecs.py +++ b/util/filterspecs.py @@ -1,12 +1,11 @@ # Copyright 2010 VPAC -# Copyright 2014-2021 Marcus Furlong +# Copyright 2014-2025 Marcus Furlong # # This file is part of Patchman. # # Patchman is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. +# the Free Software Foundation, version 3 only. # # Patchman is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of @@ -14,7 +13,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with Patchman If not, see . +# along with Patchman. If not, see from django.utils.safestring import mark_safe from django.db.models.query import QuerySet From 2a9b67afb5f3f115e1c13295a118c92b07192030 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Fri, 3 Oct 2025 11:57:40 -0400 Subject: [PATCH 019/146] use GPL-3.0-only for debian copyright --- debian/copyright | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/copyright b/debian/copyright index ab051037..5202ff0e 100644 --- a/debian/copyright +++ b/debian/copyright @@ -6,7 +6,7 @@ Source: https://github.com/furlongm/patchman Files: * Copyright: 2011-2012 VPAC http://www.vpac.org 2013-2021 Marcus Furlong -License: GPL-3.0 +License: GPL-3.0-only This package is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; version 3 only. From 7ae40dc9e8d3651ef8b7d98c2b5ae7d4e0941528 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Mon, 20 Oct 2025 20:47:26 -0400 Subject: [PATCH 020/146] fix tag handling Signed-off-by: Marcus Furlong --- hosts/models.py | 2 +- hosts/utils.py | 19 +++++++++++++++++-- sbin/patchman | 2 ++ 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/hosts/models.py b/hosts/models.py index a6c451b5..5b7b3979 100644 --- a/hosts/models.py +++ b/hosts/models.py @@ -85,7 +85,7 @@ def show(self): text += f'Packages : {self.get_num_packages()}\n' text += f'Repos : {self.get_num_repos()}\n' text += f'Updates : {self.get_num_updates()}\n' - text += f'Tags : {self.tags}\n' + text += f'Tags : {" ".join(self.tags.slugs())}\n' text += f'Needs reboot : {self.reboot_required}\n' text += f'Updated at : {self.updated_at}\n' text += f'Host repos : {self.host_repos_only}\n' diff --git a/hosts/utils.py b/hosts/utils.py index b328129f..f07d5d1e 100644 --- a/hosts/utils.py +++ b/hosts/utils.py @@ -18,8 +18,9 @@ from socket import gethostbyaddr, gaierror, herror from django.db import transaction, IntegrityError +from taggit.models import Tag -from patchman.signals import error_message +from patchman.signals import error_message, info_message def update_rdns(host): @@ -62,7 +63,7 @@ def get_or_create_host(report, arch, osvariant, domain): host.osvariant = osvariant host.domain = domain host.lastreport = report.created - host.tags = report.tags + host.tags.set(report.tags.split(','), clear=True) if report.reboot == 'True': host.reboot_required = True else: @@ -73,3 +74,17 @@ def get_or_create_host(report, arch, osvariant, domain): if host: host.check_rdns() return host + + +def clean_tags(): + """ Delete Tags that have no Host + """ + tags = Tag.objects.filter( + host__isnull=True, + ) + tlen = tags.count() + if tlen == 0: + info_message.send(sender=None, text='No orphaned Tags found.') + else: + info_message.send(sender=None, text=f'{tlen} orphaned Tags found.') + tags.delete() diff --git a/sbin/patchman b/sbin/patchman index 9cc6048e..c0911434 100755 --- a/sbin/patchman +++ b/sbin/patchman @@ -34,6 +34,7 @@ from errata.utils import mark_errata_security_updates, enrich_errata, \ scan_package_updates_for_affected_packages from errata.tasks import update_errata from hosts.models import Host +from hosts.utils import clean_tags from modules.utils import clean_modules from packages.utils import clean_packages, clean_packageupdates, clean_packagenames from repos.models import Repository @@ -362,6 +363,7 @@ def dbcheck(remove_duplicates=False): clean_repos() clean_modules() clean_packageupdates() + clean_tags() def collect_args(): From ee13ad310bb86789f78f110eb2d82dbd228a5d86 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Tue, 21 Oct 2025 22:35:58 -0400 Subject: [PATCH 021/146] fix some flake8-bugbear bugs Signed-off-by: Marcus Furlong --- patchman/urls.py | 2 +- repos/repo_types/gentoo.py | 2 +- repos/repo_types/rpm.py | 2 +- util/__init__.py | 6 +++++- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/patchman/urls.py b/patchman/urls.py index 337d6b63..ee786566 100644 --- a/patchman/urls.py +++ b/patchman/urls.py @@ -44,7 +44,7 @@ router.register(r'package', package_views.PackageViewSet) router.register(r'package-update', package_views.PackageUpdateViewSet) router.register(r'cve', security_views.CVEViewSet) -router.register(r'reference', security_views.ReferenceViewSet), +router.register(r'reference', security_views.ReferenceViewSet) router.register(r'erratum', errata_views.ErratumViewSet) router.register(r'repo', repo_views.RepositoryViewSet) router.register(r'mirror', repo_views.MirrorViewSet) diff --git a/repos/repo_types/gentoo.py b/repos/repo_types/gentoo.py index 94df139a..8e4198d9 100644 --- a/repos/repo_types/gentoo.py +++ b/repos/repo_types/gentoo.py @@ -226,7 +226,7 @@ def extract_gentoo_overlay_ebuilds(t): """ Extract ebuilds from a Gentoo overlay tarball """ extracted_ebuilds = {} - for root, dirs, files in os.walk(t): + for root, _, files in os.walk(t): for name in files: if fnmatch(name, '*.ebuild'): package_name = root.replace(t + '/', '') diff --git a/repos/repo_types/rpm.py b/repos/repo_types/rpm.py index aa3354c7..d9501cde 100644 --- a/repos/repo_types/rpm.py +++ b/repos/repo_types/rpm.py @@ -69,7 +69,7 @@ def refresh_rpm_repo_mirrors(repo, errata_only=False): ] ts = get_datetime_now() enabled_mirrors = repo.mirror_set.filter(mirrorlist=False, refresh=True, enabled=True) - for i, mirror in enumerate(enabled_mirrors): + for mirror in enabled_mirrors: res = find_mirror_url(mirror.url, formats) if not res: mirror.fail() diff --git a/util/__init__.py b/util/__init__.py index a56ed3b6..4958dfb0 100644 --- a/util/__init__.py +++ b/util/__init__.py @@ -107,10 +107,14 @@ def fetch_content(response, text='', ljust=35): wait=wait_exponential(multiplier=1, min=1, max=10), reraise=False, ) -def get_url(url, headers={}, params={}): +def get_url(url, headers=None, params=None): """ Perform a http GET on a URL. Return None on error. """ response = None + if not headers: + headers = {} + if not params: + params = {} try: debug_message.send(sender=None, text=f'Trying {url} headers:{headers} params:{params}') response = requests.get(url, headers=headers, params=params, stream=True, timeout=30) From 346b02ca119b0197ba434bfdeb45de4448d21a92 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Wed, 29 Oct 2025 23:34:45 -0400 Subject: [PATCH 022/146] fix package filter list for errata --- packages/views.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/packages/views.py b/packages/views.py index cd53fa6e..c55a6c72 100644 --- a/packages/views.py +++ b/packages/views.py @@ -62,9 +62,16 @@ def package_list(request): if 'affected_by_errata' in request.GET: affected_by_errata = request.GET['affected_by_errata'] == 'true' if affected_by_errata: - packages = packages.filter(erratum__isnull=False) + packages = packages.filter(affected_by_erratum__isnull=False) else: - packages = packages.filter(erratum__isnull=True) + packages = packages.filter(affected_by_erratum__isnull=True) + + if 'provides_fix_in_erratum' in request.GET: + provides_fix_in_erratum = request.GET['provides_fix_in_erratum'] == 'true' + if provides_fix_in_erratum: + packages = packages.filter(provides_fix_in_erratum__isnull=False) + else: + packages = packages.filter(provides_fix_in_erratum__isnull=True) if 'installed_on_hosts' in request.GET: installed_on_hosts = request.GET['installed_on_hosts'] == 'true' @@ -102,6 +109,8 @@ def package_list(request): filter_list = [] filter_list.append(Filter(request, 'Affected by Errata', 'affected_by_errata', {'true': 'Yes', 'false': 'No'})) + filter_list.append(Filter(request, 'Provides Fix in Errata', 'provides_fix_in_erratum', + {'true': 'Yes', 'false': 'No'})) filter_list.append(Filter(request, 'Installed on Hosts', 'installed_on_hosts', {'true': 'Yes', 'false': 'No'})) filter_list.append(Filter(request, 'Available in Repos', 'available_in_repos', {'true': 'Yes', 'false': 'No'})) filter_list.append(Filter(request, 'Package Type', 'packagetype', Package.PACKAGE_TYPES)) From c4caf0ce60cedcedf27b6f53efe098b6837b92b4 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Wed, 29 Oct 2025 23:34:29 -0400 Subject: [PATCH 023/146] add support for zstd compression in deb and rpm repos fixes: #698 Signed-off-by: Marcus Furlong --- debian/control | 2 +- repos/repo_types/deb.py | 7 ++++++- repos/repo_types/rpm.py | 2 ++ requirements.txt | 1 + setup.cfg | 1 + util/__init__.py | 17 +++++++++++++++++ 6 files changed, 28 insertions(+), 2 deletions(-) diff --git a/debian/control b/debian/control index 67026269..7bfe320e 100644 --- a/debian/control +++ b/debian/control @@ -20,7 +20,7 @@ Depends: ${misc:Depends}, python3 (>= 3.11), python3-django (>= 4.2), python3-requests, python3-colorama, python3-magic, python3-humanize, python3-yaml, libapache2-mod-wsgi-py3, apache2, sqlite3, celery, python3-celery, python3-django-celery-beat, redis-server, - python3-redis, python3-git, python3-django-taggit + python3-redis, python3-git, python3-django-taggit, python3-zstandard Suggests: python3-mysqldb, python3-psycopg2, python3-pymemcache, memcached Description: Django-based patch status monitoring tool for linux systems. . diff --git a/repos/repo_types/deb.py b/repos/repo_types/deb.py index 25d8eba7..1d3607c5 100644 --- a/repos/repo_types/deb.py +++ b/repos/repo_types/deb.py @@ -71,7 +71,12 @@ def refresh_deb_repo(repo): are and then fetches and extracts packages from those files. """ - formats = ['Packages.xz', 'Packages.bz2', 'Packages.gz', 'Packages'] + formats = [ + 'Packages.xz', + 'Packages.bz2', + 'Packages.gz', + 'Packages', + ] ts = get_datetime_now() enabled_mirrors = repo.mirror_set.filter(refresh=True, enabled=True) diff --git a/repos/repo_types/rpm.py b/repos/repo_types/rpm.py index d9501cde..93d47007 100644 --- a/repos/repo_types/rpm.py +++ b/repos/repo_types/rpm.py @@ -57,10 +57,12 @@ def refresh_rpm_repo_mirrors(repo, errata_only=False): which type of repo it is, then refreshes the mirrors """ formats = [ + 'repodata/repomd.xml.zst', 'repodata/repomd.xml.xz', 'repodata/repomd.xml.bz2', 'repodata/repomd.xml.gz', 'repodata/repomd.xml', + 'suse/repodata/repomd.xml.zst', 'suse/repodata/repomd.xml.xz', 'suse/repodata/repomd.xml.bz2', 'suse/repodata/repomd.xml.gz', diff --git a/requirements.txt b/requirements.txt index 2f264c9b..788f4241 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,3 +19,4 @@ redis==6.4.0 django-celery-beat==2.7.0 tqdm==4.67.1 cvss==3.4 +zstandard==0.25.0 diff --git a/setup.cfg b/setup.cfg index 7af9ccb0..b1d5ee4e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -25,6 +25,7 @@ requires = /usr/bin/python3 python3-importlib-metadata python3-cvss python3-redis + python3-zstandard redis celery python3-django-celery-beat diff --git a/util/__init__.py b/util/__init__.py index ac6f8f1b..4a3f9caa 100644 --- a/util/__init__.py +++ b/util/__init__.py @@ -21,6 +21,11 @@ import zlib import lzma import os +try: + # python 3.14+ - can also remove the dependency at that stage + from compression import zstd +except ImportError: + import zstandard as zstd from datetime import datetime, timezone from enum import Enum from hashlib import md5, sha1, sha256, sha512 @@ -202,6 +207,16 @@ def unxz(contents): error_message.send(sender=None, text='lzma: ' + e) +def unzstd(contents): + """ unzstd contents in memory and return the data + """ + try: + zstddata = zstd.ZstdDecompressor().stream_reader(contents).read() + return zstddata + except zstd.ZstdError as e: + error_message.send(sender=None, text='zstd: ' + e) + + def extract(data, fmt): """ Extract the contents based on mimetype or file ending. Return the unmodified data if neither mimetype nor file ending matches, otherwise @@ -214,6 +229,8 @@ def extract(data, fmt): m = magic.open(magic.MAGIC_MIME) m.load() mime = m.buffer(data).split(';')[0] + if mime == 'application/zstd' or fmt.endswith('zst'): + return unzstd(data) if mime == 'application/x-xz' or fmt.endswith('xz'): return unxz(data) elif mime == 'application/x-bzip2' or fmt.endswith('bz2'): From b90a04bb225144817d4b5ad71874dc3a420231e2 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Wed, 29 Oct 2025 23:34:41 -0400 Subject: [PATCH 024/146] simplify logging --- arch/utils.py | 10 ++--- errata/models.py | 12 ++--- errata/sources/distros/arch.py | 5 ++- errata/sources/distros/centos.py | 5 ++- errata/sources/distros/debian.py | 7 +-- errata/sources/distros/rocky.py | 11 ++--- errata/sources/distros/ubuntu.py | 7 +-- errata/sources/repos/yum.py | 5 ++- errata/tasks.py | 2 +- errata/utils.py | 9 ++-- hosts/models.py | 14 +++--- hosts/tasks.py | 4 +- hosts/utils.py | 8 ++-- modules/utils.py | 8 ++-- packages/utils.py | 26 +++++------ patchman/receivers.py | 21 ++++----- patchman/signals.py | 8 ++-- reports/models.py | 10 ++--- reports/tasks.py | 4 +- reports/utils.py | 5 ++- repos/models.py | 18 ++++---- repos/repo_types/arch.py | 11 ++--- repos/repo_types/deb.py | 11 ++--- repos/repo_types/gentoo.py | 15 ++++--- repos/repo_types/rpm.py | 8 ++-- repos/repo_types/yast.py | 5 ++- repos/repo_types/yum.py | 23 +++++----- repos/utils.py | 37 ++++++++-------- sbin/patchman | 76 ++++++++++++++++---------------- security/models.py | 8 ++-- util/__init__.py | 24 +++++----- util/logging.py | 42 ++++++++++++++++++ 32 files changed, 258 insertions(+), 201 deletions(-) create mode 100644 util/logging.py diff --git a/arch/utils.py b/arch/utils.py index 1498fdec..04d0b350 100644 --- a/arch/utils.py +++ b/arch/utils.py @@ -15,7 +15,7 @@ # along with Patchman. If not, see from arch.models import PackageArchitecture, MachineArchitecture -from patchman.signals import info_message +from util.logging import info_message def clean_package_architectures(): @@ -24,9 +24,9 @@ def clean_package_architectures(): parches = PackageArchitecture.objects.filter(package__isnull=True) plen = parches.count() if plen == 0: - info_message.send(sender=None, text='No orphaned PackageArchitectures found.') + info_message(text='No orphaned PackageArchitectures found.') else: - info_message.send(sender=None, text=f'Removing {plen} orphaned PackageArchitectures') + info_message(text=f'Removing {plen} orphaned PackageArchitectures') parches.delete() @@ -39,9 +39,9 @@ def clean_machine_architectures(): ) mlen = marches.count() if mlen == 0: - info_message.send(sender=None, text='No orphaned MachineArchitectures found.') + info_message(text='No orphaned MachineArchitectures found.') else: - info_message.send(sender=None, text=f'Removing {mlen} orphaned MachineArchitectures') + info_message(text=f'Removing {mlen} orphaned MachineArchitectures') marches.delete() diff --git a/errata/models.py b/errata/models.py index b10daf4d..cfc9bd0d 100644 --- a/errata/models.py +++ b/errata/models.py @@ -25,7 +25,7 @@ from errata.managers import ErratumManager from security.models import CVE, Reference from security.utils import get_or_create_cve, get_or_create_reference -from patchman.signals import error_message +from util.logging import error_message from util import get_url @@ -70,7 +70,7 @@ def scan_for_security_updates(self): try: affected_update.save() except IntegrityError as e: - error_message.send(sender=None, text=e) + error_message(text=e) # a version of this update already exists that is # marked as a security update, so delete this one affected_update.delete() @@ -84,7 +84,7 @@ def scan_for_security_updates(self): try: affected_update.save() except IntegrityError as e: - error_message.send(sender=None, text=e) + error_message(text=e) # a version of this update already exists that is # marked as a security update, so delete this one affected_update.delete() @@ -93,7 +93,7 @@ def fetch_osv_dev_data(self): osv_dev_url = f'https://api.osv.dev/v1/vulns/{self.name}' res = get_url(osv_dev_url) if res.status_code == 404: - error_message.send(sender=None, text=f'404 - Skipping {self.name} - {osv_dev_url}') + error_message(text=f'404 - Skipping {self.name} - {osv_dev_url}') return data = res.content osv_dev_json = json.loads(data) @@ -102,7 +102,7 @@ def fetch_osv_dev_data(self): def parse_osv_dev_data(self, osv_dev_json): name = osv_dev_json.get('id') if name != self.name: - error_message.send(sender=None, text=f'Erratum name mismatch - {self.name} != {name}') + error_message(text=f'Erratum name mismatch - {self.name} != {name}') return related = osv_dev_json.get('related') if related: @@ -155,7 +155,7 @@ def add_cve(self, cve_id): """ Add a CVE to an Erratum object """ if not cve_id.startswith('CVE') or not cve_id.split('-')[1].isdigit(): - error_message.send(sender=None, text=f'Not a CVE ID: {cve_id}') + error_message(text=f'Not a CVE ID: {cve_id}') return self.cves.add(get_or_create_cve(cve_id)) diff --git a/errata/sources/distros/arch.py b/errata/sources/distros/arch.py index 40d0dada..87c6c47a 100644 --- a/errata/sources/distros/arch.py +++ b/errata/sources/distros/arch.py @@ -20,7 +20,8 @@ from django.db import connections from operatingsystems.utils import get_or_create_osrelease -from patchman.signals import error_message, pbar_start, pbar_update +from util.logging import error_message +from patchman.signals import pbar_start, pbar_update from packages.models import Package from packages.utils import find_evr, get_matching_packages, get_or_create_package from util import get_url, fetch_content @@ -99,7 +100,7 @@ def process_arch_erratum(advisory, osrelease): add_arch_erratum_references(e, advisory) add_arch_erratum_packages(e, advisory) except Exception as exc: - error_message.send(sender=None, text=exc) + error_message(text=exc) def add_arch_linux_osrelease(): diff --git a/errata/sources/distros/centos.py b/errata/sources/distros/centos.py index eefb2b88..d2722a6b 100644 --- a/errata/sources/distros/centos.py +++ b/errata/sources/distros/centos.py @@ -20,7 +20,8 @@ from operatingsystems.utils import get_or_create_osrelease from packages.models import Package from packages.utils import parse_package_string, get_or_create_package -from patchman.signals import error_message, pbar_start, pbar_update +from util.logging import error_message +from patchman.signals import pbar_start, pbar_update from util import bunzip2, get_url, fetch_content, get_sha1, get_setting_of_type @@ -34,7 +35,7 @@ def update_centos_errata(): if actual_checksum != expected_checksum: e = 'CEFS checksum mismatch, skipping CentOS errata parsing\n' e += f'{actual_checksum} (actual) != {expected_checksum} (expected)' - error_message.send(sender=None, text=e) + error_message(text=e) else: if data: parse_centos_errata(bunzip2(data)) diff --git a/errata/sources/distros/debian.py b/errata/sources/distros/debian.py index 93ae2bd5..1ae919e4 100644 --- a/errata/sources/distros/debian.py +++ b/errata/sources/distros/debian.py @@ -27,7 +27,8 @@ from operatingsystems.utils import get_or_create_osrelease from packages.models import Package from packages.utils import get_or_create_package, find_evr -from patchman.signals import error_message, pbar_start, pbar_update, warning_message +from util.logging import error_message, warning_message +from patchman.signals import pbar_start, pbar_update from util import get_url, fetch_content, get_setting_of_type, extract DSCs = {} @@ -217,7 +218,7 @@ def process_debian_erratum(erratum, accepted_codenames): for package in packages: process_debian_erratum_fixed_packages(e, package) except Exception as exc: - error_message.send(sender=None, text=exc) + error_message(text=exc) def parse_debian_erratum_package(line, accepted_codenames): @@ -249,7 +250,7 @@ def fetch_debian_dsc_package_list(package, version): """ Fetch the package list from a DSC file for a given source package/version """ if not DSCs.get(package) or not DSCs[package].get(version): - warning_message.send(sender=None, text=f'No DSC found for {package} {version}') + warning_message(text=f'No DSC found for {package} {version}') return source_url = DSCs[package][version]['url'] res = get_url(source_url) diff --git a/errata/sources/distros/rocky.py b/errata/sources/distros/rocky.py index 693d7b0c..16d4d12c 100644 --- a/errata/sources/distros/rocky.py +++ b/errata/sources/distros/rocky.py @@ -25,7 +25,8 @@ from packages.models import Package from packages.utils import parse_package_string, get_or_create_package from patchman.signals import pbar_start, pbar_update -from util import get_url, fetch_content, info_message, error_message +from util import get_url, fetch_content +from util.logging import info_message, error_message def update_rocky_errata(concurrent_processing=True): @@ -50,16 +51,16 @@ def check_rocky_errata_endpoint_health(rocky_errata_api_host): health = json.loads(data) if health.get('status') == 'ok': s = f'Rocky Errata API healthcheck OK: {rocky_errata_healthcheck_url}' - info_message.send(sender=None, text=s) + info_message(text=s) return True else: s = f'Rocky Errata API healthcheck FAILED: {rocky_errata_healthcheck_url}' - error_message.send(sender=None, text=s) + error_message(text=s) return False except Exception as e: s = f'Rocky Errata API healthcheck exception occured: {rocky_errata_healthcheck_url}\n' s += str(e) - error_message.send(sender=None, text=s) + error_message(text=s) return False @@ -194,7 +195,7 @@ def process_rocky_erratum(advisory): add_rocky_erratum_oses(e, advisory) add_rocky_erratum_packages(e, advisory) except Exception as exc: - error_message.send(sender=None, text=exc) + error_message(text=exc) def add_rocky_erratum_references(e, advisory): diff --git a/errata/sources/distros/ubuntu.py b/errata/sources/distros/ubuntu.py index 7f50962c..d1ce7cc5 100644 --- a/errata/sources/distros/ubuntu.py +++ b/errata/sources/distros/ubuntu.py @@ -28,7 +28,8 @@ from packages.models import Package from packages.utils import get_or_create_package, parse_package_string, find_evr, get_matching_packages from util import get_url, fetch_content, get_sha256, bunzip2, get_setting_of_type -from patchman.signals import error_message, pbar_start, pbar_update +from util.logging import error_message +from patchman.signals import pbar_start, pbar_update def update_ubuntu_errata(concurrent_processing=False): @@ -45,7 +46,7 @@ def update_ubuntu_errata(concurrent_processing=False): else: e = 'Ubuntu USN DB checksum mismatch, skipping Ubuntu errata parsing\n' e += f'{actual_checksum} (actual) != {expected_checksum} (expected)' - error_message.send(sender=None, text=e) + error_message(text=e) def fetch_ubuntu_usn_db(): @@ -126,7 +127,7 @@ def process_usn(usn_id, advisory, accepted_releases): add_ubuntu_erratum_references(e, usn_id, advisory) add_ubuntu_erratum_packages(e, advisory) except Exception as exc: - error_message.send(sender=None, text=exc) + error_message(text=exc) def add_ubuntu_erratum_osreleases(e, affected_releases, accepted_releases): diff --git a/errata/sources/repos/yum.py b/errata/sources/repos/yum.py index dfeed879..f361d10e 100644 --- a/errata/sources/repos/yum.py +++ b/errata/sources/repos/yum.py @@ -23,7 +23,8 @@ from operatingsystems.utils import get_or_create_osrelease from packages.models import Package from packages.utils import get_or_create_package -from patchman.signals import pbar_start, pbar_update, error_message +from util.logging import error_message +from patchman.signals import pbar_start, pbar_update from security.models import Reference from util import extract, get_url @@ -38,7 +39,7 @@ def extract_updateinfo(data, url, concurrent_processing=True): elen = root.__len__() updates = root.findall('update') except ElementTree.ParseError as e: - error_message.send(sender=None, text=f'Error parsing updateinfo file from {url} : {e}') + error_message(text=f'Error parsing updateinfo file from {url} : {e}') if concurrent_processing: extract_updateinfo_concurrently(updates, elen) else: diff --git a/errata/tasks.py b/errata/tasks.py index fe53b415..f1d6eeee 100644 --- a/errata/tasks.py +++ b/errata/tasks.py @@ -22,7 +22,7 @@ from errata.sources.distros.centos import update_centos_errata from errata.sources.distros.rocky import update_rocky_errata from errata.sources.distros.ubuntu import update_ubuntu_errata -from patchman.signals import error_message +from util.logging import error_message from repos.models import Repository from security.tasks import update_cves, update_cwes from util import get_setting_of_type diff --git a/errata/utils.py b/errata/utils.py index d8099db4..a8d8d424 100644 --- a/errata/utils.py +++ b/errata/utils.py @@ -21,7 +21,8 @@ from util import tz_aware_datetime from errata.models import Erratum from packages.models import PackageUpdate -from patchman.signals import pbar_start, pbar_update, warning_message +from util.logging import warning_message +from patchman.signals import pbar_start, pbar_update def get_or_create_erratum(name, e_type, issue_date, synopsis): @@ -36,16 +37,16 @@ def get_or_create_erratum(name, e_type, issue_date, synopsis): days_delta = abs(e.issue_date.date() - issue_date_tz.date()).days updated = False if e.e_type != e_type: - warning_message.send(sender=None, text=f'Updating {name} type `{e.e_type}` -> `{e_type}`') + warning_message(text=f'Updating {name} type `{e.e_type}` -> `{e_type}`') e.e_type = e_type updated = True if days_delta > 1: text = f'Updating {name} issue date `{e.issue_date.date()}` -> `{issue_date_tz.date()}`' - warning_message.send(sender=None, text=text) + warning_message(text=text) e.issue_date = issue_date_tz updated = True if e.synopsis != synopsis: - warning_message.send(sender=None, text=f'Updating {name} synopsis `{e.synopsis}` -> `{synopsis}`') + warning_message(text=f'Updating {name} synopsis `{e.synopsis}` -> `{synopsis}`') e.synopsis = synopsis updated = True if updated: diff --git a/hosts/models.py b/hosts/models.py index 5b7b3979..650544dc 100644 --- a/hosts/models.py +++ b/hosts/models.py @@ -34,7 +34,7 @@ from operatingsystems.models import OSVariant from packages.models import Package, PackageUpdate from packages.utils import get_or_create_package_update -from patchman.signals import info_message +from util.logging import info_message from repos.models import Repository from repos.utils import find_best_repo @@ -90,7 +90,7 @@ def show(self): text += f'Updated at : {self.updated_at}\n' text += f'Host repos : {self.host_repos_only}\n' - info_message.send(sender=None, text=text) + info_message(text=text) def get_absolute_url(self): return reverse('hosts:host_detail', args=[self.hostname]) @@ -114,13 +114,13 @@ def check_rdns(self): if self.check_dns: update_rdns(self) if self.hostname.lower() == self.reversedns.lower(): - info_message.send(sender=None, text='Reverse DNS matches') + info_message(text='Reverse DNS matches') else: text = 'Reverse DNS mismatch found: ' text += f'{self.hostname} != {self.reversedns}' - info_message.send(sender=None, text=text) + info_message(text=text) else: - info_message.send(sender=None, text='Reverse DNS check disabled') + info_message(text='Reverse DNS check disabled') def clean_reports(self): """ Remove all but the last 3 reports for a host @@ -131,7 +131,7 @@ def clean_reports(self): for report in Report.objects.filter(host=self).order_by('-created')[3:]: report.delete() if rlen > 0: - info_message.send(sender=None, text=f'{self.hostname}: removed {rlen} old reports') + info_message(text=f'{self.hostname}: removed {rlen} old reports') def get_host_repo_packages(self): if self.host_repos_only: @@ -163,7 +163,7 @@ def process_update(self, package, highest_package): security = True update = get_or_create_package_update(oldpackage=package, newpackage=highest_package, security=security) self.updates.add(update) - info_message.send(sender=None, text=f'{update}') + info_message(text=f'{update}') return update.id def find_updates(self): diff --git a/hosts/tasks.py b/hosts/tasks.py index 2fdce96f..1643901d 100755 --- a/hosts/tasks.py +++ b/hosts/tasks.py @@ -20,7 +20,7 @@ from hosts.models import Host from util import get_datetime_now -from patchman.signals import info_message +from util.logging import info_message @shared_task @@ -78,4 +78,4 @@ def find_all_host_updates_homogenous(): phost.updated_at = ts phost.save() updated_hosts.append(phost) - info_message.send(sender=None, text=f'Added the same updates to {phost}') + info_message(text=f'Added the same updates to {phost}') diff --git a/hosts/utils.py b/hosts/utils.py index f07d5d1e..d6e663cf 100644 --- a/hosts/utils.py +++ b/hosts/utils.py @@ -20,7 +20,7 @@ from django.db import transaction, IntegrityError from taggit.models import Tag -from patchman.signals import error_message, info_message +from util.logging import error_message, info_message def update_rdns(host): @@ -70,7 +70,7 @@ def get_or_create_host(report, arch, osvariant, domain): host.reboot_required = False host.save() except IntegrityError as e: - error_message.send(sender=None, text=e) + error_message(text=e) if host: host.check_rdns() return host @@ -84,7 +84,7 @@ def clean_tags(): ) tlen = tags.count() if tlen == 0: - info_message.send(sender=None, text='No orphaned Tags found.') + info_message(text='No orphaned Tags found.') else: - info_message.send(sender=None, text=f'{tlen} orphaned Tags found.') + info_message(text=f'{tlen} orphaned Tags found.') tags.delete() diff --git a/modules/utils.py b/modules/utils.py index f56a0f62..05c57c80 100644 --- a/modules/utils.py +++ b/modules/utils.py @@ -15,7 +15,7 @@ # along with Patchman. If not, see from django.db import IntegrityError -from patchman.signals import error_message, info_message +from util.logging import error_message, info_message from modules.models import Module from arch.models import PackageArchitecture @@ -37,7 +37,7 @@ def get_or_create_module(name, stream, version, context, arch, repo): repo=repo, ) except IntegrityError as e: - error_message.send(sender=None, text=e) + error_message(text=e) module = Module.objects.get( name=name, stream=stream, @@ -73,7 +73,7 @@ def clean_modules(): ) mlen = modules.count() if mlen == 0: - info_message.send(sender=None, text='No orphaned Modules found.') + info_message(text='No orphaned Modules found.') else: - info_message.send(sender=None, text=f'{mlen} orphaned Modules found.') + info_message(text=f'{mlen} orphaned Modules found.') modules.delete() diff --git a/packages/utils.py b/packages/utils.py index 9b098225..f00f6710 100644 --- a/packages/utils.py +++ b/packages/utils.py @@ -22,7 +22,7 @@ from arch.models import PackageArchitecture from packages.models import PackageName, Package, PackageUpdate, PackageCategory, PackageString -from patchman.signals import error_message, info_message, warning_message +from util.logging import error_message, info_message, warning_message def convert_package_to_packagestring(package): @@ -141,7 +141,7 @@ def parse_redhat_package_string(pkg_str): name, epoch, ver, rel, dist, arch = m.groups() else: e = f'Error parsing package string: "{pkg_str}"' - error_message.send(sender=None, text=e) + error_message(text=e) return if dist: rel = f'{rel}.{dist}' @@ -195,7 +195,7 @@ def get_or_create_package(name, epoch, version, release, arch, p_type): package = packages.first() # TODO this should handle gentoo package categories too, otherwise we may be deleting packages # that should be kept - warning_message.send(sender=None, text=f'Deleting duplicate packages: {packages.exclude(id=package.id)}') + warning_message(text=f'Deleting duplicate packages: {packages.exclude(id=package.id)}') packages.exclude(id=package.id).delete() return package @@ -218,10 +218,10 @@ def get_or_create_package_update(oldpackage, newpackage, security): except MultipleObjectsReturned: e = 'Error: MultipleObjectsReturned when attempting to add package \n' e += f'update with oldpackage={oldpackage} | newpackage={newpackage}:' - error_message.send(sender=None, text=e) + error_message(text=e) updates = PackageUpdate.objects.filter(oldpackage=oldpackage, newpackage=newpackage) for update in updates: - error_message.send(sender=None, text=str(update)) + error_message(text=str(update)) return try: if update: @@ -281,13 +281,13 @@ def clean_packageupdates(): for update in package_updates: if update.host_set.count() == 0: text = f'Removing unused PackageUpdate {update}' - info_message.send(sender=None, text=text) + info_message(text=text) update.delete() for duplicate in package_updates: if update.oldpackage == duplicate.oldpackage and update.newpackage == duplicate.newpackage and \ update.security == duplicate.security and update.id != duplicate.id: text = f'Removing duplicate PackageUpdate: {update}' - info_message.send(sender=None, text=text) + info_message(text=text) for host in duplicate.host_set.all(): host.updates.remove(duplicate) host.updates.add(update) @@ -307,12 +307,12 @@ def clean_packages(remove_duplicates=False): ) plen = packages.count() if plen == 0: - info_message.send(sender=None, text='No orphaned Packages found.') + info_message(text='No orphaned Packages found.') else: - info_message.send(sender=None, text=f'Removing {plen} orphaned Packages') + info_message(text=f'Removing {plen} orphaned Packages') packages.delete() if remove_duplicates: - info_message.send(sender=None, text='Checking for duplicate Packages...') + info_message(text='Checking for duplicate Packages...') for package in Package.objects.all(): potential_duplicates = Package.objects.filter( name=package.name, @@ -326,7 +326,7 @@ def clean_packages(remove_duplicates=False): if potential_duplicates.count() > 1: for dupe in potential_duplicates: if dupe.id != package.id: - info_message.send(sender=None, text=f'Removing duplicate Package {dupe}') + info_message(text=f'Removing duplicate Package {dupe}') dupe.delete() @@ -336,7 +336,7 @@ def clean_packagenames(): names = PackageName.objects.filter(package__isnull=True) nlen = names.count() if nlen == 0: - info_message.send(sender=None, text='No orphaned PackageNames found.') + info_message(text='No orphaned PackageNames found.') else: - info_message.send(sender=None, text=f'Removing {nlen} orphaned PackageNames') + info_message(text=f'Removing {nlen} orphaned PackageNames') names.delete() diff --git a/patchman/receivers.py b/patchman/receivers.py index 5ec32cdd..8d8893ca 100644 --- a/patchman/receivers.py +++ b/patchman/receivers.py @@ -21,7 +21,8 @@ from django.dispatch import receiver from util import create_pbar, update_pbar, get_verbosity -from patchman.signals import pbar_start, pbar_update, info_message, warning_message, error_message, debug_message +from patchman.signals import pbar_start, pbar_update, \ + info_message_s, warning_message_s, error_message_s, debug_message_s from django.conf import settings @@ -47,36 +48,36 @@ def pbar_update_receiver(**kwargs): update_pbar(index) -@receiver(info_message) -def print_info_message(sender=None, **kwargs): - """ Receiver to print an info message, no color +@receiver(info_message_s) +def print_info_message(**kwargs): + """ Receiver to handle an info message, no color """ text = str(kwargs.get('text')) if get_verbosity(): tqdm.write(Style.RESET_ALL + Fore.RESET + text) -@receiver(warning_message) +@receiver(warning_message_s) def print_warning_message(**kwargs): - """ Receiver to print a warning message in yellow text + """ Receiver to handle a warning message, yellow text """ text = str(kwargs.get('text')) if get_verbosity(): tqdm.write(Style.BRIGHT + Fore.YELLOW + text) -@receiver(error_message) +@receiver(error_message_s) def print_error_message(**kwargs): - """ Receiver to print an error message in red text + """ Receiver to handle an error message, red text """ text = str(kwargs.get('text')) if text: tqdm.write(Style.BRIGHT + Fore.RED + text) -@receiver(debug_message) +@receiver(debug_message_s) def print_debug_message(**kwargs): - """ Receiver to print a debug message in blue, if verbose and DEBUG are set + """ Receiver to handle a debug message, blue text if verbose and DEBUG are set """ text = str(kwargs.get('text')) if get_verbosity() and settings.DEBUG and text: diff --git a/patchman/signals.py b/patchman/signals.py index 917a48e4..799b9c98 100644 --- a/patchman/signals.py +++ b/patchman/signals.py @@ -19,7 +19,7 @@ pbar_start = Signal() pbar_update = Signal() -info_message = Signal() -warning_message = Signal() -error_message = Signal() -debug_message = Signal() +info_message_s = Signal() +warning_message_s = Signal() +error_message_s = Signal() +debug_message_s = Signal() diff --git a/reports/models.py b/reports/models.py index 6818ea23..d529804b 100644 --- a/reports/models.py +++ b/reports/models.py @@ -19,7 +19,7 @@ from django.urls import reverse from hosts.utils import get_or_create_host -from patchman.signals import error_message, info_message +from util.logging import error_message, info_message class Report(models.Model): @@ -97,11 +97,11 @@ def process(self, find_updates=True, verbose=False): """ Process a report and extract os, arch, domain, packages, repos etc """ if not self.os or not self.kernel or not self.arch: - error_message.send(sender=None, text=f'Error: OS, kernel or arch not sent with report {self.id}') + error_message(text=f'Error: OS, kernel or arch not sent with report {self.id}') return if self.processed: - info_message.send(sender=None, text=f'Report {self.id} has already been processed') + info_message(text=f'Report {self.id} has already been processed') return from reports.utils import get_arch, get_os, get_domain @@ -111,7 +111,7 @@ def process(self, find_updates=True, verbose=False): host = get_or_create_host(self, arch, osvariant, domain) if verbose: - info_message.send(sender=None, text=f'Processing report {self.id} - {self.host}') + info_message(text=f'Processing report {self.id} - {self.host}') from reports.utils import process_packages, process_repos, process_updates, process_modules process_repos(report=self, host=host) @@ -124,5 +124,5 @@ def process(self, find_updates=True, verbose=False): if find_updates: if verbose: - info_message.send(sender=None, text=f'Finding updates for report {self.id} - {self.host}') + info_message(text=f'Finding updates for report {self.id} - {self.host}') host.find_updates() diff --git a/reports/tasks.py b/reports/tasks.py index db9e4103..fe294e8d 100755 --- a/reports/tasks.py +++ b/reports/tasks.py @@ -21,7 +21,7 @@ from hosts.models import Host from reports.models import Report -from util import info_message +from util.logging import info_message @shared_task(bind=True, autoretry_for=(OperationalError,), retry_backoff=True, retry_kwargs={'max_retries': 5}) @@ -48,5 +48,5 @@ def clean_reports_with_no_hosts(): for report in Report.objects.filter(processed=True): if not Host.objects.filter(hostname=report.host).exists(): text = f'Deleting report {report.id} for Host `{report.host}` as the host no longer exists' - info_message.send(sender=None, text=text) + info_message(text=text) report.delete() diff --git a/reports/utils.py b/reports/utils.py index 641f90df..76b6e09c 100644 --- a/reports/utils.py +++ b/reports/utils.py @@ -26,7 +26,8 @@ from operatingsystems.utils import get_or_create_osrelease, get_or_create_osvariant from packages.models import Package, PackageCategory from packages.utils import find_evr, get_or_create_package, get_or_create_package_update, parse_package_string -from patchman.signals import pbar_start, pbar_update, info_message +from util.logging import info_message +from patchman.signals import pbar_start, pbar_update from repos.models import Repository, Mirror, MirrorPackage from repos.utils import get_or_create_repo @@ -93,7 +94,7 @@ def process_packages(report, host): host.packages.add(package) else: if pkg_str[0].lower() != 'gpg-pubkey': - info_message.send(sender=None, text=f'No package returned for {pkg_str}') + info_message(text=f'No package returned for {pkg_str}') pbar_update.send(sender=None, index=i + 1) for package in host.packages.all(): diff --git a/repos/models.py b/repos/models.py index 181a103d..a1db2a93 100644 --- a/repos/models.py +++ b/repos/models.py @@ -26,7 +26,7 @@ from repos.repo_types.rpm import refresh_rpm_repo, refresh_repo_errata from repos.repo_types.arch import refresh_arch_repo from repos.repo_types.gentoo import refresh_gentoo_repo -from patchman.signals import info_message, warning_message, error_message +from util.logging import info_message, warning_message, error_message class Repository(models.Model): @@ -72,7 +72,7 @@ def show(self): text += f'arch: {self.arch}\n' text += 'Mirrors:' - info_message.send(sender=None, text=text) + info_message(text=text) for mirror in self.mirror_set.all(): mirror.show() @@ -99,10 +99,10 @@ def refresh(self, force=False): refresh_gentoo_repo(self) else: text = f'Error: unknown repo type for repo {self.id}: {self.repotype}' - error_message.send(sender=None, text=text) + error_message(text=text) else: text = 'Repo requires authentication, not updating' - warning_message.send(sender=None, text=text) + warning_message(text=text) def refresh_errata(self, force=False): """ Refresh errata metadata for all of a repos mirrors @@ -168,7 +168,7 @@ def show(self): text = f' {self.id} : {self.url}\n' text += ' last updated: ' text += f'{self.timestamp} checksum: {self.packages_checksum}\n' - info_message.send(sender=None, text=text) + info_message(text=text) def fail(self): """ Records that the mirror has failed @@ -178,10 +178,10 @@ def fail(self): """ if self.repo.auth_required: text = f'Mirror requires authentication, not updating - {self.url}' - warning_message.send(sender=None, text=text) + warning_message(text=text) return text = f'No usable mirror found at {self.url}' - error_message.send(sender=None, text=text) + error_message(text=text) default_max_mirror_failures = 28 max_mirror_failures = get_setting_of_type( setting_name='MAX_MIRROR_FAILURES', @@ -191,11 +191,11 @@ def fail(self): self.fail_count = self.fail_count + 1 if max_mirror_failures == -1: text = f'Mirror has failed {self.fail_count} times, but MAX_MIRROR_FAILURES=-1, not disabling refresh' - error_message.send(sender=None, text=text) + error_message(text=text) elif self.fail_count > max_mirror_failures: self.refresh = False text = f'Mirror has failed {self.fail_count} times (max={max_mirror_failures}), disabling refresh' - error_message.send(sender=None, text=text) + error_message(text=text) self.last_access_ok = False self.save() diff --git a/repos/repo_types/arch.py b/repos/repo_types/arch.py index 6e85b153..09719428 100644 --- a/repos/repo_types/arch.py +++ b/repos/repo_types/arch.py @@ -18,7 +18,8 @@ from io import BytesIO from packages.models import PackageString -from patchman.signals import info_message, warning_message, pbar_start, pbar_update +from util.logging import info_message, warning_message +from patchman.signals import pbar_start, pbar_update from repos.utils import get_max_mirrors, fetch_mirror_data, find_mirror_url, update_mirror_packages from util import get_datetime_now, get_checksum, Checksum @@ -34,7 +35,7 @@ def refresh_arch_repo(repo): for i, mirror in enumerate(enabled_mirrors): if i >= max_mirrors: text = f'{max_mirrors} Mirrors already refreshed (max={max_mirrors}), skipping further refreshes' - warning_message.send(sender=None, text=text) + warning_message(text=text) break res = find_mirror_url(mirror.url, [fname]) @@ -42,7 +43,7 @@ def refresh_arch_repo(repo): continue mirror_url = res.url text = f'Found Arch Repo - {mirror_url}' - info_message.send(sender=None, text=text) + info_message(text=text) package_data = fetch_mirror_data( mirror=mirror, @@ -54,7 +55,7 @@ def refresh_arch_repo(repo): computed_checksum = get_checksum(package_data, Checksum.sha1) if mirror.packages_checksum == computed_checksum: text = 'Mirror checksum has not changed, not refreshing Package metadata' - warning_message.send(sender=None, text=text) + warning_message(text=text) continue else: mirror.packages_checksum = computed_checksum @@ -111,5 +112,5 @@ def extract_arch_packages(data): packagetype='A') packages.add(package) else: - info_message.send(sender=None, text='No Packages found in Repo') + info_message(text='No Packages found in Repo') return packages diff --git a/repos/repo_types/deb.py b/repos/repo_types/deb.py index 1d3607c5..c6c26d78 100644 --- a/repos/repo_types/deb.py +++ b/repos/repo_types/deb.py @@ -19,7 +19,8 @@ from debian.debian_support import Version from packages.models import PackageString -from patchman.signals import error_message, pbar_start, pbar_update, info_message, warning_message +from util.logging import error_message, info_message, warning_message +from patchman.signals import pbar_start, pbar_update from repos.utils import fetch_mirror_data, update_mirror_packages, find_mirror_url from util import get_datetime_now, get_checksum, Checksum, extract @@ -30,7 +31,7 @@ def extract_deb_packages(data, url): try: extracted = extract(data, url).decode('utf-8') except UnicodeDecodeError as e: - error_message.send(sender=None, text=f'Skipping {url} : {e}') + error_message(text=f'Skipping {url} : {e}') return package_re = re.compile('^Package: ', re.M) plen = len(package_re.findall(extracted)) @@ -61,7 +62,7 @@ def extract_deb_packages(data, url): packagetype='D') packages.add(package) else: - info_message.send(sender=None, text='No packages found in repo') + info_message(text='No packages found in repo') return packages @@ -86,7 +87,7 @@ def refresh_deb_repo(repo): continue mirror_url = res.url text = f'Found deb Repo - {mirror_url}' - info_message.send(sender=None, text=text) + info_message(text=text) package_data = fetch_mirror_data( mirror=mirror, @@ -98,7 +99,7 @@ def refresh_deb_repo(repo): computed_checksum = get_checksum(package_data, Checksum.sha1) if mirror.packages_checksum == computed_checksum: text = 'Mirror checksum has not changed, not refreshing Package metadata' - warning_message.send(sender=None, text=text) + warning_message(text=text) continue else: mirror.packages_checksum = computed_checksum diff --git a/repos/repo_types/gentoo.py b/repos/repo_types/gentoo.py index 8e4198d9..e440f0d5 100644 --- a/repos/repo_types/gentoo.py +++ b/repos/repo_types/gentoo.py @@ -26,7 +26,8 @@ from packages.models import PackageString from packages.utils import find_evr -from patchman.signals import info_message, warning_message, error_message, pbar_start, pbar_update +from util.logging import info_message, warning_message, error_message +from patchman.signals import pbar_start, pbar_update from repos.utils import add_mirrors_from_urls, mirror_checksum_is_valid, update_mirror_packages from util import extract, get_url, get_datetime_now, get_checksum, Checksum, fetch_content, response_is_valid @@ -56,7 +57,7 @@ def refresh_gentoo_main_repo(repo): if mirror.packages_checksum == checksum: text = 'Mirror checksum has not changed, not refreshing Package metadata' - warning_message.send(sender=None, text=text) + warning_message(text=text) continue res = get_url(mirror.url) @@ -70,7 +71,7 @@ def refresh_gentoo_main_repo(repo): mirror.fail() continue extracted = extract(data, mirror.url) - info_message.send(sender=None, text=f'Found Gentoo Repo - {mirror.url}') + info_message(text=f'Found Gentoo Repo - {mirror.url}') computed_checksum = get_checksum(data, Checksum.md5) if not mirror_checksum_is_valid(computed_checksum, checksum, mirror, 'package'): @@ -165,7 +166,7 @@ def get_gentoo_overlay_mirrors(repo_name): if element.text.startswith('http'): mirrors.append(element.text) except ElementTree.ParseError as e: - error_message.send(sender=None, text=f'Error parsing {gentoo_overlays_url}: {e}') + error_message(text=f'Error parsing {gentoo_overlays_url}: {e}') return mirrors @@ -199,7 +200,7 @@ def get_gentoo_mirror_urls(): if element.get('protocol') == 'http': mirrors[name]['urls'].append(element.text) except ElementTree.ParseError as e: - error_message.send(sender=None, text=f'Error parsing {gentoo_distfiles_url}: {e}') + error_message(text=f'Error parsing {gentoo_distfiles_url}: {e}') mirror_urls = [] # for now, ignore region data and choose MAX_MIRRORS mirrors at random for _, v in mirrors.items(): @@ -274,7 +275,7 @@ def extract_gentoo_packages_from_ebuilds(extracted_ebuilds): ) packages.add(package) plen = len(packages) - info_message.send(sender=None, text=f'Extracted {plen} Packages', plen=plen) + info_message(text=f'Extracted {plen} Packages', plen=plen) return packages @@ -282,7 +283,7 @@ def extract_gentoo_overlay_packages(mirror): """ Extract packages from gentoo overlay repo """ t = tempfile.mkdtemp() - info_message.send(sender=None, text=f'Extracting Gentoo packages from {mirror.url}') + info_message(text=f'Extracting Gentoo packages from {mirror.url}') git.Repo.clone_from(mirror.url, t, depth=1) packages = set() extracted_ebuilds = extract_gentoo_overlay_ebuilds(t) diff --git a/repos/repo_types/rpm.py b/repos/repo_types/rpm.py index 93d47007..51661809 100644 --- a/repos/repo_types/rpm.py +++ b/repos/repo_types/rpm.py @@ -16,7 +16,7 @@ from django.db.models import Q -from patchman.signals import info_message, warning_message +from util.logging import info_message, warning_message from repos.repo_types.yast import refresh_yast_repo from repos.repo_types.yum import refresh_yum_repo from repos.utils import check_for_metalinks, check_for_mirrorlists, find_mirror_url, get_max_mirrors, fetch_mirror_data @@ -47,7 +47,7 @@ def max_mirrors_refreshed(repo, checksum, ts): have_checksum_and_ts = repo.mirror_set.filter(mirrors_q).count() if have_checksum_and_ts >= max_mirrors: text = f'{max_mirrors} Mirrors already have this checksum and timestamp, skipping further refreshes' - warning_message.send(sender=None, text=text) + warning_message(text=text) return True return False @@ -87,11 +87,11 @@ def refresh_rpm_repo_mirrors(repo, errata_only=False): if mirror_url.endswith('content'): text = f'Found yast rpm Repo - {mirror_url}' - info_message.send(sender=None, text=text) + info_message(text=text) refresh_yast_repo(mirror, repo_data) else: text = f'Found yum rpm Repo - {mirror_url}' - info_message.send(sender=None, text=text) + info_message(text=text) refresh_yum_repo(mirror, repo_data, mirror_url, errata_only) if mirror.last_access_ok: mirror.timestamp = ts diff --git a/repos/repo_types/yast.py b/repos/repo_types/yast.py index 0ef54358..bf594040 100644 --- a/repos/repo_types/yast.py +++ b/repos/repo_types/yast.py @@ -17,7 +17,8 @@ import re from packages.models import PackageString -from patchman.signals import pbar_start, pbar_update, info_message +from util.logging import info_message +from patchman.signals import pbar_start, pbar_update from repos.utils import fetch_mirror_data, update_mirror_packages from util import extract @@ -65,5 +66,5 @@ def extract_yast_packages(data): packagetype='R') packages.add(package) else: - info_message.send(sender=None, text='No packages found in repo') + info_message(text='No packages found in repo') return packages diff --git a/repos/repo_types/yum.py b/repos/repo_types/yum.py index 7ac85816..bc0fbc4b 100644 --- a/repos/repo_types/yum.py +++ b/repos/repo_types/yum.py @@ -22,7 +22,8 @@ from errata.sources.repos.yum import extract_updateinfo from packages.models import Package, PackageString from packages.utils import get_or_create_package, parse_package_string -from patchman.signals import warning_message, error_message, pbar_start, pbar_update +from util.logging import warning_message, error_message +from patchman.signals import pbar_start, pbar_update from repos.utils import fetch_mirror_data, update_mirror_packages from util import extract @@ -50,7 +51,7 @@ def get_repomd_url(mirror_url, data, url_type='primary'): checksum = grandchild.text checksum_type = grandchild.attrib.get('type') except ElementTree.ParseError as e: - error_message.send(sender=None, text=(f'Error parsing repomd from {mirror_url}: {e}')) + error_message(text=(f'Error parsing repomd from {mirror_url}: {e}')) if not location: return None, None, None url = str(mirror_url.rsplit('/', 2)[0]) + '/' + location @@ -65,7 +66,7 @@ def extract_module_metadata(data, url, repo): try: modules_yaml = yaml.safe_load_all(extracted) except yaml.YAMLError as e: - error_message.send(sender=None, text=f'Error parsing modules.yaml: {e}') + error_message(text=f'Error parsing modules.yaml: {e}') mlen = len(re.findall(r'---', yaml.dump(extracted.decode()))) pbar_start.send(sender=None, ptext=f'Extracting {mlen} Modules ', plen=mlen) @@ -150,10 +151,10 @@ def extract_yum_packages(data, url): i += 1 else: text = f'Error parsing Package: {name} {epoch} {version} {release} {arch}' - error_message.send(sender=None, text=text) + error_message(text=text) elem.clear() except ElementTree.ParseError as e: - error_message.send(sender=None, text=f'Error parsing yum primary.xml from {url}: {e}') + error_message(text=f'Error parsing yum primary.xml from {url}: {e}') return packages @@ -162,7 +163,7 @@ def refresh_repomd_updateinfo(mirror, data, mirror_url): """ url, checksum, checksum_type = get_repomd_url(mirror_url, data, url_type='updateinfo') if not url: - warning_message.send(sender=None, text=f'No Errata metadata found in {mirror_url}') + warning_message(text=f'No Errata metadata found in {mirror_url}') return data = fetch_mirror_data( mirror=mirror, @@ -177,7 +178,7 @@ def refresh_repomd_updateinfo(mirror, data, mirror_url): if mirror.errata_checksum and mirror.errata_checksum == checksum: text = 'Mirror Errata checksum has not changed, skipping Erratum refresh' - warning_message.send(sender=None, text=text) + warning_message(text=text) return else: mirror.errata_checksum = checksum @@ -191,7 +192,7 @@ def refresh_repomd_modules(mirror, data, mirror_url): """ url, checksum, checksum_type = get_repomd_url(mirror_url, data, url_type='modules') if not url: - warning_message.send(sender=None, text=f'No Module metadata found in {mirror_url}') + warning_message(text=f'No Module metadata found in {mirror_url}') return data = fetch_mirror_data( mirror=mirror, @@ -206,7 +207,7 @@ def refresh_repomd_modules(mirror, data, mirror_url): if mirror.modules_checksum and mirror.modules_checksum == checksum: text = 'Mirror Modules checksum has not changed, skipping Module refresh' - warning_message.send(sender=None, text=text) + warning_message(text=text) return else: mirror.modules_checksum = checksum @@ -220,7 +221,7 @@ def refresh_repomd_primary(mirror, data, mirror_url): """ url, checksum, checksum_type = get_repomd_url(mirror_url, data, url_type='primary') if not url: - warning_message.send(sender=None, text=f'No Package metadata found in {mirror_url}') + warning_message(text=f'No Package metadata found in {mirror_url}') data = fetch_mirror_data( mirror=mirror, url=url, @@ -234,7 +235,7 @@ def refresh_repomd_primary(mirror, data, mirror_url): if mirror.packages_checksum and mirror.packages_checksum == checksum: text = 'Mirror Packages checksum has not changed, skipping Package refresh' - warning_message.send(sender=None, text=text) + warning_message(text=text) return else: mirror.packages_checksum = checksum diff --git a/repos/utils.py b/repos/utils.py index 49b5d07f..c11c41ad 100644 --- a/repos/utils.py +++ b/repos/utils.py @@ -26,7 +26,8 @@ from packages.models import Package from packages.utils import convert_package_to_packagestring, convert_packagestring_to_package from util import get_url, fetch_content, response_is_valid, extract, get_checksum, Checksum, get_setting_of_type -from patchman.signals import info_message, warning_message, error_message, debug_message, pbar_start, pbar_update +from util.logging import info_message, warning_message, error_message, debug_message +from patchman.signals import pbar_start, pbar_update def get_or_create_repo(r_name, r_arch, r_type, r_id=None): @@ -77,7 +78,7 @@ def update_mirror_packages(mirror, packages): package = convert_packagestring_to_package(strpackage) mirror_package, c = MirrorPackage.objects.get_or_create(mirror=mirror, package=package) except Package.MultipleObjectsReturned: - error_message.send(sender=None, text=f'Duplicate Package found in {mirror}: {strpackage}') + error_message(text=f'Duplicate Package found in {mirror}: {strpackage}') def find_mirror_url(stored_mirror_url, formats): @@ -89,7 +90,7 @@ def find_mirror_url(stored_mirror_url, formats): if mirror_url.endswith(f): mirror_url = mirror_url[:-len(f)] mirror_url = f"{mirror_url.rstrip('/')}/{fmt}" - debug_message.send(sender=None, text=f'Checking for Mirror at {mirror_url}') + debug_message(text=f'Checking for Mirror at {mirror_url}') try: res = get_url(mirror_url) except RetryError: @@ -133,7 +134,7 @@ def get_metalink_urls(url): if greatgreatgrandchild.attrib.get('protocol') in ['https', 'http']: metalink_urls.append(greatgreatgrandchild.text) except ElementTree.ParseError as e: - error_message.send(sender=None, text=f'Error parsing metalink {url}: {e}') + error_message(text=f'Error parsing metalink {url}: {e}') return metalink_urls @@ -152,12 +153,12 @@ def get_mirrorlist_urls(url): return mirror_urls = re.findall(r'^http[s]*://.*$|^ftp://.*$', data.decode('utf-8'), re.MULTILINE) if mirror_urls: - debug_message.send(sender=None, text=f'Found mirrorlist: {url}') + debug_message(text=f'Found mirrorlist: {url}') return mirror_urls else: - debug_message.send(sender=None, text=f'Not a mirrorlist: {url}') + debug_message(text=f'Not a mirrorlist: {url}') except Exception as e: - error_message.send(sender=None, text=f'Error attempting to parse a mirrorlist: {e} {url}') + error_message(text=f'Error attempting to parse a mirrorlist: {e} {url}') def add_mirrors_from_urls(repo, mirror_urls): @@ -172,7 +173,7 @@ def add_mirrors_from_urls(repo, mirror_urls): existing = repo.mirror_set.filter(q).count() if existing >= max_mirrors: text = f'{existing} Mirrors already exist (max={max_mirrors}), not adding more' - warning_message.send(sender=None, text=text) + warning_message(text=text) break from repos.models import Mirror # FIXME: maybe we should store the mirrorlist url with full path to repomd.xml? @@ -180,7 +181,7 @@ def add_mirrors_from_urls(repo, mirror_urls): m, c = Mirror.objects.get_or_create(repo=repo, url=mirror_url.rstrip('/').replace('repodata/repomd.xml', '')) if c: text = f'Added Mirror - {mirror_url}' - info_message.send(sender=None, text=text) + info_message(text=text) def check_for_mirrorlists(repo): @@ -193,7 +194,7 @@ def check_for_mirrorlists(repo): mirror.mirrorlist = True mirror.last_access_ok = True mirror.save() - info_message.send(sender=None, text=f'Found mirrorlist - {mirror.url}') + info_message(text=f'Found mirrorlist - {mirror.url}') add_mirrors_from_urls(repo, mirror_urls) @@ -210,7 +211,7 @@ def check_for_metalinks(repo): mirror.mirrorlist = True mirror.last_access_ok = True mirror.save() - info_message.send(sender=None, text=f'Found metalink - {mirror.url}') + info_message(text=f'Found metalink - {mirror.url}') add_mirrors_from_urls(repo, mirror_urls) @@ -249,9 +250,9 @@ def mirror_checksum_is_valid(computed, provided, mirror, metadata_type): """ if not computed or computed != provided: text = f'Checksum failed for mirror {mirror.id}, not refreshing {metadata_type} metadata' - error_message.send(sender=None, text=text) + error_message(text=text) text = f'Found checksum: {computed}\nExpected checksum: {provided}' - error_message.send(sender=None, text=text) + error_message(text=text) mirror.last_access_ok = False mirror.fail() return False @@ -296,9 +297,9 @@ def clean_repos(): repos = Repository.objects.filter(mirror__isnull=True) rlen = repos.count() if rlen == 0: - info_message.send(sender=None, text='No Repositories with zero Mirrors found.') + info_message(text='No Repositories with zero Mirrors found.') else: - info_message.send(sender=None, text=f'Removing {rlen} empty Repositories.') + info_message(text=f'Removing {rlen} empty Repositories.') repos.delete() @@ -309,13 +310,13 @@ def remove_mirror_trailing_slashes(): mirrors = Mirror.objects.filter(url__endswith='/') mlen = mirrors.count() if mlen == 0: - info_message.send(sender=None, text='No Mirrors with trailing slashes found.') + info_message(text='No Mirrors with trailing slashes found.') else: - info_message.send(sender=None, text=f'Removing trailing slashes from {mlen} Mirrors.') + info_message(text=f'Removing trailing slashes from {mlen} Mirrors.') for mirror in mirrors: mirror.url = mirror.url.rstrip('/') try: mirror.save() except IntegrityError: - warning_message.send(sender=None, text=f'Deleting duplicate Mirror {mirror.id}: {mirror.url}') + warning_message(text=f'Deleting duplicate Mirror {mirror.id}: {mirror.url}') mirror.delete() diff --git a/sbin/patchman b/sbin/patchman index c0911434..47d89b06 100755 --- a/sbin/patchman +++ b/sbin/patchman @@ -43,7 +43,7 @@ from reports.models import Report from reports.tasks import clean_reports_with_no_hosts from security.utils import update_cves, update_cwes from util import set_verbosity, get_datetime_now -from patchman.signals import info_message +from util.logging import info_message def get_host(host=None, action='Performing action'): @@ -64,7 +64,7 @@ def get_host(host=None, action='Performing action'): matches = Host.objects.filter(hostname__startswith=host).count() text = f'{matches} Hosts match hostname "{host}"' - info_message.send(sender=None, text=text) + info_message(text=text) return host_obj @@ -84,7 +84,7 @@ def get_hosts(hosts=None, action='Performing action'): host_objs.append(host_obj) else: text = f'{action} for all Hosts\n' - info_message.send(sender=None, text=text) + info_message(text=text) host_objs = Host.objects.all() return host_objs @@ -107,7 +107,7 @@ def get_repos(repo=None, action='Performing action', only_enabled=False): else: repos = Repository.objects.all() - info_message.send(sender=None, text=text) + info_message(text=text) return repos @@ -118,9 +118,9 @@ def refresh_repos(repo=None, force=False): repos = get_repos(repo, 'Refreshing metadata', True) for repo in repos: text = f'Repository {repo.id} : {repo}' - info_message.send(sender=None, text=text) + info_message(text=text) repo.refresh(force) - info_message.send(sender=None, text='') + info_message(text='') def list_repos(repos=None): @@ -161,10 +161,10 @@ def host_updates_alt(host=None): hosts = get_hosts(host, 'Finding updates') ts = get_datetime_now() for host in hosts: - info_message.send(sender=None, text=str(host)) + info_message(text=str(host)) if host not in updated_hosts: host.find_updates() - info_message.send(sender=None, text='') + info_message(text='') host.updated_at = ts host.save() @@ -200,10 +200,10 @@ def host_updates_alt(host=None): phost.save() updated_hosts.append(phost) text = f'Added the same updates to {phost}' - info_message.send(sender=None, text=text) + info_message(text=text) else: text = 'Updates already added in this run' - info_message.send(sender=None, text=text) + info_message(text=text) def host_updates(host=None): @@ -211,9 +211,9 @@ def host_updates(host=None): """ hosts = get_hosts(host, 'Finding updates') for host in hosts: - info_message.send(sender=None, text=str(host)) + info_message(text=str(host)) host.find_updates() - info_message.send(sender=None, text='') + info_message(text='') def diff_hosts(hosts): @@ -236,47 +236,47 @@ def diff_hosts(hosts): repo_diff_AB = reposA.difference(reposB) repo_diff_BA = reposB.difference(reposA) - info_message.send(sender=None, text=f'+ {hostA.hostname}') - info_message.send(sender=None, text=f'- {hostB.hostname}') + info_message(text=f'+ {hostA.hostname}') + info_message(text=f'- {hostB.hostname}') if hostA.os != hostB.os: - info_message.send(sender=None, text='\nOperating Systems') - info_message.send(sender=None, text=f'+ {hostA.os}') - info_message.send(sender=None, text=f'- {hostB.os}') + info_message(text='\nOperating Systems') + info_message(text=f'+ {hostA.os}') + info_message(text=f'- {hostB.os}') else: - info_message.send(sender=None, text='\nNo OS differences') + info_message(text='\nNo OS differences') if hostA.arch != hostB.arch: - info_message.send(sender=None, text='\nArchitecture') - info_message.send(sender=None, text=f'+ {hostA.arch}') - info_message.send(sender=None, text=f'- {hostB.arch}') + info_message(text='\nArchitecture') + info_message(text=f'+ {hostA.arch}') + info_message(text=f'- {hostB.arch}') else: - info_message.send(sender=None, text='\nNo Architecture differences') + info_message(text='\nNo Architecture differences') if hostA.kernel != hostB.kernel: - info_message.send(sender=None, text='\nKernels') - info_message.send(sender=None, text=f'+ {hostA.kernel}') - info_message.send(sender=None, text=f'- {hostB.kernel}') + info_message(text='\nKernels') + info_message(text=f'+ {hostA.kernel}') + info_message(text=f'- {hostB.kernel}') else: - info_message.send(sender=None, text='\nNo Kernel differences') + info_message(text='\nNo Kernel differences') if len(package_diff_AB) != 0 or len(package_diff_BA) != 0: - info_message.send(sender=None, text='\nPackages') + info_message(text='\nPackages') for package in package_diff_AB: - info_message.send(sender=None, text=f'+ {package}') + info_message(text=f'+ {package}') for package in package_diff_BA: - info_message.send(sender=None, text=f'- {package}') + info_message(text=f'- {package}') else: - info_message.send(sender=None, text='\nNo Package differences') + info_message(text='\nNo Package differences') if len(repo_diff_AB) != 0 or len(repo_diff_BA) != 0: - info_message.send(sender=None, text='\nRepositories') + info_message(text='\nRepositories') for repo in repo_diff_AB: - info_message.send(sender=None, text=f'+ {repo}') + info_message(text=f'+ {repo}') for repo in repo_diff_BA: - info_message.send(sender=None, text=f'- {repo}') + info_message(text=f'- {repo}') else: - info_message.send(sender=None, text='\nNo Repo differences') + info_message(text='\nNo Repo differences') def delete_hosts(hosts=None): @@ -286,7 +286,7 @@ def delete_hosts(hosts=None): matching_hosts = get_hosts(hosts) for host in matching_hosts: text = f'Deleting host: {host.hostname}:' - info_message.send(sender=None, text=text) + info_message(text=text) host.delete() @@ -300,7 +300,7 @@ def toggle_host_hro(hosts=None, host_repos_only=True): if hosts: matching_hosts = get_hosts(hosts, f'{toggle} host_repos_only') for host in matching_hosts: - info_message.send(sender=None, text=str(host)) + info_message(text=str(host)) host.host_repos_only = host_repos_only host.save() @@ -315,7 +315,7 @@ def toggle_host_check_dns(hosts=None, check_dns=True): if hosts: matching_hosts = get_hosts(hosts, f'{toggle} check_dns') for host in matching_hosts: - info_message.send(sender=None, text=str(host)) + info_message(text=str(host)) host.check_dns = check_dns host.save() @@ -347,7 +347,7 @@ def process_reports(host=None, force=False): text = 'Processing Reports for all Hosts' reports = Report.objects.filter(processed=force).order_by('created') - info_message.send(sender=None, text=text) + info_message(text=text) for report in reports: report.process(find_updates=False) diff --git a/security/models.py b/security/models.py index 9c097eed..7f674a0b 100644 --- a/security/models.py +++ b/security/models.py @@ -152,7 +152,7 @@ def fetch_mitre_cve_data(self): mitre_cve_url = f'https://cveawg.mitre.org/api/cve/{self.cve_id}' res = get_url(mitre_cve_url) if res.status_code == 404: - error_message.send(sender=None, text=f'404 - Skipping {self.cve_id} - {mitre_cve_url}') + error_message(text=f'404 - Skipping {self.cve_id} - {mitre_cve_url}') return data = fetch_content(res, f'Fetching {self.cve_id} MITRE data') cve_json = json.loads(data) @@ -162,7 +162,7 @@ def fetch_osv_dev_cve_data(self): osv_dev_cve_url = f'https://api.osv.dev/v1/vulns/{self.cve_id}' res = get_url(osv_dev_cve_url) if res.status_code == 404: - error_message.send(sender=None, text=f'404 - Skipping {self.cve_id} - {osv_dev_cve_url}') + error_message(text=f'404 - Skipping {self.cve_id} - {osv_dev_cve_url}') return data = fetch_content(res, f'Fetching {self.cve_id} OSV data') cve_json = json.loads(data) @@ -186,7 +186,7 @@ def fetch_nist_cve_data(self): res = get_url(nist_cve_url) data = fetch_content(res, f'Fetching {self.cve_id} NIST data') if res.status_code == 404: - error_message.send(sender=None, text=f'404 - Skipping {self.cve_id} - {nist_cve_url}') + error_message(text=f'404 - Skipping {self.cve_id} - {nist_cve_url}') cve_json = json.loads(data) self.parse_nist_cve_data(cve_json) @@ -197,7 +197,7 @@ def parse_nist_cve_data(self, cve_json): cve = vulnerability.get('cve') cve_id = cve.get('id') if cve_id != self.cve_id: - error_message.send(sender=None, text=f'CVE ID mismatch - {self.cve_id} != {cve_id}') + error_message(text=f'CVE ID mismatch - {self.cve_id} != {cve_id}') return metrics = cve.get('metrics') for metric, score_data in metrics.items(): diff --git a/util/__init__.py b/util/__init__.py index 4a3f9caa..c3dfd6bd 100644 --- a/util/__init__.py +++ b/util/__init__.py @@ -38,7 +38,7 @@ from django.utils.dateparse import parse_datetime from django.conf import settings -from patchman.signals import error_message, info_message, debug_message +from util.logging import error_message, info_message, debug_message pbar = None verbose = None @@ -109,7 +109,7 @@ def fetch_content(response, text='', ljust=35): data += chunk return data else: - info_message.send(sender=None, text=text) + info_message(text=text) return response.content @@ -128,16 +128,16 @@ def get_url(url, headers=None, params=None): if not params: params = {} try: - debug_message.send(sender=None, text=f'Trying {url} headers:{headers} params:{params}') + debug_message(text=f'Trying {url} headers:{headers} params:{params}') response = requests.get(url, headers=headers, params=params, stream=True, proxies=proxies, timeout=30) - debug_message.send(sender=None, text=f'{response.status_code}: {response.headers}') + debug_message(text=f'{response.status_code}: {response.headers}') if response.status_code in [403, 404]: return response response.raise_for_status() except requests.exceptions.TooManyRedirects: - error_message.send(sender=None, text=f'Too many redirects - {url}') + error_message(text=f'Too many redirects - {url}') except ConnectionError: - error_message.send(sender=None, text=f'Connection error - {url}') + error_message(text=f'Connection error - {url}') return response @@ -180,7 +180,7 @@ def gunzip(contents): wbits = zlib.MAX_WBITS | 32 return zlib.decompress(contents, wbits) except zlib.error as e: - error_message.send(sender=None, text='gunzip: ' + str(e)) + error_message(text='gunzip: ' + str(e)) def bunzip2(contents): @@ -191,10 +191,10 @@ def bunzip2(contents): return bzip2data except IOError as e: if e == 'invalid data stream': - error_message.send(sender=None, text='bunzip2: ' + e) + error_message(text='bunzip2: ' + e) except ValueError as e: if e == "couldn't find end of stream": - error_message.send(sender=None, text='bunzip2: ' + e) + error_message(text='bunzip2: ' + e) def unxz(contents): @@ -204,7 +204,7 @@ def unxz(contents): xzdata = lzma.decompress(contents) return xzdata except lzma.LZMAError as e: - error_message.send(sender=None, text='lzma: ' + e) + error_message(text='lzma: ' + e) def unzstd(contents): @@ -214,7 +214,7 @@ def unzstd(contents): zstddata = zstd.ZstdDecompressor().stream_reader(contents).read() return zstddata except zstd.ZstdError as e: - error_message.send(sender=None, text='zstd: ' + e) + error_message(text=f'zstd: {e}') def extract(data, fmt): @@ -253,7 +253,7 @@ def get_checksum(data, checksum_type): checksum = get_md5(data) else: text = f'Unknown checksum type: {checksum_type}' - error_message.send(sender=None, text=text) + error_message(text=text) return checksum diff --git a/util/logging.py b/util/logging.py new file mode 100644 index 00000000..dd79d296 --- /dev/null +++ b/util/logging.py @@ -0,0 +1,42 @@ +# Copyright 2025 Marcus Furlong +# +# This file is part of Patchman. +# +# Patchman is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, version 3 only. +# +# Patchman is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Patchman. If not, see + +from datetime import datetime + +from patchman.signals import info_message_s +from patchman.signals import warning_message_s +from patchman.signals import error_message_s +from patchman.signals import debug_message_s + + +def info_message(text): + ts = datetime.now() + info_message_s.send(sender=None, text=text, ts=ts) + + +def warning_message(text): + ts = datetime.now() + warning_message_s.send(sender=None, text=text, ts=ts) + + +def debug_message(text): + ts = datetime.now() + debug_message_s.send(sender=None, text=text, ts=ts) + + +def error_message(text): + ts = datetime.now() + error_message_s.send(sender=None, text=text, ts=ts) From f46b57b24f18e79aa374334b95e38d663ac5328e Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Wed, 29 Oct 2025 23:34:50 -0400 Subject: [PATCH 025/146] use redis for caching and use locks for tasks --- errata/tasks.py | 68 ++++++++++++++++++++-------------- etc/patchman/local_settings.py | 16 +++----- reports/tasks.py | 15 +++++++- repos/tasks.py | 17 ++++++++- security/tasks.py | 33 ++++++++++++++--- 5 files changed, 101 insertions(+), 48 deletions(-) diff --git a/errata/tasks.py b/errata/tasks.py index f1d6eeee..d9aaf3d4 100644 --- a/errata/tasks.py +++ b/errata/tasks.py @@ -16,13 +16,15 @@ from celery import shared_task +from django.core.cache import cache + from errata.sources.distros.arch import update_arch_errata from errata.sources.distros.alma import update_alma_errata from errata.sources.distros.debian import update_debian_errata from errata.sources.distros.centos import update_centos_errata from errata.sources.distros.rocky import update_rocky_errata from errata.sources.distros.ubuntu import update_ubuntu_errata -from util.logging import error_message +from util.logging import error_message, warning_message from repos.models import Repository from security.tasks import update_cves, update_cwes from util import get_setting_of_type @@ -44,34 +46,44 @@ def update_yum_repo_errata(repo_id=None, force=False): def update_errata(erratum_type=None, force=False, repo=None): """ Update all distros errata """ - errata_os_updates = [] - erratum_types = ['yum', 'rocky', 'alma', 'arch', 'ubuntu', 'debian', 'centos'] - erratum_type_defaults = ['yum', 'rocky', 'alma', 'arch', 'ubuntu', 'debian'] - if erratum_type: - if erratum_type not in erratum_types: - error_message.send(sender=None, text=f'Erratum type `{erratum_type}` not in {erratum_types}') - else: - errata_os_updates = erratum_type + lock_key = 'update_errata_lock' + # lock will expire after 48 hours + lock_expire = 60 * 60 * 48 + + if cache.add(lock_key, 'true', lock_expire): + try: + errata_os_updates = [] + erratum_types = ['yum', 'rocky', 'alma', 'arch', 'ubuntu', 'debian', 'centos'] + erratum_type_defaults = ['yum', 'rocky', 'alma', 'arch', 'ubuntu', 'debian'] + if erratum_type: + if erratum_type not in erratum_types: + error_message(text=f'Erratum type `{erratum_type}` not in {erratum_types}') + else: + errata_os_updates = erratum_type + else: + errata_os_updates = get_setting_of_type( + setting_name='ERRATA_OS_UPDATES', + setting_type=list, + default=erratum_type_defaults, + ) + if 'yum' in errata_os_updates: + update_yum_repo_errata(repo_id=repo, force=force) + if 'arch' in errata_os_updates: + update_arch_errata() + if 'alma' in errata_os_updates: + update_alma_errata() + if 'rocky' in errata_os_updates: + update_rocky_errata() + if 'debian' in errata_os_updates: + update_debian_errata() + if 'ubuntu' in errata_os_updates: + update_ubuntu_errata() + if 'centos' in errata_os_updates: + update_centos_errata() + finally: + cache.delete(lock_key) else: - errata_os_updates = get_setting_of_type( - setting_name='ERRATA_OS_UPDATES', - setting_type=list, - default=erratum_type_defaults, - ) - if 'yum' in errata_os_updates: - update_yum_repo_errata(repo_id=repo, force=force) - if 'arch' in errata_os_updates: - update_arch_errata() - if 'alma' in errata_os_updates: - update_alma_errata() - if 'rocky' in errata_os_updates: - update_rocky_errata() - if 'debian' in errata_os_updates: - update_debian_errata() - if 'ubuntu' in errata_os_updates: - update_ubuntu_errata() - if 'centos' in errata_os_updates: - update_centos_errata() + warning_message('Already updating Errata, skipping task.') @shared_task diff --git a/etc/patchman/local_settings.py b/etc/patchman/local_settings.py index 181c4c4d..9e7ca21b 100644 --- a/etc/patchman/local_settings.py +++ b/etc/patchman/local_settings.py @@ -44,22 +44,16 @@ # Whether to run patchman under the gunicorn web server RUN_GUNICORN = False +# Set the default timeout to e.g. 30 seconds to enable UI caching +# Note that the UI results may be out of date for this amount of time CACHES = { 'default': { - 'BACKEND': 'django.core.cache.backends.dummy.DummyCache', + 'BACKEND': 'django.core.cache.backends.redis.RedisCache', + 'LOCATION': 'redis://127.0.0.1:6379', + 'TIMEOUT': 0, } } -# Uncomment to enable redis caching for e.g. 30 seconds -# Note that the UI results may be out of date for this amount of time -# CACHES = { -# 'default': { -# 'BACKEND': 'django.core.cache.backends.redis.RedisCache', -# 'LOCATION': 'redis://127.0.0.1:6379', -# 'TIMEOUT': 30, -# } -# } - from datetime import timedelta # noqa from celery.schedules import crontab # noqa CELERY_BEAT_SCHEDULE = { diff --git a/reports/tasks.py b/reports/tasks.py index fe294e8d..d2a47e8f 100755 --- a/reports/tasks.py +++ b/reports/tasks.py @@ -17,11 +17,12 @@ from celery import shared_task +from django.core.cache import cache from django.db.utils import OperationalError from hosts.models import Host from reports.models import Report -from util.logging import info_message +from util.logging import info_message, warning_message @shared_task(bind=True, autoretry_for=(OperationalError,), retry_backoff=True, retry_kwargs={'max_retries': 5}) @@ -29,7 +30,17 @@ def process_report(self, report_id): """ Task to process a single report """ report = Report.objects.get(id=report_id) - report.process() + lock_key = f'process_report_lock_{report_id}' + # lock will expire after 1 hour + lock_expire = 60 * 60 + + if cache.add(lock_key, 'true', lock_expire): + try: + report.process() + finally: + cache.delete(lock_key) + else: + warning_message(f'Already processing report {report_id}, skipping task.') @shared_task diff --git a/repos/tasks.py b/repos/tasks.py index 39098fa8..436da82a 100644 --- a/repos/tasks.py +++ b/repos/tasks.py @@ -16,7 +16,10 @@ from celery import shared_task +from django.core.cache import cache + from repos.models import Repository +from util.logging import warning_message @shared_task @@ -32,5 +35,15 @@ def refresh_repos(force=False): """ Refresh metadata for all enabled repos """ repos = Repository.objects.filter(enabled=True) - for repo in repos: - refresh_repo.delay(repo.id, force) + lock_key = 'refresh_repos_lock' + # lock will expire after 1 day + lock_expire = 60 * 60 * 24 + + if cache.add(lock_key, 'true', lock_expire): + try: + for repo in repos: + refresh_repo.delay(repo.id, force) + finally: + cache.delete(lock_key) + else: + warning_message('Already refreshing repos, skipping task.') diff --git a/security/tasks.py b/security/tasks.py index a04bb1c8..ce60df83 100644 --- a/security/tasks.py +++ b/security/tasks.py @@ -16,7 +16,10 @@ from celery import shared_task +from django.core.cache import cache + from security.models import CVE, CWE +from util.logging import warning_message @shared_task @@ -31,8 +34,18 @@ def update_cve(cve_id): def update_cves(): """ Task to update all CVEs """ - for cve in CVE.objects.all(): - update_cve.delay(cve.id) + lock_key = 'update_cves_lock' + # lock will expire after 1 week + lock_expire = 60 * 60 * 168 + + if cache.add(lock_key, 'true', lock_expire): + try: + for cve in CVE.objects.all(): + update_cve.delay(cve.id) + finally: + cache.delete(lock_key) + else: + warning_message('Already updating CVEs, skipping task.') @shared_task @@ -45,7 +58,17 @@ def update_cwe(cwe_id): @shared_task def update_cwes(): - """ Task to update all CWEa + """ Task to update all CWEs """ - for cwe in CWE.objects.all(): - update_cwe.delay(cwe.id) + lock_key = 'update_cwes_lock' + # lock will expire after 1 week + lock_expire = 60 * 60 * 168 + + if cache.add(lock_key, 'true', lock_expire): + try: + for cwe in CWE.objects.all(): + update_cwe.delay(cwe.id) + finally: + cache.delete(lock_key) + else: + warning_message('Already updating CWEs, skipping task.') From 652333f638afdb246fa63869a4aaa00043dbebe4 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Fri, 18 Apr 2025 18:38:04 -0400 Subject: [PATCH 026/146] add errata source options to config file --- errata/sources/distros/debian.py | 2 +- errata/sources/distros/ubuntu.py | 2 +- etc/patchman/local_settings.py | 12 ++++++++++++ 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/errata/sources/distros/debian.py b/errata/sources/distros/debian.py index 1ae919e4..ece3754d 100644 --- a/errata/sources/distros/debian.py +++ b/errata/sources/distros/debian.py @@ -264,7 +264,7 @@ def get_accepted_debian_codenames(): """ Get acceptable Debian OS codenames Can be overridden by specifying DEBIAN_CODENAMES in settings """ - default_codenames = ['bookworm', 'bullseye'] + default_codenames = ['bookworm', 'trixie'] accepted_codenames = get_setting_of_type( setting_name='DEBIAN_CODENAMES', setting_type=list, diff --git a/errata/sources/distros/ubuntu.py b/errata/sources/distros/ubuntu.py index d1ce7cc5..6fafb40a 100644 --- a/errata/sources/distros/ubuntu.py +++ b/errata/sources/distros/ubuntu.py @@ -203,7 +203,7 @@ def get_accepted_ubuntu_codenames(): """ Get acceptable Ubuntu OS codenames Can be overridden by specifying UBUNTU_CODENAMES in settings """ - default_codenames = ['focal', 'jammy', 'noble'] + default_codenames = ['jammy', 'noble'] accepted_codenames = get_setting_of_type( setting_name='UBUNTU_CODENAMES', setting_type=list, diff --git a/etc/patchman/local_settings.py b/etc/patchman/local_settings.py index 181c4c4d..ab93c89d 100644 --- a/etc/patchman/local_settings.py +++ b/etc/patchman/local_settings.py @@ -41,6 +41,18 @@ # Number of days to wait before raising that a host has not reported DAYS_WITHOUT_REPORT = 14 +# list of errata sources to update, remove unwanted ones to improve performance +ERRATA_OS_UPDATES = ['yum', 'rocky', 'alma', 'arch', 'ubuntu', 'debian'] + +# list of Alma Linux releases to update +ALMA_RELEASES = [8, 9, 10] + +# list of Debian Linux releases to update +DEBIAN_CODENAMES = ['bookworm', 'trixie'] + +# list of Ubuntu Linux releases to update +UBUNTU_CODENAMES = ['jammy', 'noble'] + # Whether to run patchman under the gunicorn web server RUN_GUNICORN = False From 78d76b55546d667cae0a68dcbda4f92cd0337993 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Wed, 14 May 2025 22:39:29 -0400 Subject: [PATCH 027/146] remove daily cronjob in favour of patchman-celery --- debian/python3-patchman.cron.daily | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 debian/python3-patchman.cron.daily diff --git a/debian/python3-patchman.cron.daily b/debian/python3-patchman.cron.daily deleted file mode 100644 index d4752f75..00000000 --- a/debian/python3-patchman.cron.daily +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -/usr/bin/patchman -a -q From 0699d4bed5835dc51705faf0621dfe80bc4d406c Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Thu, 30 Oct 2025 22:02:44 -0400 Subject: [PATCH 028/146] add isort check --- .github/workflows/lint-and-test.yml | 4 +++ arch/admin.py | 3 ++- arch/serializers.py | 2 +- arch/utils.py | 2 +- arch/views.py | 7 ++--- domains/admin.py | 1 + errata/admin.py | 1 + errata/models.py | 7 +++-- errata/sources/distros/alma.py | 2 +- errata/sources/distros/arch.py | 10 ++++--- errata/sources/distros/centos.py | 7 ++--- errata/sources/distros/debian.py | 8 +++--- errata/sources/distros/rocky.py | 12 +++++---- errata/sources/distros/ubuntu.py | 13 ++++++--- errata/sources/repos/yum.py | 4 +-- errata/tasks.py | 7 +++-- errata/utils.py | 4 +-- errata/views.py | 7 +++-- etc/patchman/local_settings.py | 4 ++- hooks/yum/patchman.py | 1 + hooks/zypper/patchman.py | 3 ++- hosts/admin.py | 1 + hosts/migrations/0001_initial.py | 3 ++- hosts/migrations/0002_initial.py | 2 +- .../0004_remove_host_tags_host_tags.py | 3 ++- hosts/migrations/0006_migrate_to_tz_aware.py | 1 + hosts/migrations/0007_alter_host_tags.py | 2 +- hosts/models.py | 3 ++- hosts/tasks.py | 1 - hosts/templatetags/report_alert.py | 2 +- hosts/utils.py | 4 +-- hosts/views.py | 23 ++++++++-------- modules/admin.py | 1 + modules/migrations/0001_initial.py | 2 +- modules/utils.py | 4 +-- modules/views.py | 7 +++-- operatingsystems/admin.py | 3 ++- operatingsystems/forms.py | 4 +-- operatingsystems/migrations/0002_initial.py | 2 +- operatingsystems/migrations/0003_os_arch.py | 2 +- operatingsystems/serializers.py | 2 +- operatingsystems/views.py | 17 +++++++----- packages/admin.py | 1 + packages/migrations/0001_initial.py | 2 +- .../migrations/0002_auto_20250207_1319.py | 2 +- packages/serializers.py | 2 +- packages/utils.py | 4 ++- packages/views.py | 13 ++++----- patchman/__init__.py | 3 +-- patchman/celery.py | 3 ++- patchman/receivers.py | 16 +++++------ patchman/urls.py | 3 +-- patchman/wsgi.py | 3 +-- reports/admin.py | 1 + .../migrations/0004_migrate_to_tz_aware.py | 1 + reports/models.py | 6 +++-- reports/tasks.py | 1 - reports/utils.py | 13 ++++++--- reports/views.py | 19 ++++++------- repos/admin.py | 3 ++- repos/forms.py | 7 +++-- repos/migrations/0001_initial.py | 2 +- repos/migrations/0003_migrate_to_tz_aware.py | 1 + repos/models.py | 9 +++---- repos/repo_types/arch.py | 9 ++++--- repos/repo_types/deb.py | 9 ++++--- repos/repo_types/gentoo.py | 16 +++++++---- repos/repo_types/rpm.py | 7 +++-- repos/repo_types/yast.py | 2 +- repos/repo_types/yum.py | 5 ++-- repos/serializers.py | 2 +- repos/tasks.py | 1 - repos/utils.py | 18 +++++++++---- repos/views.py | 27 ++++++++++--------- sbin/patchman | 21 +++++++++------ security/admin.py | 2 +- security/models.py | 4 +-- security/tasks.py | 1 - security/views.py | 11 ++++---- setup.py | 3 ++- util/__init__.py | 23 +++++++++------- util/filterspecs.py | 5 ++-- util/logging.py | 8 +++--- util/tasks.py | 4 ++- util/templatetags/common.py | 7 +++-- util/views.py | 6 ++--- 86 files changed, 292 insertions(+), 212 deletions(-) diff --git a/.github/workflows/lint-and-test.yml b/.github/workflows/lint-and-test.yml index 20b221aa..436698be 100644 --- a/.github/workflows/lint-and-test.yml +++ b/.github/workflows/lint-and-test.yml @@ -28,6 +28,10 @@ jobs: pip install flake8 flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics flake8 . --count --max-line-length=120 --show-source --statistics + - name: Check isort + run: | + pip install isort + isort --check --profile=django . - name: Set secret key run: ./sbin/patchman-set-secret-key - name: Test with django diff --git a/arch/admin.py b/arch/admin.py index 624a3720..5224711c 100644 --- a/arch/admin.py +++ b/arch/admin.py @@ -16,7 +16,8 @@ # along with Patchman. If not, see from django.contrib import admin -from arch.models import PackageArchitecture, MachineArchitecture + +from arch.models import MachineArchitecture, PackageArchitecture admin.site.register(PackageArchitecture) admin.site.register(MachineArchitecture) diff --git a/arch/serializers.py b/arch/serializers.py index 5319e796..a5765128 100644 --- a/arch/serializers.py +++ b/arch/serializers.py @@ -16,7 +16,7 @@ from rest_framework import serializers -from arch.models import PackageArchitecture, MachineArchitecture +from arch.models import MachineArchitecture, PackageArchitecture class PackageArchitectureSerializer(serializers.HyperlinkedModelSerializer): diff --git a/arch/utils.py b/arch/utils.py index 04d0b350..3db6ac70 100644 --- a/arch/utils.py +++ b/arch/utils.py @@ -14,7 +14,7 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -from arch.models import PackageArchitecture, MachineArchitecture +from arch.models import MachineArchitecture, PackageArchitecture from util.logging import info_message diff --git a/arch/views.py b/arch/views.py index 56a2a150..21f6b7c7 100644 --- a/arch/views.py +++ b/arch/views.py @@ -16,9 +16,10 @@ from rest_framework import viewsets -from arch.models import PackageArchitecture, MachineArchitecture -from arch.serializers import PackageArchitectureSerializer, \ - MachineArchitectureSerializer +from arch.models import MachineArchitecture, PackageArchitecture +from arch.serializers import ( + MachineArchitectureSerializer, PackageArchitectureSerializer, +) class PackageArchitectureViewSet(viewsets.ModelViewSet): diff --git a/domains/admin.py b/domains/admin.py index 2ef883e3..5cb0fee3 100644 --- a/domains/admin.py +++ b/domains/admin.py @@ -16,6 +16,7 @@ # along with Patchman. If not, see from django.contrib import admin + from domains.models import Domain admin.site.register(Domain) diff --git a/errata/admin.py b/errata/admin.py index 88190ff6..ac4b8a50 100644 --- a/errata/admin.py +++ b/errata/admin.py @@ -15,6 +15,7 @@ # along with Patchman. If not, see from django.contrib import admin + from errata.models import Erratum diff --git a/errata/models.py b/errata/models.py index cfc9bd0d..8c21bcfa 100644 --- a/errata/models.py +++ b/errata/models.py @@ -16,17 +16,16 @@ import json -from django.db import models +from django.db import IntegrityError, models from django.urls import reverse -from django.db import IntegrityError +from errata.managers import ErratumManager from packages.models import Package, PackageUpdate from packages.utils import find_evr, get_matching_packages -from errata.managers import ErratumManager from security.models import CVE, Reference from security.utils import get_or_create_cve, get_or_create_reference -from util.logging import error_message from util import get_url +from util.logging import error_message class Erratum(models.Model): diff --git a/errata/sources/distros/alma.py b/errata/sources/distros/alma.py index e0f2d4ae..0091b8bf 100644 --- a/errata/sources/distros/alma.py +++ b/errata/sources/distros/alma.py @@ -22,8 +22,8 @@ from operatingsystems.utils import get_or_create_osrelease from packages.models import Package from packages.utils import get_or_create_package, parse_package_string -from util import get_url, fetch_content, get_setting_of_type from patchman.signals import pbar_start, pbar_update +from util import fetch_content, get_setting_of_type, get_url def update_alma_errata(concurrent_processing=True): diff --git a/errata/sources/distros/arch.py b/errata/sources/distros/arch.py index 87c6c47a..e22de403 100644 --- a/errata/sources/distros/arch.py +++ b/errata/sources/distros/arch.py @@ -20,11 +20,13 @@ from django.db import connections from operatingsystems.utils import get_or_create_osrelease -from util.logging import error_message -from patchman.signals import pbar_start, pbar_update from packages.models import Package -from packages.utils import find_evr, get_matching_packages, get_or_create_package -from util import get_url, fetch_content +from packages.utils import ( + find_evr, get_matching_packages, get_or_create_package, +) +from patchman.signals import pbar_start, pbar_update +from util import fetch_content, get_url +from util.logging import error_message def update_arch_errata(concurrent_processing=False): diff --git a/errata/sources/distros/centos.py b/errata/sources/distros/centos.py index d2722a6b..8f4aa4a1 100644 --- a/errata/sources/distros/centos.py +++ b/errata/sources/distros/centos.py @@ -15,14 +15,15 @@ # along with Patchman. If not, see import re + from defusedxml import ElementTree from operatingsystems.utils import get_or_create_osrelease from packages.models import Package -from packages.utils import parse_package_string, get_or_create_package -from util.logging import error_message +from packages.utils import get_or_create_package, parse_package_string from patchman.signals import pbar_start, pbar_update -from util import bunzip2, get_url, fetch_content, get_sha1, get_setting_of_type +from util import bunzip2, fetch_content, get_setting_of_type, get_sha1, get_url +from util.logging import error_message def update_centos_errata(): diff --git a/errata/sources/distros/debian.py b/errata/sources/distros/debian.py index ece3754d..8025b1bf 100644 --- a/errata/sources/distros/debian.py +++ b/errata/sources/distros/debian.py @@ -18,18 +18,18 @@ import csv import re from datetime import datetime -from debian.deb822 import Dsc from io import StringIO +from debian.deb822 import Dsc from django.db import connections from operatingsystems.models import OSRelease from operatingsystems.utils import get_or_create_osrelease from packages.models import Package -from packages.utils import get_or_create_package, find_evr -from util.logging import error_message, warning_message +from packages.utils import find_evr, get_or_create_package from patchman.signals import pbar_start, pbar_update -from util import get_url, fetch_content, get_setting_of_type, extract +from util import extract, fetch_content, get_setting_of_type, get_url +from util.logging import error_message, warning_message DSCs = {} diff --git a/errata/sources/distros/rocky.py b/errata/sources/distros/rocky.py index 16d4d12c..2d805985 100644 --- a/errata/sources/distros/rocky.py +++ b/errata/sources/distros/rocky.py @@ -14,19 +14,21 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -import json import concurrent.futures -from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential +import json from django.db import connections from django.db.utils import OperationalError +from tenacity import ( + retry, retry_if_exception_type, stop_after_attempt, wait_exponential, +) from operatingsystems.utils import get_or_create_osrelease from packages.models import Package -from packages.utils import parse_package_string, get_or_create_package +from packages.utils import get_or_create_package, parse_package_string from patchman.signals import pbar_start, pbar_update -from util import get_url, fetch_content -from util.logging import info_message, error_message +from util import fetch_content, get_url +from util.logging import error_message, info_message def update_rocky_errata(concurrent_processing=True): diff --git a/errata/sources/distros/ubuntu.py b/errata/sources/distros/ubuntu.py index 6fafb40a..5616331f 100644 --- a/errata/sources/distros/ubuntu.py +++ b/errata/sources/distros/ubuntu.py @@ -16,8 +16,8 @@ import concurrent.futures import csv -import os import json +import os from io import StringIO from urllib.parse import urlparse @@ -26,10 +26,15 @@ from operatingsystems.models import OSRelease, OSVariant from operatingsystems.utils import get_or_create_osrelease from packages.models import Package -from packages.utils import get_or_create_package, parse_package_string, find_evr, get_matching_packages -from util import get_url, fetch_content, get_sha256, bunzip2, get_setting_of_type -from util.logging import error_message +from packages.utils import ( + find_evr, get_matching_packages, get_or_create_package, + parse_package_string, +) from patchman.signals import pbar_start, pbar_update +from util import ( + bunzip2, fetch_content, get_setting_of_type, get_sha256, get_url, +) +from util.logging import error_message def update_ubuntu_errata(concurrent_processing=False): diff --git a/errata/sources/repos/yum.py b/errata/sources/repos/yum.py index f361d10e..8b6732c4 100644 --- a/errata/sources/repos/yum.py +++ b/errata/sources/repos/yum.py @@ -16,17 +16,17 @@ import concurrent.futures from io import BytesIO -from defusedxml import ElementTree +from defusedxml import ElementTree from django.db import connections from operatingsystems.utils import get_or_create_osrelease from packages.models import Package from packages.utils import get_or_create_package -from util.logging import error_message from patchman.signals import pbar_start, pbar_update from security.models import Reference from util import extract, get_url +from util.logging import error_message def extract_updateinfo(data, url, concurrent_processing=True): diff --git a/errata/tasks.py b/errata/tasks.py index d9aaf3d4..8ded3a79 100644 --- a/errata/tasks.py +++ b/errata/tasks.py @@ -15,19 +15,18 @@ # along with Patchman. If not, see from celery import shared_task - from django.core.cache import cache -from errata.sources.distros.arch import update_arch_errata from errata.sources.distros.alma import update_alma_errata -from errata.sources.distros.debian import update_debian_errata +from errata.sources.distros.arch import update_arch_errata from errata.sources.distros.centos import update_centos_errata +from errata.sources.distros.debian import update_debian_errata from errata.sources.distros.rocky import update_rocky_errata from errata.sources.distros.ubuntu import update_ubuntu_errata -from util.logging import error_message, warning_message from repos.models import Repository from security.tasks import update_cves, update_cwes from util import get_setting_of_type +from util.logging import error_message, warning_message @shared_task diff --git a/errata/utils.py b/errata/utils.py index a8d8d424..e0a5e01b 100644 --- a/errata/utils.py +++ b/errata/utils.py @@ -18,11 +18,11 @@ from django.db import connections -from util import tz_aware_datetime from errata.models import Erratum from packages.models import PackageUpdate -from util.logging import warning_message from patchman.signals import pbar_start, pbar_update +from util import tz_aware_datetime +from util.logging import warning_message def get_or_create_erratum(name, e_type, issue_date, synopsis): diff --git a/errata/views.py b/errata/views.py index 42d12f71..8e1c0b2f 100644 --- a/errata/views.py +++ b/errata/views.py @@ -14,16 +14,15 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -from django.shortcuts import get_object_or_404, render from django.contrib.auth.decorators import login_required -from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger +from django.core.paginator import EmptyPage, PageNotAnInteger, Paginator from django.db.models import Q - +from django.shortcuts import get_object_or_404, render from rest_framework import viewsets -from operatingsystems.models import OSRelease from errata.models import Erratum from errata.serializers import ErratumSerializer +from operatingsystems.models import OSRelease from util.filterspecs import Filter, FilterBar diff --git a/etc/patchman/local_settings.py b/etc/patchman/local_settings.py index f7b98cf8..40dd2efd 100644 --- a/etc/patchman/local_settings.py +++ b/etc/patchman/local_settings.py @@ -66,8 +66,10 @@ } } -from datetime import timedelta # noqa +from datetime import timedelta # noqa + from celery.schedules import crontab # noqa + CELERY_BEAT_SCHEDULE = { 'process_all_unprocessed_reports': { 'task': 'reports.tasks.process_reports', diff --git a/hooks/yum/patchman.py b/hooks/yum/patchman.py index 343144eb..52f9cc8b 100644 --- a/hooks/yum/patchman.py +++ b/hooks/yum/patchman.py @@ -15,6 +15,7 @@ # along with Patchman. If not, see import os + from yum.plugins import TYPE_CORE requires_api_version = '2.1' diff --git a/hooks/zypper/patchman.py b/hooks/zypper/patchman.py index 14781565..d9d478f3 100755 --- a/hooks/zypper/patchman.py +++ b/hooks/zypper/patchman.py @@ -18,8 +18,9 @@ # # zypp system plugin for patchman -import os import logging +import os + from zypp_plugin import Plugin diff --git a/hosts/admin.py b/hosts/admin.py index 8a42e8cc..43bf31da 100644 --- a/hosts/admin.py +++ b/hosts/admin.py @@ -16,6 +16,7 @@ # along with Patchman. If not, see from django.contrib import admin + from hosts.models import Host, HostRepo diff --git a/hosts/migrations/0001_initial.py b/hosts/migrations/0001_initial.py index 43366684..0037e094 100644 --- a/hosts/migrations/0001_initial.py +++ b/hosts/migrations/0001_initial.py @@ -1,8 +1,9 @@ # Generated by Django 3.2.19 on 2023-12-11 22:15 -from django.db import migrations, models import django.db.models.deletion import django.utils.timezone +from django.db import migrations, models + try: import tagging.fields has_tagging = True diff --git a/hosts/migrations/0002_initial.py b/hosts/migrations/0002_initial.py index cc59a70e..6c453c49 100644 --- a/hosts/migrations/0002_initial.py +++ b/hosts/migrations/0002_initial.py @@ -1,7 +1,7 @@ # Generated by Django 3.2.19 on 2023-12-11 22:15 -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/hosts/migrations/0004_remove_host_tags_host_tags.py b/hosts/migrations/0004_remove_host_tags_host_tags.py index 84e7affe..bf03a84e 100644 --- a/hosts/migrations/0004_remove_host_tags_host_tags.py +++ b/hosts/migrations/0004_remove_host_tags_host_tags.py @@ -1,8 +1,9 @@ # Generated by Django 4.2.18 on 2025-02-04 23:37 +import taggit.managers from django.apps import apps from django.db import migrations -import taggit.managers + try: import tagging # noqa except ImportError: diff --git a/hosts/migrations/0006_migrate_to_tz_aware.py b/hosts/migrations/0006_migrate_to_tz_aware.py index e36bbf1f..c14ea50b 100644 --- a/hosts/migrations/0006_migrate_to_tz_aware.py +++ b/hosts/migrations/0006_migrate_to_tz_aware.py @@ -1,6 +1,7 @@ from django.db import migrations from django.utils import timezone + def make_datetimes_tz_aware(apps, schema_editor): Host = apps.get_model('hosts', 'Host') for host in Host.objects.all(): diff --git a/hosts/migrations/0007_alter_host_tags.py b/hosts/migrations/0007_alter_host_tags.py index 3858b847..3910a06f 100644 --- a/hosts/migrations/0007_alter_host_tags.py +++ b/hosts/migrations/0007_alter_host_tags.py @@ -1,7 +1,7 @@ # Generated by Django 4.2.19 on 2025-02-28 19:53 -from django.db import migrations import taggit.managers +from django.db import migrations class Migration(migrations.Migration): diff --git a/hosts/models.py b/hosts/models.py index 650544dc..8ea5e3d5 100644 --- a/hosts/models.py +++ b/hosts/models.py @@ -24,6 +24,7 @@ from version_utils.rpm import labelCompare except ImportError: from rpm import labelCompare + from taggit.managers import TaggableManager from arch.models import MachineArchitecture @@ -34,9 +35,9 @@ from operatingsystems.models import OSVariant from packages.models import Package, PackageUpdate from packages.utils import get_or_create_package_update -from util.logging import info_message from repos.models import Repository from repos.utils import find_best_repo +from util.logging import info_message class Host(models.Model): diff --git a/hosts/tasks.py b/hosts/tasks.py index 1643901d..226652f6 100755 --- a/hosts/tasks.py +++ b/hosts/tasks.py @@ -15,7 +15,6 @@ # along with Patchman. If not, see from celery import shared_task - from django.db.models import Count from hosts.models import Host diff --git a/hosts/templatetags/report_alert.py b/hosts/templatetags/report_alert.py index a28c5058..48d8f966 100644 --- a/hosts/templatetags/report_alert.py +++ b/hosts/templatetags/report_alert.py @@ -17,9 +17,9 @@ from datetime import timedelta from django.template import Library -from django.utils.html import format_html from django.templatetags.static import static from django.utils import timezone +from django.utils.html import format_html from util import get_setting_of_type diff --git a/hosts/utils.py b/hosts/utils.py index d6e663cf..44441f9b 100644 --- a/hosts/utils.py +++ b/hosts/utils.py @@ -15,9 +15,9 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -from socket import gethostbyaddr, gaierror, herror +from socket import gaierror, gethostbyaddr, herror -from django.db import transaction, IntegrityError +from django.db import IntegrityError, transaction from taggit.models import Tag from util.logging import error_message, info_message diff --git a/hosts/views.py b/hosts/views.py index 0fc83ffa..8f20ab19 100644 --- a/hosts/views.py +++ b/hosts/views.py @@ -15,24 +15,23 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -from django.shortcuts import get_object_or_404, render, redirect +from django.contrib import messages from django.contrib.auth.decorators import login_required -from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger -from django.urls import reverse +from django.core.paginator import EmptyPage, PageNotAnInteger, Paginator from django.db.models import Q -from django.contrib import messages - -from taggit.models import Tag +from django.shortcuts import get_object_or_404, redirect, render +from django.urls import reverse from rest_framework import viewsets +from taggit.models import Tag -from util.filterspecs import Filter, FilterBar -from hosts.models import Host, HostRepo -from domains.models import Domain from arch.models import MachineArchitecture -from operatingsystems.models import OSVariant, OSRelease -from reports.models import Report +from domains.models import Domain from hosts.forms import EditHostForm -from hosts.serializers import HostSerializer, HostRepoSerializer +from hosts.models import Host, HostRepo +from hosts.serializers import HostRepoSerializer, HostSerializer +from operatingsystems.models import OSRelease, OSVariant +from reports.models import Report +from util.filterspecs import Filter, FilterBar @login_required diff --git a/modules/admin.py b/modules/admin.py index 33b94d20..9cf21e6c 100644 --- a/modules/admin.py +++ b/modules/admin.py @@ -15,6 +15,7 @@ # along with Patchman. If not, see from django.contrib import admin + from modules.models import Module admin.site.register(Module) diff --git a/modules/migrations/0001_initial.py b/modules/migrations/0001_initial.py index 12a8e278..9c27d425 100644 --- a/modules/migrations/0001_initial.py +++ b/modules/migrations/0001_initial.py @@ -1,7 +1,7 @@ # Generated by Django 3.2.19 on 2023-12-11 22:17 -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/modules/utils.py b/modules/utils.py index 05c57c80..0d669478 100644 --- a/modules/utils.py +++ b/modules/utils.py @@ -15,10 +15,10 @@ # along with Patchman. If not, see from django.db import IntegrityError -from util.logging import error_message, info_message -from modules.models import Module from arch.models import PackageArchitecture +from modules.models import Module +from util.logging import error_message, info_message def get_or_create_module(name, stream, version, context, arch, repo): diff --git a/modules/views.py b/modules/views.py index b897a709..2d017220 100644 --- a/modules/views.py +++ b/modules/views.py @@ -14,12 +14,11 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -from django.shortcuts import get_object_or_404, render from django.contrib.auth.decorators import login_required -from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger +from django.core.paginator import EmptyPage, PageNotAnInteger, Paginator from django.db.models import Q - -from rest_framework import viewsets, permissions +from django.shortcuts import get_object_or_404, render +from rest_framework import permissions, viewsets from modules.models import Module from modules.serializers import ModuleSerializer diff --git a/operatingsystems/admin.py b/operatingsystems/admin.py index 15f5e200..4884b5eb 100644 --- a/operatingsystems/admin.py +++ b/operatingsystems/admin.py @@ -16,7 +16,8 @@ # along with Patchman. If not, see from django.contrib import admin -from operatingsystems.models import OSVariant, OSRelease + +from operatingsystems.models import OSRelease, OSVariant class OSReleaseAdmin(admin.ModelAdmin): diff --git a/operatingsystems/forms.py b/operatingsystems/forms.py index 548a7d88..fa319182 100644 --- a/operatingsystems/forms.py +++ b/operatingsystems/forms.py @@ -15,10 +15,10 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -from django.forms import ModelForm, ModelMultipleChoiceField from django.contrib.admin.widgets import FilteredSelectMultiple +from django.forms import ModelForm, ModelMultipleChoiceField -from operatingsystems.models import OSVariant, OSRelease +from operatingsystems.models import OSRelease, OSVariant from repos.models import Repository diff --git a/operatingsystems/migrations/0002_initial.py b/operatingsystems/migrations/0002_initial.py index 517a3f9a..04cbb411 100644 --- a/operatingsystems/migrations/0002_initial.py +++ b/operatingsystems/migrations/0002_initial.py @@ -1,7 +1,7 @@ # Generated by Django 3.2.19 on 2023-12-11 22:15 -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/operatingsystems/migrations/0003_os_arch.py b/operatingsystems/migrations/0003_os_arch.py index 2778ca3f..4d0e0f93 100644 --- a/operatingsystems/migrations/0003_os_arch.py +++ b/operatingsystems/migrations/0003_os_arch.py @@ -1,7 +1,7 @@ # Generated by Django 3.2.25 on 2025-02-07 13:02 -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/operatingsystems/serializers.py b/operatingsystems/serializers.py index 8418c720..be178909 100644 --- a/operatingsystems/serializers.py +++ b/operatingsystems/serializers.py @@ -16,7 +16,7 @@ from rest_framework import serializers -from operatingsystems.models import OSVariant, OSRelease +from operatingsystems.models import OSRelease, OSVariant class OSVariantSerializer(serializers.HyperlinkedModelSerializer): diff --git a/operatingsystems/views.py b/operatingsystems/views.py index 2b696f92..6009f119 100644 --- a/operatingsystems/views.py +++ b/operatingsystems/views.py @@ -15,19 +15,22 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -from django.shortcuts import get_object_or_404, render, redirect +from django.contrib import messages from django.contrib.auth.decorators import login_required -from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger +from django.core.paginator import EmptyPage, PageNotAnInteger, Paginator from django.db.models import Q -from django.contrib import messages +from django.shortcuts import get_object_or_404, redirect, render from django.urls import reverse - from rest_framework import viewsets from hosts.models import Host -from operatingsystems.models import OSVariant, OSRelease -from operatingsystems.forms import AddOSVariantToOSReleaseForm, AddReposToOSReleaseForm, CreateOSReleaseForm -from operatingsystems.serializers import OSVariantSerializer, OSReleaseSerializer +from operatingsystems.forms import ( + AddOSVariantToOSReleaseForm, AddReposToOSReleaseForm, CreateOSReleaseForm, +) +from operatingsystems.models import OSRelease, OSVariant +from operatingsystems.serializers import ( + OSReleaseSerializer, OSVariantSerializer, +) @login_required diff --git a/packages/admin.py b/packages/admin.py index 979ba779..bc4b1aaa 100644 --- a/packages/admin.py +++ b/packages/admin.py @@ -16,6 +16,7 @@ # along with Patchman. If not, see from django.contrib import admin + from packages.models import Package, PackageName, PackageUpdate diff --git a/packages/migrations/0001_initial.py b/packages/migrations/0001_initial.py index 07e7bcb0..bfea3e1a 100644 --- a/packages/migrations/0001_initial.py +++ b/packages/migrations/0001_initial.py @@ -1,7 +1,7 @@ # Generated by Django 3.2.19 on 2023-12-11 22:15 -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/packages/migrations/0002_auto_20250207_1319.py b/packages/migrations/0002_auto_20250207_1319.py index 1563d139..4c744203 100644 --- a/packages/migrations/0002_auto_20250207_1319.py +++ b/packages/migrations/0002_auto_20250207_1319.py @@ -1,7 +1,7 @@ # Generated by Django 3.2.25 on 2025-02-07 13:19 -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/packages/serializers.py b/packages/serializers.py index 902cb3e0..b6e3fb83 100644 --- a/packages/serializers.py +++ b/packages/serializers.py @@ -16,7 +16,7 @@ from rest_framework import serializers -from packages.models import PackageName, Package, PackageUpdate +from packages.models import Package, PackageName, PackageUpdate class PackageNameSerializer(serializers.HyperlinkedModelSerializer): diff --git a/packages/utils.py b/packages/utils.py index f00f6710..87395ff6 100644 --- a/packages/utils.py +++ b/packages/utils.py @@ -21,7 +21,9 @@ from django.db import IntegrityError, transaction from arch.models import PackageArchitecture -from packages.models import PackageName, Package, PackageUpdate, PackageCategory, PackageString +from packages.models import ( + Package, PackageCategory, PackageName, PackageString, PackageUpdate, +) from util.logging import error_message, info_message, warning_message diff --git a/packages/views.py b/packages/views.py index c55a6c72..413faee0 100644 --- a/packages/views.py +++ b/packages/views.py @@ -15,17 +15,18 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -from django.shortcuts import get_object_or_404, render from django.contrib.auth.decorators import login_required -from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger +from django.core.paginator import EmptyPage, PageNotAnInteger, Paginator from django.db.models import Q - +from django.shortcuts import get_object_or_404, render from rest_framework import viewsets -from util.filterspecs import Filter, FilterBar -from packages.models import PackageName, Package, PackageUpdate from arch.models import PackageArchitecture -from packages.serializers import PackageNameSerializer, PackageSerializer, PackageUpdateSerializer +from packages.models import Package, PackageName, PackageUpdate +from packages.serializers import ( + PackageNameSerializer, PackageSerializer, PackageUpdateSerializer, +) +from util.filterspecs import Filter, FilterBar @login_required diff --git a/patchman/__init__.py b/patchman/__init__.py index af122cc6..321dd7e5 100644 --- a/patchman/__init__.py +++ b/patchman/__init__.py @@ -14,10 +14,9 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -from .receivers import * # noqa - # This will make sure the app is always imported when # Django starts so that shared_task will use this app. from .celery import app as celery_app +from .receivers import * # noqa __all__ = ('celery_app',) diff --git a/patchman/celery.py b/patchman/celery.py index 3c58edc5..c47f994d 100644 --- a/patchman/celery.py +++ b/patchman/celery.py @@ -15,10 +15,11 @@ # along with Patchman. If not, see import os + from celery import Celery os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'patchman.settings') # noqa -from django.conf import settings # noqa +from django.conf import settings # noqa app = Celery('patchman') app.config_from_object('django.conf:settings', namespace='CELERY') diff --git a/patchman/receivers.py b/patchman/receivers.py index 8d8893ca..19312ed5 100644 --- a/patchman/receivers.py +++ b/patchman/receivers.py @@ -15,16 +15,16 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -from colorama import init, Fore, Style -from tqdm import tqdm - +from colorama import Fore, Style, init +from django.conf import settings from django.dispatch import receiver +from tqdm import tqdm -from util import create_pbar, update_pbar, get_verbosity -from patchman.signals import pbar_start, pbar_update, \ - info_message_s, warning_message_s, error_message_s, debug_message_s - -from django.conf import settings +from patchman.signals import ( + debug_message_s, error_message_s, info_message_s, pbar_start, pbar_update, + warning_message_s, +) +from util import create_pbar, get_verbosity, update_pbar init(autoreset=True) diff --git a/patchman/urls.py b/patchman/urls.py index ee786566..2ae64f56 100644 --- a/patchman/urls.py +++ b/patchman/urls.py @@ -15,12 +15,11 @@ # You should have received a copy of the GNU General Public License # along with If not, see -from django.conf.urls import include, handler404, handler500 # noqa from django.conf import settings +from django.conf.urls import handler404, handler500, include # noqa from django.contrib import admin from django.urls import path from django.views import static - from rest_framework import routers from arch import views as arch_views diff --git a/patchman/wsgi.py b/patchman/wsgi.py index 9a9b4b7f..16f02d5a 100644 --- a/patchman/wsgi.py +++ b/patchman/wsgi.py @@ -19,7 +19,6 @@ from django.core.wsgi import get_wsgi_application os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'patchman.settings') # noqa -from django.conf import settings # noqa - +from django.conf import settings # noqa application = get_wsgi_application() diff --git a/reports/admin.py b/reports/admin.py index a37ec4d0..66e0bf5b 100644 --- a/reports/admin.py +++ b/reports/admin.py @@ -16,6 +16,7 @@ # along with Patchman. If not, see from django.contrib import admin + from reports.models import Report diff --git a/reports/migrations/0004_migrate_to_tz_aware.py b/reports/migrations/0004_migrate_to_tz_aware.py index 98176510..20510dcd 100644 --- a/reports/migrations/0004_migrate_to_tz_aware.py +++ b/reports/migrations/0004_migrate_to_tz_aware.py @@ -1,6 +1,7 @@ from django.db import migrations from django.utils import timezone + def make_datetimes_tz_aware(apps, schema_editor): Report = apps.get_model('reports', 'Report') for report in Report.objects.all(): diff --git a/reports/models.py b/reports/models.py index d529804b..f1e0f6f3 100644 --- a/reports/models.py +++ b/reports/models.py @@ -104,7 +104,7 @@ def process(self, find_updates=True, verbose=False): info_message(text=f'Report {self.id} has already been processed') return - from reports.utils import get_arch, get_os, get_domain + from reports.utils import get_arch, get_domain, get_os arch = get_arch(self.arch) osvariant = get_os(self.os, arch) domain = get_domain(self.domain) @@ -113,7 +113,9 @@ def process(self, find_updates=True, verbose=False): if verbose: info_message(text=f'Processing report {self.id} - {self.host}') - from reports.utils import process_packages, process_repos, process_updates, process_modules + from reports.utils import ( + process_modules, process_packages, process_repos, process_updates, + ) process_repos(report=self, host=host) process_modules(report=self, host=host) process_packages(report=self, host=host) diff --git a/reports/tasks.py b/reports/tasks.py index d2a47e8f..07fb3004 100755 --- a/reports/tasks.py +++ b/reports/tasks.py @@ -16,7 +16,6 @@ # along with Patchman. If not, see from celery import shared_task - from django.core.cache import cache from django.db.utils import OperationalError diff --git a/reports/utils.py b/reports/utils.py index 76b6e09c..8b12f046 100644 --- a/reports/utils.py +++ b/reports/utils.py @@ -23,13 +23,18 @@ from domains.models import Domain from hosts.models import HostRepo from modules.utils import get_or_create_module -from operatingsystems.utils import get_or_create_osrelease, get_or_create_osvariant +from operatingsystems.utils import ( + get_or_create_osrelease, get_or_create_osvariant, +) from packages.models import Package, PackageCategory -from packages.utils import find_evr, get_or_create_package, get_or_create_package_update, parse_package_string -from util.logging import info_message +from packages.utils import ( + find_evr, get_or_create_package, get_or_create_package_update, + parse_package_string, +) from patchman.signals import pbar_start, pbar_update -from repos.models import Repository, Mirror, MirrorPackage +from repos.models import Mirror, MirrorPackage, Repository from repos.utils import get_or_create_repo +from util.logging import info_message def process_repos(report, host): diff --git a/reports/views.py b/reports/views.py index ccef1bb2..f247deb2 100644 --- a/reports/views.py +++ b/reports/views.py @@ -15,20 +15,21 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential - -from django.http import HttpResponse, Http404 -from django.views.decorators.csrf import csrf_exempt -from django.shortcuts import get_object_or_404, render, redirect +from django.contrib import messages from django.contrib.auth.decorators import login_required -from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger -from django.urls import reverse +from django.core.paginator import EmptyPage, PageNotAnInteger, Paginator from django.db.models import Q -from django.contrib import messages from django.db.utils import OperationalError +from django.http import Http404, HttpResponse +from django.shortcuts import get_object_or_404, redirect, render +from django.urls import reverse +from django.views.decorators.csrf import csrf_exempt +from tenacity import ( + retry, retry_if_exception_type, stop_after_attempt, wait_exponential, +) -from util.filterspecs import Filter, FilterBar from reports.models import Report +from util.filterspecs import Filter, FilterBar @retry( diff --git a/repos/admin.py b/repos/admin.py index bea87567..a516ff8a 100644 --- a/repos/admin.py +++ b/repos/admin.py @@ -16,7 +16,8 @@ # along with Patchman. If not, see from django.contrib import admin -from repos.models import Repository, Mirror, MirrorPackage + +from repos.models import Mirror, MirrorPackage, Repository class MirrorAdmin(admin.ModelAdmin): diff --git a/repos/forms.py b/repos/forms.py index 0800a5c3..9cb66897 100644 --- a/repos/forms.py +++ b/repos/forms.py @@ -15,10 +15,13 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -from django.forms import ModelForm, ModelMultipleChoiceField, TextInput, Form, ModelChoiceField, ValidationError from django.contrib.admin.widgets import FilteredSelectMultiple +from django.forms import ( + Form, ModelChoiceField, ModelForm, ModelMultipleChoiceField, TextInput, + ValidationError, +) -from repos.models import Repository, Mirror +from repos.models import Mirror, Repository class EditRepoForm(ModelForm): diff --git a/repos/migrations/0001_initial.py b/repos/migrations/0001_initial.py index a99f6878..1ae96a98 100644 --- a/repos/migrations/0001_initial.py +++ b/repos/migrations/0001_initial.py @@ -1,7 +1,7 @@ # Generated by Django 3.2.19 on 2023-12-11 22:15 -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/repos/migrations/0003_migrate_to_tz_aware.py b/repos/migrations/0003_migrate_to_tz_aware.py index dddd78ba..38e30488 100644 --- a/repos/migrations/0003_migrate_to_tz_aware.py +++ b/repos/migrations/0003_migrate_to_tz_aware.py @@ -1,6 +1,7 @@ from django.db import migrations from django.utils import timezone + def make_datetimes_tz_aware(apps, schema_editor): Mirror = apps.get_model('repos', 'Mirror') for mirror in Mirror.objects.all(): diff --git a/repos/models.py b/repos/models.py index a1db2a93..9b9082af 100644 --- a/repos/models.py +++ b/repos/models.py @@ -20,13 +20,12 @@ from arch.models import MachineArchitecture from packages.models import Package -from util import get_setting_of_type - -from repos.repo_types.deb import refresh_deb_repo -from repos.repo_types.rpm import refresh_rpm_repo, refresh_repo_errata from repos.repo_types.arch import refresh_arch_repo +from repos.repo_types.deb import refresh_deb_repo from repos.repo_types.gentoo import refresh_gentoo_repo -from util.logging import info_message, warning_message, error_message +from repos.repo_types.rpm import refresh_repo_errata, refresh_rpm_repo +from util import get_setting_of_type +from util.logging import error_message, info_message, warning_message class Repository(models.Model): diff --git a/repos/repo_types/arch.py b/repos/repo_types/arch.py index 09719428..390b321d 100644 --- a/repos/repo_types/arch.py +++ b/repos/repo_types/arch.py @@ -18,10 +18,13 @@ from io import BytesIO from packages.models import PackageString -from util.logging import info_message, warning_message from patchman.signals import pbar_start, pbar_update -from repos.utils import get_max_mirrors, fetch_mirror_data, find_mirror_url, update_mirror_packages -from util import get_datetime_now, get_checksum, Checksum +from repos.utils import ( + fetch_mirror_data, find_mirror_url, get_max_mirrors, + update_mirror_packages, +) +from util import Checksum, get_checksum, get_datetime_now +from util.logging import info_message, warning_message def refresh_arch_repo(repo): diff --git a/repos/repo_types/deb.py b/repos/repo_types/deb.py index c6c26d78..33d1f2c4 100644 --- a/repos/repo_types/deb.py +++ b/repos/repo_types/deb.py @@ -15,14 +15,17 @@ # along with Patchman. If not, see import re + from debian.deb822 import Packages from debian.debian_support import Version from packages.models import PackageString -from util.logging import error_message, info_message, warning_message from patchman.signals import pbar_start, pbar_update -from repos.utils import fetch_mirror_data, update_mirror_packages, find_mirror_url -from util import get_datetime_now, get_checksum, Checksum, extract +from repos.utils import ( + fetch_mirror_data, find_mirror_url, update_mirror_packages, +) +from util import Checksum, extract, get_checksum, get_datetime_now +from util.logging import error_message, info_message, warning_message def extract_deb_packages(data, url): diff --git a/repos/repo_types/gentoo.py b/repos/repo_types/gentoo.py index e440f0d5..a0584618 100644 --- a/repos/repo_types/gentoo.py +++ b/repos/repo_types/gentoo.py @@ -14,22 +14,28 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -import git import os import shutil import tarfile import tempfile -from defusedxml import ElementTree from fnmatch import fnmatch from io import BytesIO from pathlib import Path +import git +from defusedxml import ElementTree + from packages.models import PackageString from packages.utils import find_evr -from util.logging import info_message, warning_message, error_message from patchman.signals import pbar_start, pbar_update -from repos.utils import add_mirrors_from_urls, mirror_checksum_is_valid, update_mirror_packages -from util import extract, get_url, get_datetime_now, get_checksum, Checksum, fetch_content, response_is_valid +from repos.utils import ( + add_mirrors_from_urls, mirror_checksum_is_valid, update_mirror_packages, +) +from util import ( + Checksum, extract, fetch_content, get_checksum, get_datetime_now, get_url, + response_is_valid, +) +from util.logging import error_message, info_message, warning_message def refresh_gentoo_main_repo(repo): diff --git a/repos/repo_types/rpm.py b/repos/repo_types/rpm.py index 51661809..5ffbb708 100644 --- a/repos/repo_types/rpm.py +++ b/repos/repo_types/rpm.py @@ -16,11 +16,14 @@ from django.db.models import Q -from util.logging import info_message, warning_message from repos.repo_types.yast import refresh_yast_repo from repos.repo_types.yum import refresh_yum_repo -from repos.utils import check_for_metalinks, check_for_mirrorlists, find_mirror_url, get_max_mirrors, fetch_mirror_data +from repos.utils import ( + check_for_metalinks, check_for_mirrorlists, fetch_mirror_data, + find_mirror_url, get_max_mirrors, +) from util import get_datetime_now +from util.logging import info_message, warning_message def refresh_repo_errata(repo): diff --git a/repos/repo_types/yast.py b/repos/repo_types/yast.py index bf594040..e37b9934 100644 --- a/repos/repo_types/yast.py +++ b/repos/repo_types/yast.py @@ -17,10 +17,10 @@ import re from packages.models import PackageString -from util.logging import info_message from patchman.signals import pbar_start, pbar_update from repos.utils import fetch_mirror_data, update_mirror_packages from util import extract +from util.logging import info_message def refresh_yast_repo(mirror, data): diff --git a/repos/repo_types/yum.py b/repos/repo_types/yum.py index bc0fbc4b..1e96db39 100644 --- a/repos/repo_types/yum.py +++ b/repos/repo_types/yum.py @@ -15,17 +15,18 @@ # along with Patchman. If not, see from celery import shared_task - from django.core.cache import cache from repos.models import Repository diff --git a/repos/utils.py b/repos/utils.py index c11c41ad..29e9cdb3 100644 --- a/repos/utils.py +++ b/repos/utils.py @@ -17,17 +17,24 @@ import re from io import BytesIO -from defusedxml import ElementTree -from tenacity import RetryError +from defusedxml import ElementTree from django.db import IntegrityError from django.db.models import Q +from tenacity import RetryError from packages.models import Package -from packages.utils import convert_package_to_packagestring, convert_packagestring_to_package -from util import get_url, fetch_content, response_is_valid, extract, get_checksum, Checksum, get_setting_of_type -from util.logging import info_message, warning_message, error_message, debug_message +from packages.utils import ( + convert_package_to_packagestring, convert_packagestring_to_package, +) from patchman.signals import pbar_start, pbar_update +from util import ( + Checksum, extract, fetch_content, get_checksum, get_setting_of_type, + get_url, response_is_valid, +) +from util.logging import ( + debug_message, error_message, info_message, warning_message, +) def get_or_create_repo(r_name, r_arch, r_type, r_id=None): @@ -176,6 +183,7 @@ def add_mirrors_from_urls(repo, mirror_urls): warning_message(text=text) break from repos.models import Mirror + # FIXME: maybe we should store the mirrorlist url with full path to repomd.xml? # that is what metalink urls return now m, c = Mirror.objects.get_or_create(repo=repo, url=mirror_url.rstrip('/').replace('repodata/repomd.xml', '')) diff --git a/repos/views.py b/repos/views.py index 199c834e..1f0c2bfa 100644 --- a/repos/views.py +++ b/repos/views.py @@ -15,24 +15,27 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -from django.shortcuts import get_object_or_404, render, redirect -from django.http import HttpResponse -from django.contrib.auth.decorators import login_required -from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger -from django.urls import reverse -from django.db.models import Q from django.contrib import messages +from django.contrib.auth.decorators import login_required +from django.core.paginator import EmptyPage, PageNotAnInteger, Paginator from django.db import IntegrityError - +from django.db.models import Q +from django.http import HttpResponse +from django.shortcuts import get_object_or_404, redirect, render +from django.urls import reverse from rest_framework import viewsets -from util.filterspecs import Filter, FilterBar +from arch.models import MachineArchitecture from hosts.models import HostRepo -from repos.models import Repository, Mirror, MirrorPackage from operatingsystems.models import OSRelease -from arch.models import MachineArchitecture -from repos.forms import EditRepoForm, LinkRepoForm, CreateRepoForm, EditMirrorForm -from repos.serializers import RepositorySerializer, MirrorSerializer, MirrorPackageSerializer +from repos.forms import ( + CreateRepoForm, EditMirrorForm, EditRepoForm, LinkRepoForm, +) +from repos.models import Mirror, MirrorPackage, Repository +from repos.serializers import ( + MirrorPackageSerializer, MirrorSerializer, RepositorySerializer, +) +from util.filterspecs import Filter, FilterBar @login_required diff --git a/sbin/patchman b/sbin/patchman index 47d89b06..c415abec 100755 --- a/sbin/patchman +++ b/sbin/patchman @@ -17,32 +17,37 @@ # along with Patchman. If not, see +import argparse import os import sys -import argparse +from django import setup as django_setup from django.core.exceptions import MultipleObjectsReturned from django.db.models import Count -from django import setup as django_setup os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'patchman.settings') from django.conf import settings # noqa + django_setup() from arch.utils import clean_architectures -from errata.utils import mark_errata_security_updates, enrich_errata, \ - scan_package_updates_for_affected_packages from errata.tasks import update_errata +from errata.utils import ( + enrich_errata, mark_errata_security_updates, + scan_package_updates_for_affected_packages, +) from hosts.models import Host from hosts.utils import clean_tags from modules.utils import clean_modules -from packages.utils import clean_packages, clean_packageupdates, clean_packagenames -from repos.models import Repository -from repos.utils import clean_repos +from packages.utils import ( + clean_packagenames, clean_packages, clean_packageupdates, +) from reports.models import Report from reports.tasks import clean_reports_with_no_hosts +from repos.models import Repository +from repos.utils import clean_repos from security.utils import update_cves, update_cwes -from util import set_verbosity, get_datetime_now +from util import get_datetime_now, set_verbosity from util.logging import info_message diff --git a/security/admin.py b/security/admin.py index 196a9468..aedeaea9 100644 --- a/security/admin.py +++ b/security/admin.py @@ -15,8 +15,8 @@ # along with Patchman. If not, see from django.contrib import admin -from security.models import CWE, CVSS, CVE, Reference +from security.models import CVE, CVSS, CWE, Reference admin.site.register(CWE) admin.site.register(CVSS) diff --git a/security/models.py b/security/models.py index 7f674a0b..0f848260 100644 --- a/security/models.py +++ b/security/models.py @@ -16,14 +16,14 @@ import json import re -from cvss import CVSS2, CVSS3, CVSS4 from time import sleep +from cvss import CVSS2, CVSS3, CVSS4 from django.db import models from django.urls import reverse from security.managers import CVEManager -from util import get_url, fetch_content, tz_aware_datetime, error_message +from util import error_message, fetch_content, get_url, tz_aware_datetime class Reference(models.Model): diff --git a/security/tasks.py b/security/tasks.py index ce60df83..7bff4149 100644 --- a/security/tasks.py +++ b/security/tasks.py @@ -15,7 +15,6 @@ # along with Patchman. If not, see from celery import shared_task - from django.core.cache import cache from security.models import CVE, CWE diff --git a/security/views.py b/security/views.py index 58a686b5..c9e606a6 100644 --- a/security/views.py +++ b/security/views.py @@ -14,17 +14,18 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -from django.shortcuts import get_object_or_404, render from django.contrib.auth.decorators import login_required -from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger +from django.core.paginator import EmptyPage, PageNotAnInteger, Paginator from django.db.models import Q - +from django.shortcuts import get_object_or_404, render from rest_framework import viewsets -from packages.models import Package from operatingsystems.models import OSRelease +from packages.models import Package from security.models import CVE, CWE, Reference -from security.serializers import CVESerializer, CWESerializer, ReferenceSerializer +from security.serializers import ( + CVESerializer, CWESerializer, ReferenceSerializer, +) from util.filterspecs import Filter, FilterBar diff --git a/setup.py b/setup.py index 6ec6d974..8e18eaf9 100755 --- a/setup.py +++ b/setup.py @@ -17,7 +17,8 @@ # along with Patchman. If not, see import os -from setuptools import setup, find_packages + +from setuptools import find_packages, setup with open('VERSION.txt', 'r', encoding='utf_8') as v: version = v.readline().strip() diff --git a/util/__init__.py b/util/__init__.py index c3dfd6bd..c6c9aa0d 100644 --- a/util/__init__.py +++ b/util/__init__.py @@ -15,30 +15,35 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -import requests import bz2 -import magic -import zlib import lzma import os +import zlib + +import magic +import requests + try: # python 3.14+ - can also remove the dependency at that stage from compression import zstd except ImportError: import zstandard as zstd + from datetime import datetime, timezone from enum import Enum from hashlib import md5, sha1, sha256, sha512 -from requests.exceptions import HTTPError, Timeout, ConnectionError -from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential from time import time -from tqdm import tqdm -from django.utils.timezone import make_aware -from django.utils.dateparse import parse_datetime from django.conf import settings +from django.utils.dateparse import parse_datetime +from django.utils.timezone import make_aware +from requests.exceptions import ConnectionError, HTTPError, Timeout +from tenacity import ( + retry, retry_if_exception_type, stop_after_attempt, wait_exponential, +) +from tqdm import tqdm -from util.logging import error_message, info_message, debug_message +from util.logging import debug_message, error_message, info_message pbar = None verbose = None diff --git a/util/filterspecs.py b/util/filterspecs.py index 722b45df..eac0f747 100644 --- a/util/filterspecs.py +++ b/util/filterspecs.py @@ -15,10 +15,11 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -from django.utils.safestring import mark_safe -from django.db.models.query import QuerySet from operator import itemgetter +from django.db.models.query import QuerySet +from django.utils.safestring import mark_safe + def get_query_string(qs): new_qs = [f'{k}={v}' for k, v in list(qs.items())] diff --git a/util/logging.py b/util/logging.py index dd79d296..cb00ccce 100644 --- a/util/logging.py +++ b/util/logging.py @@ -14,12 +14,12 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see + from datetime import datetime -from patchman.signals import info_message_s -from patchman.signals import warning_message_s -from patchman.signals import error_message_s -from patchman.signals import debug_message_s +from patchman.signals import ( + debug_message_s, error_message_s, info_message_s, warning_message_s, +) def info_message(text): diff --git a/util/tasks.py b/util/tasks.py index f650e3e2..bd76bac6 100644 --- a/util/tasks.py +++ b/util/tasks.py @@ -18,7 +18,9 @@ from arch.utils import clean_architectures from modules.utils import clean_modules -from packages.utils import clean_packages, clean_packageupdates, clean_packagenames +from packages.utils import ( + clean_packagenames, clean_packages, clean_packageupdates, +) from repos.utils import clean_repos, remove_mirror_trailing_slashes diff --git a/util/templatetags/common.py b/util/templatetags/common.py index 2aea1e5e..674e1721 100644 --- a/util/templatetags/common.py +++ b/util/templatetags/common.py @@ -15,16 +15,15 @@ # along with Patchman. If not, see import re - -from humanize import naturaltime from datetime import datetime, timedelta from urllib.parse import urlencode +from django.core.paginator import Paginator from django.template import Library from django.template.loader import get_template -from django.utils.html import format_html from django.templatetags.static import static -from django.core.paginator import Paginator +from django.utils.html import format_html +from humanize import naturaltime from util import get_setting_of_type diff --git a/util/views.py b/util/views.py index b66db6b0..fd003bca 100644 --- a/util/views.py +++ b/util/views.py @@ -17,16 +17,16 @@ from datetime import datetime, timedelta -from django.shortcuts import render from django.contrib.auth.decorators import login_required from django.contrib.sites.models import Site from django.db.models import F +from django.shortcuts import render from hosts.models import Host -from operatingsystems.models import OSVariant, OSRelease -from repos.models import Repository, Mirror +from operatingsystems.models import OSRelease, OSVariant from packages.models import Package from reports.models import Report +from repos.models import Mirror, Repository from util import get_setting_of_type From 4c760e263e56a1a4cc63ab9452977c7f5df633d5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 6 Nov 2025 00:40:08 +0000 Subject: [PATCH 029/146] Bump django from 4.2.25 to 4.2.26 Bumps [django](https://github.com/django/django) from 4.2.25 to 4.2.26. - [Commits](https://github.com/django/django/compare/4.2.25...4.2.26) --- updated-dependencies: - dependency-name: django dependency-version: 4.2.26 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 788f4241..f67fc2ce 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -Django==4.2.25 +Django==4.2.26 django-taggit==4.0.0 django-extensions==3.2.3 django-bootstrap3==23.1 From 38857deb4f0c09b9504abbaeb914e051934242b7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 3 Dec 2025 14:57:03 +0000 Subject: [PATCH 030/146] Bump django from 4.2.26 to 4.2.27 Bumps [django](https://github.com/django/django) from 4.2.26 to 4.2.27. - [Commits](https://github.com/django/django/compare/4.2.26...4.2.27) --- updated-dependencies: - dependency-name: django dependency-version: 4.2.27 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f67fc2ce..08ce4573 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -Django==4.2.26 +Django==4.2.27 django-taggit==4.0.0 django-extensions==3.2.3 django-bootstrap3==23.1 From 206c4157e92e98c8d50c37eb58551626cabf1274 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Sat, 20 Dec 2025 08:24:21 +0000 Subject: [PATCH 031/146] Modified tag handling to preserve case --- hosts/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hosts/models.py b/hosts/models.py index 8ea5e3d5..4689ccc5 100644 --- a/hosts/models.py +++ b/hosts/models.py @@ -86,7 +86,7 @@ def show(self): text += f'Packages : {self.get_num_packages()}\n' text += f'Repos : {self.get_num_repos()}\n' text += f'Updates : {self.get_num_updates()}\n' - text += f'Tags : {" ".join(self.tags.slugs())}\n' + text += f'Tags : {" ".join(self.tags.names())}\n' text += f'Needs reboot : {self.reboot_required}\n' text += f'Updated at : {self.updated_at}\n' text += f'Host repos : {self.host_repos_only}\n' From 98306089ddcfb0aec9e8982f7bbfa0e13d91aacb Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Wed, 17 Dec 2025 14:11:34 -0500 Subject: [PATCH 032/146] fix same module in different repos --- .../0005_alter_module_unique_together.py | 19 +++++++++++++++++++ modules/models.py | 2 +- modules/utils.py | 5 ++--- 3 files changed, 22 insertions(+), 4 deletions(-) create mode 100644 modules/migrations/0005_alter_module_unique_together.py diff --git a/modules/migrations/0005_alter_module_unique_together.py b/modules/migrations/0005_alter_module_unique_together.py new file mode 100644 index 00000000..046a0c54 --- /dev/null +++ b/modules/migrations/0005_alter_module_unique_together.py @@ -0,0 +1,19 @@ +# Generated by Django 4.2.25 on 2025-11-25 16:37 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('arch', '0001_initial'), + ('repos', '0006_mirror_errata_checksum_mirror_modules_checksum'), + ('modules', '0004_alter_module_options'), + ] + + operations = [ + migrations.AlterUniqueTogether( + name='module', + unique_together={('name', 'stream', 'version', 'context', 'arch', 'repo')}, + ), + ] diff --git a/modules/models.py b/modules/models.py index 931a41c3..e1f8071b 100644 --- a/modules/models.py +++ b/modules/models.py @@ -35,7 +35,7 @@ class Module(models.Model): class Meta: verbose_name = 'Module' verbose_name_plural = 'Modules' - unique_together = ['name', 'stream', 'version', 'context', 'arch'] + unique_together = ['name', 'stream', 'version', 'context', 'arch', 'repo'] ordering = ['name', 'stream'] def __str__(self): diff --git a/modules/utils.py b/modules/utils.py index 0d669478..248b8b45 100644 --- a/modules/utils.py +++ b/modules/utils.py @@ -25,10 +25,9 @@ def get_or_create_module(name, stream, version, context, arch, repo): """ Get or create a module object Returns the module """ - created = False - m_arch, c = PackageArchitecture.objects.get_or_create(name=arch) + m_arch, _ = PackageArchitecture.objects.get_or_create(name=arch) try: - module, created = Module.objects.get_or_create( + module, _ = Module.objects.get_or_create( name=name, stream=stream, version=version, From e5ae168aa5d07d613b421432f0e2039c3e67280c Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Thu, 1 Jan 2026 17:48:35 -0500 Subject: [PATCH 033/146] add priority queues for tasks (#724) * add priority queues for tasks * Update repos/tasks.py Co-authored-by: code-review-doctor[bot] <72320148+code-review-doctor[bot]@users.noreply.github.com> --- errata/tasks.py | 6 ++--- etc/patchman/local_settings.py | 6 ++--- hosts/tasks.py | 6 ++--- patchman/settings.py | 5 ++++ reports/tasks.py | 47 ++++++++++++++++++++++++++-------- repos/tasks.py | 18 ++++++++++--- sbin/patchman | 4 +-- security/tasks.py | 38 +++++++++++++++++++++------ util/tasks.py | 2 +- 9 files changed, 97 insertions(+), 35 deletions(-) diff --git a/errata/tasks.py b/errata/tasks.py index 8ded3a79..9d1f3ed9 100644 --- a/errata/tasks.py +++ b/errata/tasks.py @@ -29,7 +29,7 @@ from util.logging import error_message, warning_message -@shared_task +@shared_task(priority=1) def update_yum_repo_errata(repo_id=None, force=False): """ Update all yum repos errata """ @@ -41,7 +41,7 @@ def update_yum_repo_errata(repo_id=None, force=False): repo.refresh_errata(force) -@shared_task +@shared_task(priority=1) def update_errata(erratum_type=None, force=False, repo=None): """ Update all distros errata """ @@ -85,7 +85,7 @@ def update_errata(erratum_type=None, force=False, repo=None): warning_message('Already updating Errata, skipping task.') -@shared_task +@shared_task(priority=2) def update_errata_and_cves(): """ Task to update all errata """ diff --git a/etc/patchman/local_settings.py b/etc/patchman/local_settings.py index 40dd2efd..15fcb60a 100644 --- a/etc/patchman/local_settings.py +++ b/etc/patchman/local_settings.py @@ -56,15 +56,15 @@ # Whether to run patchman under the gunicorn web server RUN_GUNICORN = False -# Set the default timeout to e.g. 30 seconds to enable UI caching -# Note that the UI results may be out of date for this amount of time CACHES = { 'default': { 'BACKEND': 'django.core.cache.backends.redis.RedisCache', 'LOCATION': 'redis://127.0.0.1:6379', - 'TIMEOUT': 0, } } +# Set the default timeout to e.g. 30 seconds to enable UI caching +# Note that the UI results may be out of date for this amount of time +CACHE_MIDDLEWARE_SECONDS = 0 from datetime import timedelta # noqa diff --git a/hosts/tasks.py b/hosts/tasks.py index 226652f6..f186760f 100755 --- a/hosts/tasks.py +++ b/hosts/tasks.py @@ -22,7 +22,7 @@ from util.logging import info_message -@shared_task +@shared_task(priority=0) def find_host_updates(host_id): """ Task to find updates for a host """ @@ -30,7 +30,7 @@ def find_host_updates(host_id): host.find_updates() -@shared_task +@shared_task(priority=1) def find_all_host_updates(): """ Task to find updates for all hosts """ @@ -38,7 +38,7 @@ def find_all_host_updates(): find_host_updates.delay(host.id) -@shared_task +@shared_task(priority=1) def find_all_host_updates_homogenous(): """ Task to find updates for all hosts where hosts are expected to be homogenous """ diff --git a/patchman/settings.py b/patchman/settings.py index 557e8c68..23e932c4 100644 --- a/patchman/settings.py +++ b/patchman/settings.py @@ -13,6 +13,7 @@ MIDDLEWARE = [ 'django.middleware.security.SecurityMiddleware', 'django.middleware.cache.UpdateCacheMiddleware', + 'patchman.middleware.NeverCacheMiddleware', 'django.middleware.http.ConditionalGetMiddleware', 'django.contrib.sessions.middleware.SessionMiddleware', 'django.middleware.common.CommonMiddleware', @@ -109,6 +110,10 @@ TAGGIT_CASE_INSENSITIVE = True CELERY_BROKER_URL = 'redis://127.0.0.1:6379/0' +CELERY_BROKER_TRANSPORT_OPTIONS = { + 'queue_order_strategy': 'priority', +} +CELERY_WORKER_PREFETCH_MULTIPLIER = 1 LOGIN_REDIRECT_URL = '/patchman/' LOGOUT_REDIRECT_URL = '/patchman/login/' diff --git a/reports/tasks.py b/reports/tasks.py index 07fb3004..6bf16e7c 100755 --- a/reports/tasks.py +++ b/reports/tasks.py @@ -24,25 +24,50 @@ from util.logging import info_message, warning_message -@shared_task(bind=True, autoretry_for=(OperationalError,), retry_backoff=True, retry_kwargs={'max_retries': 5}) +@shared_task( + bind=True, + priority=0, + autoretry_for=(OperationalError,), + retry_backoff=True, + retry_kwargs={'max_retries': 5} +) def process_report(self, report_id): """ Task to process a single report """ report = Report.objects.get(id=report_id) - lock_key = f'process_report_lock_{report_id}' - # lock will expire after 1 hour - lock_expire = 60 * 60 + report_id_lock_key = f'process_report_id_lock_{report_id}' + if report.host: + report_host_lock_key = f'process_report_host_lock_{report.host}' + else: + report_host_lock_key = f'process_report_host_lock_{report.report_ip}' + # locks will expire after 2 hours + lock_expire = 60 * 60 * 2 - if cache.add(lock_key, 'true', lock_expire): + if cache.add(report_id_lock_key, 'true', lock_expire): try: - report.process() + processing_report_id = cache.get(report_host_lock_key) + if processing_report_id: + if processing_report_id > report.id: + warning_message(f'Currently processing a newer report for {report.host} or {report.report_ip}, \ + marking report {report.id} as processed.') + report.processed = True + report.save() + else: + warning_message(f'Currently processing an older report for {report.host} or {report.report_ip}, \ + will skip processing this report.') + else: + try: + cache.set(report_host_lock_key, report.id, lock_expire) + report.process() + finally: + cache.delete(report_host_lock_key) finally: - cache.delete(lock_key) + cache.delete(report_id_lock_key) else: warning_message(f'Already processing report {report_id}, skipping task.') -@shared_task +@shared_task(priority=1) def process_reports(): """ Task to process all unprocessed reports """ @@ -51,9 +76,9 @@ def process_reports(): process_report.delay(report.id) -@shared_task -def clean_reports_with_no_hosts(): - """ Task to clean processed reports where the host no longer exists +@shared_task(priority=2) +def remove_reports_with_no_hosts(): + """ Task to remove processed reports where the host no longer exists """ for report in Report.objects.filter(processed=True): if not Host.objects.filter(hostname=report.host).exists(): diff --git a/repos/tasks.py b/repos/tasks.py index bc177653..a9fdd5f4 100644 --- a/repos/tasks.py +++ b/repos/tasks.py @@ -21,15 +21,25 @@ from util.logging import warning_message -@shared_task +@shared_task(priority=0) def refresh_repo(repo_id, force=False): """ Refresh metadata for a single repo """ - repo = Repository.objects.get(id=repo_id) - repo.refresh(force) + repo_id_lock_key = f'refresh_repos_{repo_id}_lock' + # lock will expire after 1 day + lock_expire = 60 * 60 * 24 + + if cache.add(repo_id_lock_key, 'true', lock_expire): + try: + repo = Repository.objects.get(id=repo_id) + repo.refresh(force) + finally: + cache.delete(repo_id_lock_key) + else: + warning_message(f'Already refreshing repo {repo_id}, skipping task.') -@shared_task +@shared_task(priority=1) def refresh_repos(force=False): """ Refresh metadata for all enabled repos """ diff --git a/sbin/patchman b/sbin/patchman index c415abec..b76272a2 100755 --- a/sbin/patchman +++ b/sbin/patchman @@ -43,7 +43,7 @@ from packages.utils import ( clean_packagenames, clean_packages, clean_packageupdates, ) from reports.models import Report -from reports.tasks import clean_reports_with_no_hosts +from reports.tasks import remove_reports_with_no_hosts from repos.models import Repository from repos.utils import clean_repos from security.utils import update_cves, update_cwes @@ -156,7 +156,7 @@ def clean_reports(hoststr=None): host.clean_reports() if not hoststr: - clean_reports_with_no_hosts() + remove_reports_with_no_hosts() def host_updates_alt(host=None): diff --git a/security/tasks.py b/security/tasks.py index 7bff4149..0cfbc2f1 100644 --- a/security/tasks.py +++ b/security/tasks.py @@ -21,15 +21,26 @@ from util.logging import warning_message -@shared_task +@shared_task(priority=3) def update_cve(cve_id): """ Task to update a CVE """ - cve = CVE.objects.get(id=cve_id) - cve.fetch_cve_data() + cve_id_lock_key = f'update_cve_id_lock_{cve_id}' + + # lock will expire after 1 week + lock_expire = 60 * 60 * 168 + + if cache.add(cve_id_lock_key, 'true', lock_expire): + try: + cve = CVE.objects.get(id=cve_id) + cve.fetch_cve_data() + finally: + cache.delete(cve_id_lock_key) + else: + warning_message(f'Already updating CVE {cve_id}, skipping task.') -@shared_task +@shared_task(priority=2) def update_cves(): """ Task to update all CVEs """ @@ -47,15 +58,26 @@ def update_cves(): warning_message('Already updating CVEs, skipping task.') -@shared_task +@shared_task(priority=3) def update_cwe(cwe_id): """ Task to update a CWE """ - cwe = CWE.objects.get(id=cwe_id) - cwe.fetch_cwe_data() + cwe_id_lock_key = f'update_cwe_id_lock_{cwe_id}' + + # lock will expire after 1 week + lock_expire = 60 * 60 * 168 + + if cache.add(cwe_id_lock_key, 'true', lock_expire): + try: + cwe = CWE.objects.get(id=cwe_id) + cwe.fetch_cwe_data() + finally: + cache.delete(cwe_id_lock_key) + else: + warning_message(f'Already updating CWE {cwe_id}, skipping task.') -@shared_task +@shared_task(priority=2) def update_cwes(): """ Task to update all CWEs """ diff --git a/util/tasks.py b/util/tasks.py index bd76bac6..12825a8c 100644 --- a/util/tasks.py +++ b/util/tasks.py @@ -24,7 +24,7 @@ from repos.utils import clean_repos, remove_mirror_trailing_slashes -@shared_task +@shared_task(priority=1) def clean_database(remove_duplicate_packages=False): """ Task to check the database and remove orphaned objects Runs all clean_* functions to check database consistency From 0c50d4c29508d6361111a2837531505e44778610 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Wed, 7 Jan 2026 20:46:50 -0500 Subject: [PATCH 034/146] update celery services handling (#726) Signed-off-by: Marcus Furlong --- debian/python3-patchman.install | 3 +- debian/python3-patchman.postinst | 29 +++++++++++++++ etc/patchman/celery.conf | 3 ++ .../system/patchman-celery-beat.service | 19 ++++++++++ .../system/patchman-celery-worker.service | 21 +++++++++++ etc/systemd/system/patchman-celery.service | 14 -------- patchman-client.spec | 17 ++++----- scripts/rpm-post-install.sh | 36 +++++++++++++++---- setup.py | 9 ++++- 9 files changed, 121 insertions(+), 30 deletions(-) create mode 100644 etc/systemd/system/patchman-celery-beat.service create mode 100644 etc/systemd/system/patchman-celery-worker.service delete mode 100644 etc/systemd/system/patchman-celery.service diff --git a/debian/python3-patchman.install b/debian/python3-patchman.install index e13b11ca..71f47b3a 100755 --- a/debian/python3-patchman.install +++ b/debian/python3-patchman.install @@ -1,4 +1,5 @@ #!/usr/bin/dh-exec etc/patchman/apache.conf.example => etc/apache2/conf-available/patchman.conf etc/patchman/local_settings.py etc/patchman -etc/systemd/system/patchman-celery.service => lib/systemd/system/patchman-celery.service +etc/systemd/system/patchman-celery-worker.service => lib/systemd/system/patchman-celery-worker@.service +etc/systemd/system/patchman-celery-beat.service => lib/systemd/system/patchman-celery-beat.service diff --git a/debian/python3-patchman.postinst b/debian/python3-patchman.postinst index b64cb816..bce66010 100644 --- a/debian/python3-patchman.postinst +++ b/debian/python3-patchman.postinst @@ -25,8 +25,37 @@ if [ "$1" = "configure" ] ; then chown -R www-data:www-data /var/lib/patchman adduser --system --group patchman-celery usermod -a -G www-data patchman-celery + chown root:patchman-celery /etc/patchman/celery.conf + chmod 640 /etc/patchman/celery.conf chmod g+w /var/lib/patchman /var/lib/patchman/db /var/lib/patchman/db/patchman.db + WORKER_COUNT=1 + if [ -f /etc/patchman/celery.conf ]; then + . /etc/patchman/celery.conf + WORKER_COUNT=${CELERY_WORKER_COUNT:-1} + fi + + if [ -d /run/systemd/system ]; then + systemctl daemon-reload >/dev/null || true + for i in $(seq 1 "${WORKER_COUNT}"); do + deb-systemd-helper enable "patchman-celery-worker@$i.service" >/dev/null || true + deb-systemd-invoke start "patchman-celery-worker@$i.service" >/dev/null || true + done + + active_instances=$(systemctl list-units --type=service --state=active "patchman-celery-worker@*" --no-legend | awk '{print $1}') + + for service in $active_instances; do + inst_num=$(echo "$service" | cut -d'@' -f2 | cut -d'.' -f1) + if [ "$inst_num" -gt "${WORKER_COUNT}" ]; then + deb-systemd-invoke stop "$service" >/dev/null || true + deb-systemd-helper disable "$service" >/dev/null || true + fi + done + + deb-systemd-helper enable "patchman-celery-beat.service" >/dev/null || true + deb-systemd-invoke start "patchman-celery-beat.service" >/dev/null || true + fi + echo echo "Remember to run 'patchman-manage createsuperuser' to create a user." echo diff --git a/etc/patchman/celery.conf b/etc/patchman/celery.conf index 7afc96ee..2e6f9855 100644 --- a/etc/patchman/celery.conf +++ b/etc/patchman/celery.conf @@ -1,2 +1,5 @@ REDIS_HOST=127.0.0.1 REDIS_PORT=6379 +CELERY_POOL_TYPE=solo +CELERY_WORKER_COUNT=1 +CELERY_CONCURRENCY=1 diff --git a/etc/systemd/system/patchman-celery-beat.service b/etc/systemd/system/patchman-celery-beat.service new file mode 100644 index 00000000..b4ba9f3d --- /dev/null +++ b/etc/systemd/system/patchman-celery-beat.service @@ -0,0 +1,19 @@ +[Unit] +Description=Patchman Celery Beat Scheduler Service +Requires=network-online.target +After=network-online.target + +[Service] +Type=simple +User=patchman-celery +Group=patchman-celery +EnvironmentFile=/etc/patchman/celery.conf +ExecStart=/usr/bin/celery \ + --broker redis://${REDIS_HOST:-127.0.0.1}:${REDIS_PORT:-6379}/0 \ + --app patchman \ + beat \ + --loglevel info \ + --scheduler django_celery_beat.schedulers:DatabaseScheduler + +[Install] +WantedBy=multi-user.target diff --git a/etc/systemd/system/patchman-celery-worker.service b/etc/systemd/system/patchman-celery-worker.service new file mode 100644 index 00000000..8807cbff --- /dev/null +++ b/etc/systemd/system/patchman-celery-worker.service @@ -0,0 +1,21 @@ +[Unit] +Description=Patchman Celery Worker Service %i +Requires=network-online.target +After=network-online.target + +[Service] +Type=simple +User=patchman-celery +Group=patchman-celery +EnvironmentFile=/etc/patchman/celery.conf +ExecStart=/usr/bin/celery \ + --broker redis://${REDIS_HOST:-127.0.0.1}:${REDIS_PORT:-6379}/0 \ + --app patchman \ + worker \ + --task-events \ + --pool ${CELERY_POOL_TYPE:-solo} \ + --concurrency ${CELERY_CONCURRENCY:-1} \ + --hostname patchman-celery-worker%i@%%h + +[Install] +WantedBy=multi-user.target diff --git a/etc/systemd/system/patchman-celery.service b/etc/systemd/system/patchman-celery.service deleted file mode 100644 index 6408d818..00000000 --- a/etc/systemd/system/patchman-celery.service +++ /dev/null @@ -1,14 +0,0 @@ -[Unit] -Description=Patchman Celery Service -Requires=network-online.target -After=network-onlne.target - -[Service] -Type=simple -User=patchman-celery -Group=patchman-celery -EnvironmentFile=/etc/patchman/celery.conf -ExecStart=/usr/bin/celery --broker redis://${REDIS_HOST}:${REDIS_PORT}/0 --app patchman worker --loglevel info --beat --scheduler django_celery_beat.schedulers:DatabaseScheduler --task-events --pool threads - -[Install] -WantedBy=multi-user.target diff --git a/patchman-client.spec b/patchman-client.spec index 68736038..f2f8279a 100644 --- a/patchman-client.spec +++ b/patchman-client.spec @@ -10,7 +10,7 @@ Source: %{expand:%%(pwd)} BuildArch: noarch Requires: curl which coreutils util-linux gawk -%define binary_payload w9.gzdio +%define _binary_payload w9.gzdio %description patchman-client provides a client that uploads reports to a patchman server @@ -20,14 +20,15 @@ find . -mindepth 1 -delete cp -af %{SOURCEURL0}/. . %install -mkdir -p %{buildroot}/usr/sbin -mkdir -p %{buildroot}/etc/patchman -cp ./client/%{name} %{buildroot}/usr/sbin -cp ./client/%{name}.conf %{buildroot}/etc/patchman +mkdir -p %{buildroot}%{_sbindir} +mkdir -p %{buildroot}%{_sysconfdir}/patchman +install -m 755 client/%{name} %{buildroot}%{_sbindir}/%{name} +install -m 644 client/%{name}.conf %{buildroot}%{_sysconfdir}/patchman/%{name}.conf %files -%defattr(755,root,root) -/usr/sbin/patchman-client -%config(noreplace) /etc/patchman/patchman-client.conf +%defattr(-,root,root) +%{_sbindir}/patchman-client +%dir %{_sysconfdir}/patchman +%config(noreplace) %{_sysconfdir}/patchman/patchman-client.conf %changelog diff --git a/scripts/rpm-post-install.sh b/scripts/rpm-post-install.sh index 24ade8af..798c43ea 100644 --- a/scripts/rpm-post-install.sh +++ b/scripts/rpm-post-install.sh @@ -4,8 +4,9 @@ if [ ! -e /etc/httpd/conf.d/patchman.conf ] ; then cp /etc/patchman/apache.conf.example /etc/httpd/conf.d/patchman.conf fi -if ! grep /usr/lib/python3.9/site-packages /etc/httpd/conf.d/patchman.conf >/dev/null 2>&1 ; then - sed -i -e "s/^\(Define patchman_pythonpath\).*/\1 \/usr\/lib\/python3.9\/site-packages/" \ +PYTHON_SITEPACKAGES=$(python3 -c "import site; print(site.getsitepackages()[0])") +if ! grep "${PYTHON_SITEPACKAGES}" /etc/httpd/conf.d/patchman.conf >/dev/null 2>&1 ; then + sed -i -e "s|^\(Define patchman_pythonpath\).*|\1 ${PYTHON_SITEPACKAGES}|" \ /etc/httpd/conf.d/patchman.conf fi @@ -24,15 +25,38 @@ patchman-manage makemigrations patchman-manage migrate --run-syncdb --fake-initial sqlite3 /var/lib/patchman/db/patchman.db 'PRAGMA journal_mode=WAL;' -chown -R apache:apache /var/lib/patchman adduser --system --group patchman-celery usermod -a -G apache patchman-celery -chmod g+w /var/lib/patchman /var/lib/patchman/db /var/lib/patchman/db/patchman.db -chcon --type httpd_sys_rw_content_t /var/lib/patchman/db/patchman.db -semanage port -a -t http_port_t -p tcp 5672 +chown root:patchman-celery /etc/patchman/celery.conf +chmod 640 /etc/patchman/celery.conf + +chown -R apache:apache /var/lib/patchman +semanage fcontext -a -t httpd_sys_rw_content_t "/var/lib/patchman/db(/.*)?" +restorecon -Rv /var/lib/patchman/db setsebool -P httpd_can_network_memcache 1 setsebool -P httpd_can_network_connect 1 +WORKER_COUNT=1 +if [ -f /etc/patchman/celery.conf ]; then + . /etc/patchman/celery.conf + WORKER_COUNT=${CELERY_WORKER_COUNT:-1} +fi + +for i in $(seq 1 "${WORKER_COUNT}"); do + systemctl enable --now "patchman-celery-worker@$i.service" +done + +active_instances=$(systemctl list-units --type=service --state=active "patchman-celery-worker@*" --no-legend | awk '{print $1}') +for service in $active_instances; do + inst_num=$(echo "$service" | cut -d'@' -f2 | cut -d'.' -f1) + if [ "$inst_num" -gt "${WORKER_COUNT}" ]; then + systemctl stop "$service" + systemctl disable "$service" + fi +done + +systemctl enable --now patchman-celery-beat.service + echo echo "Remember to run 'patchman-manage createsuperuser' to create a user." echo diff --git a/setup.py b/setup.py index 8e18eaf9..a7dbfc68 100755 --- a/setup.py +++ b/setup.py @@ -17,6 +17,7 @@ # along with Patchman. If not, see import os +import sys from setuptools import find_packages, setup @@ -29,8 +30,14 @@ with open('requirements.txt', 'r', encoding='utf_8') as rt: install_requires = rt.read().splitlines() - data_files = [] +if 'bdist_rpm' in sys.argv: + data_files.append( + ('/usr/lib/systemd/system', [ + 'etc/systemd/system/patchman-celery-worker@.service', + 'etc/systemd/system/patchman-celery-beat.service' + ]) + ) for dirpath, dirnames, filenames in os.walk('etc'): # Ignore dirnames that start with '.' From cbbd43b81d503fbb7b389a196ef53cf045be3214 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Fri, 9 Jan 2026 17:50:10 -0500 Subject: [PATCH 035/146] remove non-present middleware (#729) --- patchman/settings.py | 1 - 1 file changed, 1 deletion(-) diff --git a/patchman/settings.py b/patchman/settings.py index 23e932c4..c3089caa 100644 --- a/patchman/settings.py +++ b/patchman/settings.py @@ -13,7 +13,6 @@ MIDDLEWARE = [ 'django.middleware.security.SecurityMiddleware', 'django.middleware.cache.UpdateCacheMiddleware', - 'patchman.middleware.NeverCacheMiddleware', 'django.middleware.http.ConditionalGetMiddleware', 'django.contrib.sessions.middleware.SessionMiddleware', 'django.middleware.common.CommonMiddleware', From 7f7976f3abcf39ecd3049fa51274abda826d3cd8 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Fri, 9 Jan 2026 17:50:20 -0500 Subject: [PATCH 036/146] fix wsgi so rpm module is only loaded once (#728) --- etc/patchman/apache.conf.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/patchman/apache.conf.example b/etc/patchman/apache.conf.example index 5434cf97..c055eba0 100644 --- a/etc/patchman/apache.conf.example +++ b/etc/patchman/apache.conf.example @@ -1,5 +1,5 @@ Define patchman_pythonpath /srv/patchman/ -WSGIScriptAlias /patchman ${patchman_pythonpath}/patchman/wsgi.py +WSGIScriptAlias /patchman ${patchman_pythonpath}/patchman/wsgi.py application-group=%{GLOBAL} WSGIPythonPath ${patchman_pythonpath} From bbae0b46de037870d76f62017ca874e65a212ba7 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Fri, 9 Jan 2026 17:55:12 -0500 Subject: [PATCH 037/146] give systemd units usable defaults (#727) --- etc/systemd/system/patchman-celery-beat.service | 4 +++- etc/systemd/system/patchman-celery-worker.service | 10 +++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/etc/systemd/system/patchman-celery-beat.service b/etc/systemd/system/patchman-celery-beat.service index b4ba9f3d..e219d38d 100644 --- a/etc/systemd/system/patchman-celery-beat.service +++ b/etc/systemd/system/patchman-celery-beat.service @@ -7,9 +7,11 @@ After=network-online.target Type=simple User=patchman-celery Group=patchman-celery +Environment="REDIS_HOST=127.0.0.1" +Environment="REDIS_PORT=6379" EnvironmentFile=/etc/patchman/celery.conf ExecStart=/usr/bin/celery \ - --broker redis://${REDIS_HOST:-127.0.0.1}:${REDIS_PORT:-6379}/0 \ + --broker redis://${REDIS_HOST}:${REDIS_PORT}/0 \ --app patchman \ beat \ --loglevel info \ diff --git a/etc/systemd/system/patchman-celery-worker.service b/etc/systemd/system/patchman-celery-worker.service index 8807cbff..51fa9a0e 100644 --- a/etc/systemd/system/patchman-celery-worker.service +++ b/etc/systemd/system/patchman-celery-worker.service @@ -7,14 +7,18 @@ After=network-online.target Type=simple User=patchman-celery Group=patchman-celery +Environment="REDIS_HOST=127.0.0.1" +Environment="REDIS_PORT=6379" +Environment="CELERY_POOL_TYPE=solo" +Environment="CELERY_CONCURRENCY=1" EnvironmentFile=/etc/patchman/celery.conf ExecStart=/usr/bin/celery \ - --broker redis://${REDIS_HOST:-127.0.0.1}:${REDIS_PORT:-6379}/0 \ + --broker redis://${REDIS_HOST}:${REDIS_PORT}/0 \ --app patchman \ worker \ --task-events \ - --pool ${CELERY_POOL_TYPE:-solo} \ - --concurrency ${CELERY_CONCURRENCY:-1} \ + --pool ${CELERY_POOL_TYPE} \ + --concurrency ${CELERY_CONCURRENCY} \ --hostname patchman-celery-worker%i@%%h [Install] From f5ec1dc1d870cb0d651d9651365a710252bc3e8e Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Fri, 9 Jan 2026 18:38:05 -0500 Subject: [PATCH 038/146] use consistent users/groups on rhel/debian (#730) --- debian/python3-patchman.postinst | 8 ++++---- etc/systemd/system/patchman-celery-beat.service | 4 ++-- etc/systemd/system/patchman-celery-worker.service | 4 ++-- scripts/rpm-post-install.sh | 10 ++++------ 4 files changed, 12 insertions(+), 14 deletions(-) diff --git a/debian/python3-patchman.postinst b/debian/python3-patchman.postinst index bce66010..b36a5422 100644 --- a/debian/python3-patchman.postinst +++ b/debian/python3-patchman.postinst @@ -22,12 +22,12 @@ if [ "$1" = "configure" ] ; then patchman-manage migrate --run-syncdb --fake-initial sqlite3 /var/lib/patchman/db/patchman.db 'PRAGMA journal_mode=WAL;' - chown -R www-data:www-data /var/lib/patchman - adduser --system --group patchman-celery - usermod -a -G www-data patchman-celery - chown root:patchman-celery /etc/patchman/celery.conf + adduser --quiet --system --group patchman + adduser --quiet www-data patchman + chown root:patchman /etc/patchman/celery.conf chmod 640 /etc/patchman/celery.conf chmod g+w /var/lib/patchman /var/lib/patchman/db /var/lib/patchman/db/patchman.db + chown -R patchman:patchman /var/lib/patchman WORKER_COUNT=1 if [ -f /etc/patchman/celery.conf ]; then diff --git a/etc/systemd/system/patchman-celery-beat.service b/etc/systemd/system/patchman-celery-beat.service index e219d38d..c9bce722 100644 --- a/etc/systemd/system/patchman-celery-beat.service +++ b/etc/systemd/system/patchman-celery-beat.service @@ -5,8 +5,8 @@ After=network-online.target [Service] Type=simple -User=patchman-celery -Group=patchman-celery +User=patchman +Group=patchman Environment="REDIS_HOST=127.0.0.1" Environment="REDIS_PORT=6379" EnvironmentFile=/etc/patchman/celery.conf diff --git a/etc/systemd/system/patchman-celery-worker.service b/etc/systemd/system/patchman-celery-worker.service index 51fa9a0e..b2d6f6b7 100644 --- a/etc/systemd/system/patchman-celery-worker.service +++ b/etc/systemd/system/patchman-celery-worker.service @@ -5,8 +5,8 @@ After=network-online.target [Service] Type=simple -User=patchman-celery -Group=patchman-celery +User=patchman +Group=patchman Environment="REDIS_HOST=127.0.0.1" Environment="REDIS_PORT=6379" Environment="CELERY_POOL_TYPE=solo" diff --git a/scripts/rpm-post-install.sh b/scripts/rpm-post-install.sh index 798c43ea..451f7d48 100644 --- a/scripts/rpm-post-install.sh +++ b/scripts/rpm-post-install.sh @@ -25,15 +25,13 @@ patchman-manage makemigrations patchman-manage migrate --run-syncdb --fake-initial sqlite3 /var/lib/patchman/db/patchman.db 'PRAGMA journal_mode=WAL;' -adduser --system --group patchman-celery -usermod -a -G apache patchman-celery -chown root:patchman-celery /etc/patchman/celery.conf +adduser --system --shell /sbin/nologin patchman +usermod -a -G patchman apache +chown root:patchman /etc/patchman/celery.conf chmod 640 /etc/patchman/celery.conf - -chown -R apache:apache /var/lib/patchman +chown -R patchman:patchman /var/lib/patchman semanage fcontext -a -t httpd_sys_rw_content_t "/var/lib/patchman/db(/.*)?" restorecon -Rv /var/lib/patchman/db -setsebool -P httpd_can_network_memcache 1 setsebool -P httpd_can_network_connect 1 WORKER_COUNT=1 From b7ec0003a2e4af54427b81f11b66cd9b2201a762 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Sat, 10 Jan 2026 15:10:12 -0500 Subject: [PATCH 039/146] fixes for dumping/loading fixtures from sqlite (#731) --- operatingsystems/managers.py | 4 ++-- security/models.py | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/operatingsystems/managers.py b/operatingsystems/managers.py index 630484a1..b7b9f24f 100644 --- a/operatingsystems/managers.py +++ b/operatingsystems/managers.py @@ -18,5 +18,5 @@ class OSReleaseManager(models.Manager): - def get_by_natural_key(self, name, codename): - return self.get(name=name, codename=codename) + def get_by_natural_key(self, name, codename, cpe_name): + return self.get(name=name, codename=codename, cpe_name=cpe_name) diff --git a/security/models.py b/security/models.py index 0f848260..405c8db6 100644 --- a/security/models.py +++ b/security/models.py @@ -125,6 +125,8 @@ def add_cvss_score(self, vector_string, score=None, severity=None, version=None) score = cvss_score.base_score if not severity: severity = cvss_score.severities()[0] + if isinstance(severity, str): + severity = severity.capitalize() try: cvss, created = CVSS.objects.get_or_create( version=version, From e54ce493f514ed501ce4649a52ec3ab91fbb21b3 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Sat, 10 Jan 2026 16:54:39 -0500 Subject: [PATCH 040/146] update logging to log to console and celery systemd units (#732) --- etc/patchman/local_settings.py | 17 +++++++++ patchman/receivers.py | 25 ++++++++------ reports/utils.py | 4 ++- repos/repo_types/arch.py | 2 +- repos/repo_types/deb.py | 2 +- repos/repo_types/gentoo.py | 4 +-- repos/repo_types/rpm.py | 2 +- repos/utils.py | 2 +- sbin/patchman | 6 ++-- util/__init__.py | 42 +++-------------------- util/logging.py | 63 +++++++++++++++++++++++++++++----- 11 files changed, 103 insertions(+), 66 deletions(-) diff --git a/etc/patchman/local_settings.py b/etc/patchman/local_settings.py index 15fcb60a..cb19e268 100644 --- a/etc/patchman/local_settings.py +++ b/etc/patchman/local_settings.py @@ -96,3 +96,20 @@ 'schedule': timedelta(hours=24), }, } + +LOGGING = { + 'version': 1, + 'disable_existing_loggers': False, + 'handlers': { + 'console': { + 'class': 'logging.StreamHandler', + }, + }, + 'root': { + 'handlers': ['console'], + }, + 'loggers': { + 'urllib3': {'level': 'WARNING', 'handlers': ['console'], 'propagate': False}, + 'git': {'level': 'WARNING', 'handlers': ['console'], 'propagate': False}, + } +} diff --git a/patchman/receivers.py b/patchman/receivers.py index 19312ed5..9393d891 100644 --- a/patchman/receivers.py +++ b/patchman/receivers.py @@ -18,13 +18,13 @@ from colorama import Fore, Style, init from django.conf import settings from django.dispatch import receiver -from tqdm import tqdm +from tqdm.contrib.logging import logging_redirect_tqdm from patchman.signals import ( debug_message_s, error_message_s, info_message_s, pbar_start, pbar_update, warning_message_s, ) -from util import create_pbar, get_verbosity, update_pbar +from util.logging import create_pbar, get_quiet_mode, logger, update_pbar init(autoreset=True) @@ -53,8 +53,10 @@ def print_info_message(**kwargs): """ Receiver to handle an info message, no color """ text = str(kwargs.get('text')) - if get_verbosity(): - tqdm.write(Style.RESET_ALL + Fore.RESET + text) + if not get_quiet_mode(): + with logging_redirect_tqdm(loggers=[logger]): + for line in text.splitlines(): + logger.info(Style.RESET_ALL + Fore.RESET + line) @receiver(warning_message_s) @@ -62,8 +64,9 @@ def print_warning_message(**kwargs): """ Receiver to handle a warning message, yellow text """ text = str(kwargs.get('text')) - if get_verbosity(): - tqdm.write(Style.BRIGHT + Fore.YELLOW + text) + if not get_quiet_mode(): + with logging_redirect_tqdm(): + logger.warning(Style.BRIGHT + Fore.YELLOW + text) @receiver(error_message_s) @@ -72,13 +75,15 @@ def print_error_message(**kwargs): """ text = str(kwargs.get('text')) if text: - tqdm.write(Style.BRIGHT + Fore.RED + text) + with logging_redirect_tqdm(): + logger.error(Style.BRIGHT + Fore.RED + text) @receiver(debug_message_s) def print_debug_message(**kwargs): - """ Receiver to handle a debug message, blue text if verbose and DEBUG are set + """ Receiver to handle a debug message, blue text if DEBUG is set """ text = str(kwargs.get('text')) - if get_verbosity() and settings.DEBUG and text: - tqdm.write(Style.BRIGHT + Fore.BLUE + text) + if settings.DEBUG and text: + with logging_redirect_tqdm(loggers=[logger]): + logger.debug(Style.BRIGHT + Fore.BLUE + text) diff --git a/reports/utils.py b/reports/utils.py index 8b12f046..1a08cf3c 100644 --- a/reports/utils.py +++ b/reports/utils.py @@ -34,7 +34,7 @@ from patchman.signals import pbar_start, pbar_update from repos.models import Mirror, MirrorPackage, Repository from repos.utils import get_or_create_repo -from util.logging import info_message +from util.logging import debug_message, info_message def process_repos(report, host): @@ -47,6 +47,7 @@ def process_repos(report, host): pbar_start.send(sender=None, ptext=f'{host} Repos', plen=len(repos)) for i, repo_str in enumerate(repos): + debug_message(f'Processing report {report.id} repo: {repo_str}') repo, priority = process_repo(repo_str, report.arch) if repo: repo_ids.append(repo.id) @@ -93,6 +94,7 @@ def process_packages(report, host): packages = parse_packages(report.packages) pbar_start.send(sender=None, ptext=f'{host} Packages', plen=len(packages)) for i, pkg_str in enumerate(packages): + debug_message(f'Processing report {report.id} package: {pkg_str}') package = process_package(pkg_str, report.protocol) if package: package_ids.append(package.id) diff --git a/repos/repo_types/arch.py b/repos/repo_types/arch.py index 390b321d..b339ee6a 100644 --- a/repos/repo_types/arch.py +++ b/repos/repo_types/arch.py @@ -51,7 +51,7 @@ def refresh_arch_repo(repo): package_data = fetch_mirror_data( mirror=mirror, url=mirror_url, - text='Fetching Repo data') + text='Fetching Arch Repo data') if not package_data: continue diff --git a/repos/repo_types/deb.py b/repos/repo_types/deb.py index 33d1f2c4..eea6593f 100644 --- a/repos/repo_types/deb.py +++ b/repos/repo_types/deb.py @@ -95,7 +95,7 @@ def refresh_deb_repo(repo): package_data = fetch_mirror_data( mirror=mirror, url=mirror_url, - text='Fetching Repo data') + text='Fetching Debian Repo data') if not package_data: continue diff --git a/repos/repo_types/gentoo.py b/repos/repo_types/gentoo.py index a0584618..61966f7a 100644 --- a/repos/repo_types/gentoo.py +++ b/repos/repo_types/gentoo.py @@ -51,7 +51,7 @@ def refresh_gentoo_main_repo(repo): continue res = get_url(mirror.url + '.md5sum') - data = fetch_content(res, 'Fetching Repo checksum') + data = fetch_content(res, 'Fetching Gentoo Repo checksum') if data is None: mirror.fail() continue @@ -72,7 +72,7 @@ def refresh_gentoo_main_repo(repo): mirror.fail() continue - data = fetch_content(res, 'Fetching Repo data') + data = fetch_content(res, 'Fetching Gentoo Repo data') if data is None: mirror.fail() continue diff --git a/repos/repo_types/rpm.py b/repos/repo_types/rpm.py index 5ffbb708..d1482272 100644 --- a/repos/repo_types/rpm.py +++ b/repos/repo_types/rpm.py @@ -84,7 +84,7 @@ def refresh_rpm_repo_mirrors(repo, errata_only=False): repo_data = fetch_mirror_data( mirror=mirror, url=mirror_url, - text='Fetching Repo data') + text='Fetching rpm Repo data') if not repo_data: continue diff --git a/repos/utils.py b/repos/utils.py index 29e9cdb3..13cee149 100644 --- a/repos/utils.py +++ b/repos/utils.py @@ -155,7 +155,7 @@ def get_mirrorlist_urls(url): return if response_is_valid(res): try: - data = fetch_content(res, 'Fetching Repo data') + data = fetch_content(res, 'Fetching Repo data to check for mirrorlist') if data is None: return mirror_urls = re.findall(r'^http[s]*://.*$|^ftp://.*$', data.decode('utf-8'), re.MULTILINE) diff --git a/sbin/patchman b/sbin/patchman index b76272a2..06bef981 100755 --- a/sbin/patchman +++ b/sbin/patchman @@ -47,8 +47,8 @@ from reports.tasks import remove_reports_with_no_hosts from repos.models import Repository from repos.utils import clean_repos from security.utils import update_cves, update_cwes -from util import get_datetime_now, set_verbosity -from util.logging import info_message +from util import get_datetime_now +from util.logging import info_message, set_quiet_mode def get_host(host=None, action='Performing action'): @@ -547,7 +547,7 @@ def main(): parser = collect_args() args = parser.parse_args() - set_verbosity(not args.quiet) + set_quiet_mode(args.quiet) showhelp = process_args(args) if showhelp: parser.print_help() diff --git a/util/__init__.py b/util/__init__.py index c6c9aa0d..b85e5e37 100644 --- a/util/__init__.py +++ b/util/__init__.py @@ -41,12 +41,14 @@ from tenacity import ( retry, retry_if_exception_type, stop_after_attempt, wait_exponential, ) -from tqdm import tqdm -from util.logging import debug_message, error_message, info_message +from util.logging import ( + create_pbar, debug_message, error_message, info_message, quiet_mode, + update_pbar, +) pbar = None -verbose = None +verbose = not quiet_mode Checksum = Enum('Checksum', 'md5 sha sha1 sha256 sha512') http_proxy = os.getenv('http_proxy') @@ -57,40 +59,6 @@ } -def get_verbosity(): - """ Get the global verbosity level - """ - return verbose - - -def set_verbosity(value): - """ Set the global verbosity level - """ - global verbose - verbose = value - - -def create_pbar(ptext, plength, ljust=35, **kwargs): - """ Create a global progress bar if global verbose is True - """ - global pbar - if verbose and plength > 0: - jtext = str(ptext).ljust(ljust) - pbar = tqdm(total=plength, desc=jtext, position=0, leave=True, ascii=' >=') - return pbar - - -def update_pbar(index, **kwargs): - """ Update the global progress bar if global verbose is True - """ - global pbar - if verbose and pbar: - pbar.update(n=index-pbar.n) - if index >= pbar.total: - pbar.close() - pbar = None - - def fetch_content(response, text='', ljust=35): """ Display a progress bar to fetch the request content if verbose is True. Otherwise, just return the request content diff --git a/util/logging.py b/util/logging.py index cb00ccce..bf532e54 100644 --- a/util/logging.py +++ b/util/logging.py @@ -15,28 +15,73 @@ # along with Patchman. If not, see -from datetime import datetime +import logging + +from django.conf import settings +from tqdm import tqdm from patchman.signals import ( debug_message_s, error_message_s, info_message_s, warning_message_s, ) +log_format = '[%(asctime)s] %(levelname)s: %(message)s' +if settings.DEBUG: + logging_level = logging.DEBUG +else: + logging_level = logging.INFO +logging.basicConfig(level=logging_level, format=log_format) +logger = logging.getLogger() +logging.getLogger('git.cmd').setLevel(logging.WARNING) + +quiet_mode = False +pbar = None + + +def get_quiet_mode(): + """ Get the global quiet_mode + """ + return quiet_mode + + +def set_quiet_mode(value): + """ Set the global quiet_mode + """ + global quiet_mode + quiet_mode = value + + +def create_pbar(ptext, plength, ljust=35, **kwargs): + """ Create a global progress bar if global quiet_mode is False + """ + global pbar + if not quiet_mode and plength > 0: + jtext = str(ptext).ljust(ljust) + pbar = tqdm(total=plength, desc=jtext, position=0, leave=True, ascii=' >=') + return pbar + + +def update_pbar(index, **kwargs): + """ Update the global progress bar if global quiet_mode is False + """ + global pbar + if not quiet_mode and pbar: + pbar.update(n=index-pbar.n) + if index >= pbar.total: + pbar.close() + pbar = None + def info_message(text): - ts = datetime.now() - info_message_s.send(sender=None, text=text, ts=ts) + info_message_s.send(sender=None, text=text) def warning_message(text): - ts = datetime.now() - warning_message_s.send(sender=None, text=text, ts=ts) + warning_message_s.send(sender=None, text=text) def debug_message(text): - ts = datetime.now() - debug_message_s.send(sender=None, text=text, ts=ts) + debug_message_s.send(sender=None, text=text) def error_message(text): - ts = datetime.now() - error_message_s.send(sender=None, text=text, ts=ts) + error_message_s.send(sender=None, text=text) From 91f35e1917b4bf92eb002c8af919cc8c001330bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Tue, 20 Jan 2026 13:08:57 +0000 Subject: [PATCH 041/146] Replaced exit 1 with fatal() function --- email/patchman-email | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/email/patchman-email b/email/patchman-email index dafc6328..aef9e62c 100755 --- a/email/patchman-email +++ b/email/patchman-email @@ -187,6 +187,9 @@ usage() { echo "-h: Shows this help message and exits" } +proc=$$ +trap 'exit 1' SIGUSR1 + # Checks if there are any parameters and if they start with "-" if [ $# -gt 0 ] && [[ $1 == -* ]]; then CONF="/etc/patchman/patchman-email.conf" @@ -203,8 +206,7 @@ if [ $# -gt 0 ] && [[ $1 == -* ]]; then case $arg in a) if [ ! -f $CONF ]; then - echo "$(date '+%d/%m/%Y %R') [ERROR] Configuration file not found." - exit 1 + fatal "$(date '+%d/%m/%Y %R') [ERROR] Configuration file not found." else DIR=$(getConfig General dir) DATE=$(date +%Y%m%d) @@ -226,8 +228,7 @@ if [ $# -gt 0 ] && [[ $1 == -* ]]; then ;; H) if [ ! -f $CONF ]; then - echo "$(date '+%d/%m/%Y %R') [ERROR] Configuration file not found." - exit 1 + fatal "$(date '+%d/%m/%Y %R') [ERROR] Configuration file not found." else DIR=$(getConfig General dir) DATE=$(date +%Y%m%d) @@ -239,8 +240,7 @@ if [ $# -gt 0 ] && [[ $1 == -* ]]; then ;; T) if [ ! -f $CONF ]; then - echo "$(date '+%d/%m/%Y %R') [ERROR] Configuration file not found." - exit 1 + fatal "$(date '+%d/%m/%Y %R') [ERROR] Configuration file not found." else DIR=$(getConfig General dir) DATE=$(date +%Y%m%d) @@ -254,8 +254,7 @@ if [ $# -gt 0 ] && [[ $1 == -* ]]; then email "$TAG" else - echo "$(date '+%d/%m/%Y %R') [ERROR] ${OPTARG}: Tag not found." - exit 1 + fatal "$(date '+%d/%m/%Y %R') [ERROR] ${OPTARG}: Tag not found." fi fi ;; From 9af7e207aa447ef225d376de45c87f1505d33e9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Tue, 20 Jan 2026 13:11:53 +0000 Subject: [PATCH 042/146] Implemented dependency check --- email/patchman-email | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/email/patchman-email b/email/patchman-email index aef9e62c..0b7837f0 100755 --- a/email/patchman-email +++ b/email/patchman-email @@ -1,5 +1,15 @@ #!/bin/bash +checkDependencies() { + dependencies=("mysql" "patchman" "sendmail" "uuencode" "uuidgen" "weasyprint") + + for dep in "${dependencies[@]}"; do + if [ ! "$(command -v "$dep")" ]; then + fatal "[ERROR] Command $dep not installed." + fi + done +} + email() { SENDER=$(getConfig General sender) @@ -190,6 +200,8 @@ usage() { proc=$$ trap 'exit 1' SIGUSR1 +checkDependencies + # Checks if there are any parameters and if they start with "-" if [ $# -gt 0 ] && [[ $1 == -* ]]; then CONF="/etc/patchman/patchman-email.conf" From 9e8970a950c94483f52409673081af268ba7ab7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Tue, 20 Jan 2026 13:56:13 +0000 Subject: [PATCH 043/146] Changed all internal variables to lowercase --- email/patchman-email | 148 +++++++++++++++++++++---------------------- 1 file changed, 74 insertions(+), 74 deletions(-) diff --git a/email/patchman-email b/email/patchman-email index 0b7837f0..e426f31a 100755 --- a/email/patchman-email +++ b/email/patchman-email @@ -11,23 +11,23 @@ checkDependencies() { } email() { - SENDER=$(getConfig General sender) + sender=$(getConfig General sender) - if [ -n "$SENDER" ]; then - RECIPIENT=$(getConfig "$1" recipient) + if [ -n "$sender" ]; then + recipient=$(getConfig "$1" recipient) - if [ -n "$RECIPIENT" ]; then - ATTACHMENTS=() - BOUNDARY=$(uuidgen) - CC=$(getConfig "$1" cc) - BCC=$(getConfig "$1" bcc) - FULLNAME=$(getConfig General fullname) - SUBJECT=$(echo "[$1] $(getConfig General subject)" | base64) + if [ -n "$recipient" ]; then + attachments=() + boundary=$(uuidgen) + cc=$(getConfig "$1" cc) + bcc=$(getConfig "$1" bcc) + fullname=$(getConfig General fullname) + subject=$(echo "[$1] $(getConfig General subject)" | base64) if [ -s "$custom_html" ]; then - BODY=$(cat $custom_html) + body=$(cat $custom_html) else - BODY="
- $(echo "$HOST" | awk -F ' : ' 'NR==1 {print ""; next} NR==2 || NR==4 || NR==5 || NR==6 || NR==7 || NR==9 || NR==11 { print "" }') + $(echo "$host" | awk -F ' : ' 'NR==1 {print ""; next} NR==2 || NR==4 || NR==5 || NR==6 || NR==7 || NR==9 || NR==11 { print "" }')
"$1"
"$1": "$2"
"$1"
"$1": "$2"

- $(echo "$QUERY" | sort -n | uniq | sed 's/[()]//g' | awk '{ print "" }') + $(echo "$query" | sort -n | uniq | sed 's/[()]//g' | awk '{ print "" }')
Current VersionNew VersionType
"$1""$3""$4"
"$1""$3""$4"

EOF - if weasyprint "$FILE.html" "$FILE.pdf"; then - rm "$FILE.html" + if weasyprint "$file.html" "$file.pdf"; then + rm "$file.html" fi else - echo "$(date '+%d/%m/%Y %R') [INFO] $TAG $1: Report exists, skipping." + echo "$(date '+%d/%m/%Y %R') [INFO] $tag $1: Report exists, skipping." fi else - echo "$(date '+%d/%m/%Y %R') [INFO] $TAG $1: There are no updates available." + echo "$(date '+%d/%m/%Y %R') [INFO] $tag $1: There are no updates available." fi } @@ -204,67 +204,67 @@ checkDependencies # Checks if there are any parameters and if they start with "-" if [ $# -gt 0 ] && [[ $1 == -* ]]; then - CONF="/etc/patchman/patchman-email.conf" + conf="/etc/patchman/patchman-email.conf" custom_html="/etc/patchman/patchman-email.html" # Database credentials configured in /etc/patchman/local_settings.py mysql_db=$(grep -E "'NAME'" /etc/patchman/local_settings.py | grep -v "^#" | sed -e "s/'\|,//g" | awk -F " " '{print $2}') mysql_user=$(grep -E "'USER'" /etc/patchman/local_settings.py | grep -v "^#" | sed -e "s/'\|,//g" | awk -F " " '{print $2}') mysql_pass=$(grep -E "'PASSWORD'" /etc/patchman/local_settings.py | grep -v "^#" | sed -e "s/'\|,//g" | awk -F " " '{print $2}') - mysql_host=$(grep -E "'HOST'" /etc/patchman/local_settings.py | grep -v "^#" | sed -e "s/'\|,//g" | awk -F " " '{print $2}') + mysql_host=$(grep -E "'host'" /etc/patchman/local_settings.py | grep -v "^#" | sed -e "s/'\|,//g" | awk -F " " '{print $2}') mysql_port=$(grep -E "'PORT'" /etc/patchman/local_settings.py | grep -v "^#" | sed -e "s/'\|,//g" | awk -F " " '{print $2}') while getopts 'aH:T:h' arg; do case $arg in a) - if [ ! -f $CONF ]; then + if [ ! -f $conf ]; then fatal "$(date '+%d/%m/%Y %R') [ERROR] Configuration file not found." else - DIR=$(getConfig General dir) - DATE=$(date +%Y%m%d) - HOSTS=() - TAGS=() - mapfile -t HOSTS < <(patchman -lh | tail -n +3 | awk -F ' : ' 'NR%16==1 { print $1 }' | tr -d ":" | sed 's/\x1b\[[0-9;]*m//g') - mapfile -t TAGS < <(patchman -lh | tail -n +3 | awk -F ' : ' 'NR%16==12 { print $2 }' | sort -n | uniq | sed 's/\x1b\[[0-9;]*m//g') + dir=$(getConfig General dir) + date=$(date +%Y%m%d) + hosts=() + tags=() + mapfile -t hosts < <(patchman -lh | tail -n +3 | awk -F ' : ' 'NR%16==1 { print $1 }' | tr -d ":" | sed 's/\x1b\[[0-9;]*m//g') + mapfile -t tags < <(patchman -lh | tail -n +3 | awk -F ' : ' 'NR%16==12 { print $2 }' | sort -n | uniq | sed 's/\x1b\[[0-9;]*m//g') # Generate report - for HOST in "${HOSTS[@]}"; do - report "$HOST" + for host in "${hosts[@]}"; do + report "$host" done # E-mails the report - for TAG in "${TAGS[@]}"; do - email "$TAG" + for tag in "${tags[@]}"; do + email "$tag" done fi ;; H) - if [ ! -f $CONF ]; then + if [ ! -f $conf ]; then fatal "$(date '+%d/%m/%Y %R') [ERROR] Configuration file not found." else - DIR=$(getConfig General dir) - DATE=$(date +%Y%m%d) - HOST=${OPTARG} + dir=$(getConfig General dir) + date=$(date +%Y%m%d) + host=${OPTARG} - report "$HOST" - email "$TAG" "$HOST" + report "$host" + email "$tag" "$host" fi ;; T) - if [ ! -f $CONF ]; then + if [ ! -f $conf ]; then fatal "$(date '+%d/%m/%Y %R') [ERROR] Configuration file not found." else - DIR=$(getConfig General dir) - DATE=$(date +%Y%m%d) - HOSTS=() - mapfile -t HOSTS < <(patchman -lh | tail -n +3 | awk -F ' : ' 'NR%16==1 {print $1; next} NR%16==12 {print $2}' | tr -d ":" | paste -d : - - | awk -F ':' '{ if( $2 == "'"${OPTARG}"'" ){ print $1 }}' | sed 's/\x1b\[[0-9;]*m//g') + dir=$(getConfig General dir) + date=$(date +%Y%m%d) + hosts=() + mapfile -t hosts < <(patchman -lh | tail -n +3 | awk -F ' : ' 'NR%16==1 {print $1; next} NR%16==12 {print $2}' | tr -d ":" | paste -d : - - | awk -F ':' '{ if( $2 == "'"${OPTARG}"'" ){ print $1 }}' | sed 's/\x1b\[[0-9;]*m//g') - if [ "${#HOSTS[@]}" -gt 0 ]; then - for HOST in "${HOSTS[@]}"; do - report "$HOST" + if [ "${#hosts[@]}" -gt 0 ]; then + for host in "${hosts[@]}"; do + report "$host" done - email "$TAG" + email "$tag" else fatal "$(date '+%d/%m/%Y %R') [ERROR] ${OPTARG}: Tag not found." fi From 191db74f0f6dc182becb9c819ebabd196cc8d690 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Mon, 12 Jan 2026 00:03:24 -0500 Subject: [PATCH 044/146] switch to django-tables2 --- debian/control | 3 +- errata/tables.py | 111 ++++++++ errata/templates/errata/erratum_table.html | 31 --- errata/views.py | 18 +- hosts/tables.py | 110 ++++++++ hosts/templates/hosts/host_list.html | 32 ++- hosts/templates/hosts/host_table.html | 28 -- hosts/templatetags/report_alert.py | 4 +- hosts/urls.py | 1 + hosts/views.py | 121 +++++++- modules/tables.py | 72 +++++ modules/templates/modules/module_table.html | 27 -- modules/views.py | 16 +- operatingsystems/tables.py | 167 +++++++++++ .../operatingsystemrelease_table.html | 27 -- .../operatingsystemvariant_table.html | 25 -- .../operatingsystems/osrelease_list.html | 21 +- .../operatingsystems/osvariant_list.html | 31 ++- operatingsystems/urls.py | 2 + operatingsystems/views.py | 177 ++++++++++-- packages/tables.py | 120 ++++++++ .../packages/package_name_table.html | 17 -- .../templates/packages/package_table.html | 30 -- packages/views.py | 36 +-- patchman/settings.py | 4 + patchman/sqlite3/base.py | 14 + patchman/static/css/base.css | 16 ++ patchman/static/img/icon-alert.gif | Bin 145 -> 0 bytes patchman/static/img/icon-no.gif | Bin 176 -> 0 bytes patchman/static/img/icon-yes.gif | Bin 299 -> 0 bytes patchman/static/js/ajax-jquery.js | 29 -- patchman/static/js/button-post.js | 23 -- patchman/urls.py | 2 +- reports/tables.py | 60 ++++ reports/templates/reports/report_detail.html | 4 +- reports/templates/reports/report_list.html | 32 ++- reports/templates/reports/report_table.html | 23 -- reports/urls.py | 1 + reports/views.py | 101 ++++++- repos/tables.py | 165 +++++++++++ repos/templates/repos/mirror_edit_repo.html | 33 --- repos/templates/repos/mirror_list.html | 21 +- repos/templates/repos/mirror_table.html | 39 --- .../repos/mirror_with_repo_list.html | 4 +- repos/templates/repos/repo_list.html | 32 ++- repos/templates/repos/repository_table.html | 25 -- repos/templatetags/repo_buttons.py | 54 ---- repos/urls.py | 2 + repos/views.py | 262 +++++++++++++++--- requirements.txt | 1 + security/tables.py | 183 ++++++++++++ security/templates/security/cve_table.html | 37 --- security/templates/security/cwe_table.html | 21 -- .../templates/security/reference_table.html | 19 -- security/views.py | 44 +-- setup.cfg | 1 + util/__init__.py | 9 + util/context_processors.py | 105 +++++++ util/tables.py | 26 ++ util/templates/base.html | 2 - util/templates/bulk_actions.html | 90 ++++++ util/templates/dashboard.html | 2 +- util/templates/navbar.html | 47 ++-- util/templates/objectlist.html | 16 +- util/templates/table.html | 118 ++++++++ util/templatetags/common.py | 84 +++--- util/urls.py | 2 +- 67 files changed, 2240 insertions(+), 740 deletions(-) create mode 100644 errata/tables.py delete mode 100644 errata/templates/errata/erratum_table.html create mode 100644 hosts/tables.py delete mode 100644 hosts/templates/hosts/host_table.html create mode 100644 modules/tables.py delete mode 100644 modules/templates/modules/module_table.html create mode 100644 operatingsystems/tables.py delete mode 100644 operatingsystems/templates/operatingsystems/operatingsystemrelease_table.html delete mode 100644 operatingsystems/templates/operatingsystems/operatingsystemvariant_table.html create mode 100644 packages/tables.py delete mode 100644 packages/templates/packages/package_name_table.html delete mode 100644 packages/templates/packages/package_table.html delete mode 100644 patchman/static/img/icon-alert.gif delete mode 100644 patchman/static/img/icon-no.gif delete mode 100644 patchman/static/img/icon-yes.gif delete mode 100644 patchman/static/js/ajax-jquery.js delete mode 100644 patchman/static/js/button-post.js create mode 100644 reports/tables.py delete mode 100644 reports/templates/reports/report_table.html create mode 100644 repos/tables.py delete mode 100644 repos/templates/repos/mirror_edit_repo.html delete mode 100644 repos/templates/repos/mirror_table.html delete mode 100644 repos/templates/repos/repository_table.html delete mode 100644 repos/templatetags/repo_buttons.py create mode 100644 security/tables.py delete mode 100644 security/templates/security/cve_table.html delete mode 100644 security/templates/security/cwe_table.html delete mode 100644 security/templates/security/reference_table.html create mode 100644 util/context_processors.py create mode 100644 util/tables.py create mode 100644 util/templates/bulk_actions.html create mode 100644 util/templates/table.html diff --git a/debian/control b/debian/control index 7bfe320e..cd19139c 100644 --- a/debian/control +++ b/debian/control @@ -20,7 +20,8 @@ Depends: ${misc:Depends}, python3 (>= 3.11), python3-django (>= 4.2), python3-requests, python3-colorama, python3-magic, python3-humanize, python3-yaml, libapache2-mod-wsgi-py3, apache2, sqlite3, celery, python3-celery, python3-django-celery-beat, redis-server, - python3-redis, python3-git, python3-django-taggit, python3-zstandard + python3-redis, python3-git, python3-django-taggit, python3-zstandard, + python3-django-tables2 Suggests: python3-mysqldb, python3-psycopg2, python3-pymemcache, memcached Description: Django-based patch status monitoring tool for linux systems. . diff --git a/errata/tables.py b/errata/tables.py new file mode 100644 index 00000000..c23d25be --- /dev/null +++ b/errata/tables.py @@ -0,0 +1,111 @@ +# This file is part of Patchman. +# +# Patchman is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, version 3 only. +# +# Patchman is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Patchman. If not, see + +import django_tables2 as tables + +from errata.models import Erratum +from util.tables import BaseTable + +ERRATUM_NAME_TEMPLATE = '{{ record.name }}' +PACKAGES_AFFECTED_TEMPLATE = ( + '{% with count=record.affected_packages.count %}' + '{% if count != 0 %}' + '{{ count }}' + '{% else %}{% endif %}{% endwith %}' +) +PACKAGES_FIXED_TEMPLATE = ( + '{% with count=record.fixed_packages.count %}' + '{% if count != 0 %}' + '{{ count }}' + '{% else %}{% endif %}{% endwith %}' +) +OSRELEASES_TEMPLATE = ( + '{% with count=record.osreleases.count %}' + '{% if count != 0 %}' + '{{ count }}' + '{% else %}{% endif %}{% endwith %}' +) +ERRATUM_CVES_TEMPLATE = ( + '{% with count=record.cves.count %}' + '{% if count != 0 %}' + '{{ count }}' + '{% else %}{% endif %}{% endwith %}' +) +REFERENCES_TEMPLATE = ( + '{% with count=record.references.count %}' + '{% if count != 0 %}' + '{{ count }}' + '{% else %}{% endif %}{% endwith %}' +) + + +class ErratumTable(BaseTable): + erratum_name = tables.TemplateColumn( + ERRATUM_NAME_TEMPLATE, + order_by='name', + verbose_name='ID', + attrs={'th': {'class': 'col-sm-2'}, 'td': {'class': 'col-sm-2'}}, + ) + e_type = tables.Column( + order_by='e_type', + verbose_name='Type', + attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1'}}, + ) + issue_date = tables.DateColumn( + order_by='issue_date', + verbose_name='Published Date', + attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1'}}, + ) + synopsis = tables.Column( + orderable=False, + verbose_name='Synopsis', + attrs={'th': {'class': 'col-sm-4'}, 'td': {'class': 'col-sm-4'}}, + ) + packages_affected = tables.TemplateColumn( + PACKAGES_AFFECTED_TEMPLATE, + orderable=False, + verbose_name='Packages Affected', + attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1'}}, + ) + packages_fixed = tables.TemplateColumn( + PACKAGES_FIXED_TEMPLATE, + orderable=False, + verbose_name='Packages Fixed', + attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1'}}, + ) + osreleases = tables.TemplateColumn( + OSRELEASES_TEMPLATE, + orderable=False, + verbose_name='OS Releases Affected', + attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1'}}, + ) + erratum_cves = tables.TemplateColumn( + ERRATUM_CVES_TEMPLATE, + orderable=False, + verbose_name='CVEs', + attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1'}}, + ) + references = tables.TemplateColumn( + REFERENCES_TEMPLATE, + orderable=False, + verbose_name='References', + attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1'}}, + ) + + class Meta(BaseTable.Meta): + model = Erratum + fields = ( + 'erratum_name', 'e_type', 'issue_date', 'synopsis', 'packages_affected', + 'packages_fixed', 'osreleases', 'erratum_cves', 'references', + ) diff --git a/errata/templates/errata/erratum_table.html b/errata/templates/errata/erratum_table.html deleted file mode 100644 index c319cbb5..00000000 --- a/errata/templates/errata/erratum_table.html +++ /dev/null @@ -1,31 +0,0 @@ -{% load common %} - - - - - - - - - - - - - - - - {% for erratum in object_list %} - - - - - - - - - - - - {% endfor %} - -
IDTypePublished DateSynopsisPackages AffectedPackages FixedOS Releases AffectedCVEsReferences
{{ erratum.name }}{{ erratum.e_type }}{{ erratum.issue_date|date|default_if_none:'' }}{{ erratum.synopsis }}{% with count=erratum.affected_packages.count %}{% if count != 0 %}{{ count }}{% else %} {% endif %}{% endwith %}{% with count=erratum.fixed_packages.count %}{% if count != 0 %}{{ count }}{% else %} {% endif %}{% endwith %}{% with count=erratum.osreleases.count %}{% if count != 0 %}{{ count }}{% else %} {% endif %}{% endwith %}{% with count=erratum.cves.count %}{% if count != 0 %}{{ count }}{% else %} {% endif %}{% endwith %}{% with count=erratum.references.count %}{% if count != 0 %}{{ count }}{% else %} {% endif %}{% endwith %}
diff --git a/errata/views.py b/errata/views.py index 8e1c0b2f..285f7483 100644 --- a/errata/views.py +++ b/errata/views.py @@ -15,13 +15,14 @@ # along with Patchman. If not, see from django.contrib.auth.decorators import login_required -from django.core.paginator import EmptyPage, PageNotAnInteger, Paginator from django.db.models import Q from django.shortcuts import get_object_or_404, render +from django_tables2 import RequestConfig from rest_framework import viewsets from errata.models import Erratum from errata.serializers import ErratumSerializer +from errata.tables import ErratumTable from operatingsystems.models import OSRelease from util.filterspecs import Filter, FilterBar @@ -61,16 +62,6 @@ def erratum_list(request): else: terms = '' - page_no = request.GET.get('page') - paginator = Paginator(errata, 50) - - try: - page = paginator.page(page_no) - except PageNotAnInteger: - page = paginator.page(1) - except EmptyPage: - page = paginator.page(paginator.num_pages) - filter_list = [] filter_list.append(Filter(request, 'Erratum Type', 'e_type', Erratum.objects.values_list('e_type', flat=True).distinct())) @@ -78,9 +69,12 @@ def erratum_list(request): OSRelease.objects.filter(erratum__in=errata))) filter_bar = FilterBar(request, filter_list) + table = ErratumTable(errata) + RequestConfig(request, paginate={'per_page': 50}).configure(table) + return render(request, 'errata/erratum_list.html', - {'page': page, + {'table': table, 'filter_bar': filter_bar, 'terms': terms}) diff --git a/hosts/tables.py b/hosts/tables.py new file mode 100644 index 00000000..835d8195 --- /dev/null +++ b/hosts/tables.py @@ -0,0 +1,110 @@ +# This file is part of Patchman. +# +# Patchman is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, version 3 only. +# +# Patchman is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Patchman. If not, see + +import django_tables2 as tables + +from hosts.models import Host +from util.tables import BaseTable + +CHECKBOX_TEMPLATE = '' +SELECT_ALL_CHECKBOX = '' +HOSTNAME_TEMPLATE = '{{ record.hostname }}' +SEC_UPDATES_TEMPLATE = ( + '{% with count=record.get_num_security_updates %}' + '{% if count != 0 %}{{ count }}{% else %}{% endif %}' + '{% endwith %}' +) +BUG_UPDATES_TEMPLATE = ( + '{% with count=record.get_num_bugfix_updates %}' + '{% if count != 0 %}{{ count }}{% else %}{% endif %}' + '{% endwith %}' +) +AFFECTED_ERRATA_TEMPLATE = ( + '{% with count=record.errata.count %}' + '{% if count != 0 %}' + '{{ count }}' + '{% else %}{% endif %}{% endwith %}' +) +OSVARIANT_TEMPLATE = ( + '{% if record.osvariant %}' + '{{ record.osvariant }}' + '{% endif %}' +) +LASTREPORT_TEMPLATE = ( + '{% load report_alert %}' + '{{ record.lastreport }} {% report_alert record.lastreport %}' +) +REBOOT_TEMPLATE = '{% load common %}{% no_yes_img record.reboot_required %}' + + +class HostTable(BaseTable): + selection = tables.TemplateColumn( + CHECKBOX_TEMPLATE, + orderable=False, + verbose_name=SELECT_ALL_CHECKBOX, + attrs={'th': {'class': 'min-width-col centered'}, 'td': {'class': 'min-width-col centered'}}, + ) + hostname = tables.TemplateColumn( + HOSTNAME_TEMPLATE, + order_by='hostname', + verbose_name='Hostname', + attrs={'th': {'class': 'col-sm-3'}, 'td': {'class': 'col-sm-3'}}, + ) + sec_updates = tables.TemplateColumn( + SEC_UPDATES_TEMPLATE, + order_by='sec_updates_count', + verbose_name='Security Updates', + attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1 centered'}}, + ) + bug_updates = tables.TemplateColumn( + BUG_UPDATES_TEMPLATE, + order_by='bug_updates_count', + verbose_name='Bugfix Updates', + attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1 centered'}}, + ) + affected_errata = tables.TemplateColumn( + AFFECTED_ERRATA_TEMPLATE, + order_by='errata_count', + verbose_name='Affected by Errata', + attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1 centered'}}, + ) + kernel = tables.Column( + verbose_name='Running Kernel', + attrs={'th': {'class': 'col-sm-2'}, 'td': {'class': 'col-sm-2'}}, + ) + osvariant = tables.TemplateColumn( + OSVARIANT_TEMPLATE, + order_by='osvariant__name', + verbose_name='OS Variant', + attrs={'th': {'class': 'col-sm-2'}, 'td': {'class': 'col-sm-2'}}, + ) + lastreport = tables.TemplateColumn( + LASTREPORT_TEMPLATE, + order_by='lastreport', + verbose_name='Last Report', + attrs={'th': {'class': 'col-sm-2'}, 'td': {'class': 'col-sm-2'}}, + ) + reboot_required = tables.TemplateColumn( + REBOOT_TEMPLATE, + order_by='reboot_required', + verbose_name='Reboot Status', + attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1 centered'}}, + ) + + class Meta(BaseTable.Meta): + model = Host + fields = ( + 'selection', 'hostname', 'sec_updates', 'bug_updates', 'affected_errata', + 'kernel', 'osvariant', 'lastreport', 'reboot_required', + ) diff --git a/hosts/templates/hosts/host_list.html b/hosts/templates/hosts/host_list.html index 4c7429bc..19c0e308 100644 --- a/hosts/templates/hosts/host_list.html +++ b/hosts/templates/hosts/host_list.html @@ -1,7 +1,37 @@ -{% extends "objectlist.html" %} +{% extends "base.html" %} +{% load django_tables2 common %} {% block page_title %}Hosts{% endblock %} {% block content_title %} Hosts {% endblock %} {% block breadcrumbs %} {{ block.super }}
  • Hosts
    • {% endblock %} + +{% block content %} +
    +
    + {% get_querydict request as querydict %} + {% searchform terms querydict %} + +
    + {% csrf_token %} + + + {% include "bulk_actions.html" %} + + {% render_table table %} +
    +
    + + {% if filter_bar %} +
    +
    +
    Filter by...
    +
    + {{ filter_bar|safe }} +
    +
    +
    + {% endif %} +
    +{% endblock %} diff --git a/hosts/templates/hosts/host_table.html b/hosts/templates/hosts/host_table.html deleted file mode 100644 index bebb7723..00000000 --- a/hosts/templates/hosts/host_table.html +++ /dev/null @@ -1,28 +0,0 @@ -{% load common report_alert %} - - - - - - - - - - - - - - {% for host in object_list %} - - - - - - - - - - - {% endfor %} - -
    HostnameUpdatesAffected by ErrataRunning KernelOS VariantLast ReportReboot Status
    {{ host }}{% with count=host.get_num_security_updates %}{% if count != 0 %}{{ count }}{% else %} {% endif %}{% endwith %}{% with count=host.get_num_bugfix_updates %}{% if count != 0 %}{{ count }}{% else %} {% endif %}{% endwith %}{% with count=host.errata.count %}{% if count != 0 %}{{ count }}{% else %} {% endif %}{% endwith %}{{ host.kernel }}{{ host.osvariant }}{{ host.lastreport }}{% report_alert host.lastreport %}{% no_yes_img host.reboot_required %}
    diff --git a/hosts/templatetags/report_alert.py b/hosts/templatetags/report_alert.py index 48d8f966..09b906b3 100644 --- a/hosts/templatetags/report_alert.py +++ b/hosts/templatetags/report_alert.py @@ -17,7 +17,6 @@ from datetime import timedelta from django.template import Library -from django.templatetags.static import static from django.utils import timezone from django.utils.html import format_html @@ -29,12 +28,11 @@ @register.simple_tag def report_alert(lastreport): html = '' - alert_icon = static('img/icon-alert.gif') days = get_setting_of_type( setting_name='DAYS_WITHOUT_REPORT', setting_type=int, default=14, ) if lastreport < (timezone.now() - timedelta(days=days)): - html = f'Outdated Report' + html = '' return format_html(html) diff --git a/hosts/urls.py b/hosts/urls.py index b1521135..b3db0d54 100644 --- a/hosts/urls.py +++ b/hosts/urls.py @@ -23,6 +23,7 @@ urlpatterns = [ path('', views.host_list, name='host_list'), + path('bulk_action/', views.host_bulk_action, name='host_bulk_action'), path('/', views.host_detail, name='host_detail'), path('/delete/', views.host_delete, name='host_delete'), path('/edit/', views.host_edit, name='host_edit'), diff --git a/hosts/views.py b/hosts/views.py index 8f20ab19..7d969888 100644 --- a/hosts/views.py +++ b/hosts/views.py @@ -17,10 +17,10 @@ from django.contrib import messages from django.contrib.auth.decorators import login_required -from django.core.paginator import EmptyPage, PageNotAnInteger, Paginator -from django.db.models import Q +from django.db.models import Count, Q from django.shortcuts import get_object_or_404, redirect, render from django.urls import reverse +from django_tables2 import RequestConfig from rest_framework import viewsets from taggit.models import Tag @@ -29,14 +29,57 @@ from hosts.forms import EditHostForm from hosts.models import Host, HostRepo from hosts.serializers import HostRepoSerializer, HostSerializer +from hosts.tables import HostTable from operatingsystems.models import OSRelease, OSVariant from reports.models import Report +from util import sanitize_filter_params from util.filterspecs import Filter, FilterBar +def _get_filtered_hosts(filter_params): + """Helper to reconstruct filtered queryset from filter params.""" + from urllib.parse import parse_qs + params = parse_qs(filter_params) + + hosts = Host.objects.select_related() + + if 'domain_id' in params: + hosts = hosts.filter(domain=params['domain_id'][0]) + if 'package_id' in params: + hosts = hosts.filter(packages=params['package_id'][0]) + if 'package' in params: + hosts = hosts.filter(packages__name__name=params['package'][0]) + if 'repo_id' in params: + hosts = hosts.filter(repos=params['repo_id'][0]) + if 'arch_id' in params: + hosts = hosts.filter(arch=params['arch_id'][0]) + if 'osvariant_id' in params: + hosts = hosts.filter(osvariant=params['osvariant_id'][0]) + if 'osrelease_id' in params: + hosts = hosts.filter(osvariant__osrelease=params['osrelease_id'][0]) + if 'tag' in params: + hosts = hosts.filter(tags__name__in=[params['tag'][0]]) + if 'reboot_required' in params: + reboot_required = params['reboot_required'][0] == 'true' + hosts = hosts.filter(reboot_required=reboot_required) + if 'search' in params: + terms = params['search'][0].lower() + query = Q() + for term in terms.split(' '): + q = Q(hostname__icontains=term) + query = query & q + hosts = hosts.filter(query) + + return hosts + + @login_required def host_list(request): - hosts = Host.objects.select_related() + hosts = Host.objects.select_related().annotate( + sec_updates_count=Count('updates', filter=Q(updates__security=True)), + bug_updates_count=Count('updates', filter=Q(updates__security=False)), + errata_count=Count('errata'), + ) if 'domain_id' in request.GET: hosts = hosts.filter(domain=request.GET['domain_id']) @@ -76,16 +119,6 @@ def host_list(request): else: terms = '' - page_no = request.GET.get('page') - paginator = Paginator(hosts, 50) - - try: - page = paginator.page(page_no) - except PageNotAnInteger: - page = paginator.page(1) - except EmptyPage: - page = paginator.page(paginator.num_pages) - filter_list = [] tags = {} for tag in Tag.objects.all(): @@ -99,11 +132,23 @@ def host_list(request): filter_list.append(Filter(request, 'Reboot Required', 'reboot_required', {'true': 'Yes', 'false': 'No'})) filter_bar = FilterBar(request, filter_list) + table = HostTable(hosts) + RequestConfig(request, paginate={'per_page': 50}).configure(table) + + filter_params = sanitize_filter_params(request.GET.urlencode()) + bulk_actions = [ + {'value': 'find_updates', 'label': 'Find Updates'}, + {'value': 'delete', 'label': 'Delete'}, + ] + return render(request, 'hosts/host_list.html', - {'page': page, + {'table': table, 'filter_bar': filter_bar, - 'terms': terms}) + 'terms': terms, + 'total_count': hosts.count(), + 'filter_params': filter_params, + 'bulk_actions': bulk_actions}) @login_required @@ -178,6 +223,52 @@ def host_find_updates(request, hostname): return redirect(host.get_absolute_url()) +@login_required +def host_bulk_action(request): + """Handle bulk actions on hosts.""" + if request.method != 'POST': + return redirect('hosts:host_list') + + action = request.POST.get('action', '') + select_all_filtered = request.POST.get('select_all_filtered') == '1' + filter_params = sanitize_filter_params(request.POST.get('filter_params', '')) + + if not action: + messages.warning(request, 'Please select an action') + if filter_params: + return redirect(f"{reverse('hosts:host_list')}?{filter_params}") + return redirect('hosts:host_list') + + if select_all_filtered: + hosts = _get_filtered_hosts(filter_params) + else: + selected_ids = request.POST.getlist('selected_ids') + if not selected_ids: + messages.warning(request, 'No hosts selected') + if filter_params: + return redirect(f"{reverse('hosts:host_list')}?{filter_params}") + return redirect('hosts:host_list') + hosts = Host.objects.filter(id__in=selected_ids) + + count = hosts.count() + name = Host._meta.verbose_name if count == 1 else Host._meta.verbose_name_plural + + if action == 'find_updates': + from hosts.tasks import find_host_updates + for host in hosts: + find_host_updates.delay(host.id) + messages.success(request, f'Queued {count} {name} for update check') + elif action == 'delete': + hosts.delete() + messages.success(request, f'Deleted {count} {name}') + else: + messages.warning(request, 'Invalid action') + + if filter_params: + return redirect(f"{reverse('hosts:host_list')}?{filter_params}") + return redirect('hosts:host_list') + + class HostViewSet(viewsets.ModelViewSet): """ API endpoint that allows hosts to be viewed or edited. diff --git a/modules/tables.py b/modules/tables.py new file mode 100644 index 00000000..90811c78 --- /dev/null +++ b/modules/tables.py @@ -0,0 +1,72 @@ +# This file is part of Patchman. +# +# Patchman is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, version 3 only. +# +# Patchman is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Patchman. If not, see + +import django_tables2 as tables + +from modules.models import Module +from util.tables import BaseTable + +MODULE_NAME_TEMPLATE = '{{ record.name }}' +REPO_TEMPLATE = '{{ record.repo }}' +PACKAGES_TEMPLATE = ( + '' + '{{ record.packages.count }}' +) + + +class ModuleTable(BaseTable): + module_name = tables.TemplateColumn( + MODULE_NAME_TEMPLATE, + order_by='name', + verbose_name='Name', + attrs={'th': {'class': 'col-sm-auto'}, 'td': {'class': 'col-sm-auto'}}, + ) + stream = tables.Column( + verbose_name='Stream', + attrs={'th': {'class': 'col-sm-auto'}, 'td': {'class': 'col-sm-auto'}}, + ) + module_version = tables.Column( + accessor='version', + verbose_name='Version', + attrs={'th': {'class': 'col-sm-auto'}, 'td': {'class': 'col-sm-auto'}}, + ) + context = tables.Column( + verbose_name='Context', + attrs={'th': {'class': 'col-sm-auto'}, 'td': {'class': 'col-sm-auto'}}, + ) + repo = tables.TemplateColumn( + REPO_TEMPLATE, + verbose_name='Repo', + orderable=False, + attrs={'th': {'class': 'col-sm-auto'}, 'td': {'class': 'col-sm-auto'}}, + ) + module_packages = tables.TemplateColumn( + PACKAGES_TEMPLATE, + verbose_name='Packages', + orderable=False, + attrs={'th': {'class': 'col-sm-auto'}, 'td': {'class': 'col-sm-auto'}}, + ) + enabled_on_hosts = tables.TemplateColumn( + '{{ record.host_set.count }}', + verbose_name='Enabled on Hosts', + orderable=False, + attrs={'th': {'class': 'col-md-auto'}, 'td': {'class': 'col-md-auto'}}, + ) + + class Meta(BaseTable.Meta): + model = Module + fields = ( + 'module_name', 'stream', 'module_version', 'context', + 'repo', 'module_packages', 'enabled_on_hosts', + ) diff --git a/modules/templates/modules/module_table.html b/modules/templates/modules/module_table.html deleted file mode 100644 index cda47ea3..00000000 --- a/modules/templates/modules/module_table.html +++ /dev/null @@ -1,27 +0,0 @@ -{% load common %} - - - - - - - - - - - - - - {% for module in object_list %} - - - - - - - - - - {% endfor %} - -
    NameStreamVersionContextRepoPackagesEnabled on Hosts
    {{ module.name }}{{ module.stream }}{{ module.version }}{{ module.context }}{{ module.repo }}{{ module.packages.count }}{{ module.host_set.count }}
    diff --git a/modules/views.py b/modules/views.py index 2d017220..45703f29 100644 --- a/modules/views.py +++ b/modules/views.py @@ -15,13 +15,14 @@ # along with Patchman. If not, see from django.contrib.auth.decorators import login_required -from django.core.paginator import EmptyPage, PageNotAnInteger, Paginator from django.db.models import Q from django.shortcuts import get_object_or_404, render +from django_tables2 import RequestConfig from rest_framework import permissions, viewsets from modules.models import Module from modules.serializers import ModuleSerializer +from modules.tables import ModuleTable @login_required @@ -39,19 +40,12 @@ def module_list(request): else: terms = '' - page_no = request.GET.get('page') - paginator = Paginator(modules, 50) - - try: - page = paginator.page(page_no) - except PageNotAnInteger: - page = paginator.page(1) - except EmptyPage: - page = paginator.page(paginator.num_pages) + table = ModuleTable(modules) + RequestConfig(request, paginate={'per_page': 50}).configure(table) return render(request, 'modules/module_list.html', - {'page': page, + {'table': table, 'terms': terms}) diff --git a/operatingsystems/tables.py b/operatingsystems/tables.py new file mode 100644 index 00000000..0be5b77d --- /dev/null +++ b/operatingsystems/tables.py @@ -0,0 +1,167 @@ +# This file is part of Patchman. +# +# Patchman is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, version 3 only. +# +# Patchman is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Patchman. If not, see + +import django_tables2 as tables + +from operatingsystems.models import OSRelease, OSVariant +from util.tables import BaseTable + +CHECKBOX_TEMPLATE = '' +SELECT_ALL_CHECKBOX = '' + +# OSReleaseTable templates +OSRELEASE_NAME_TEMPLATE = '{{ record.name }}' +OSRELEASE_REPOS_TEMPLATE = ( + '' + '{{ record.repos.count }}' +) +OSVARIANTS_TEMPLATE = ( + '' + '{{ record.osvariant_set.count }}' +) +OSRELEASE_HOSTS_TEMPLATE = ( + '{% load common %}' + '{% host_count record %}' +) +OSRELEASE_ERRATA_TEMPLATE = ( + '' + '{{ record.erratum_set.count }}' +) + +# OSVariantTable templates +OSVARIANT_NAME_TEMPLATE = '{{ record }}' +OSVARIANT_CODENAME_TEMPLATE = ( + '{% if record.codename %}{{ record.codename }}' + '{% else %}{% if record.osrelease %}{{ record.osrelease.codename }}{% endif %}{% endif %}' +) +OSVARIANT_HOSTS_TEMPLATE = ( + '' + '{{ record.host_set.count }}' +) +OSVARIANT_OSRELEASE_TEMPLATE = ( + '{% if record.osrelease %}' + '{{ record.osrelease }}' + '{% endif %}' +) +REPOS_OSRELEASE_TEMPLATE = ( + '{% if record.osrelease.repos.count != None %}{{ record.osrelease.repos.count }}{% else %}0{% endif %}' +) + + +class OSReleaseTable(BaseTable): + selection = tables.TemplateColumn( + CHECKBOX_TEMPLATE, + orderable=False, + verbose_name=SELECT_ALL_CHECKBOX, + attrs={'th': {'class': 'min-width-col centered'}, 'td': {'class': 'min-width-col centered'}}, + ) + osrelease_name = tables.TemplateColumn( + OSRELEASE_NAME_TEMPLATE, + order_by='name', + verbose_name='OS Release', + attrs={'th': {'class': 'col-sm-3'}, 'td': {'class': 'col-sm-3'}}, + ) + cpe_name = tables.Column( + verbose_name='CPE Name', + default='', + attrs={'th': {'class': 'col-sm-2'}, 'td': {'class': 'col-sm-2'}}, + ) + osrelease_codename = tables.Column( + accessor='codename', + verbose_name='Codename', + default='', + attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1'}}, + ) + osrelease_repos = tables.TemplateColumn( + OSRELEASE_REPOS_TEMPLATE, + verbose_name='Repos', + orderable=False, + attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1'}}, + ) + osvariants = tables.TemplateColumn( + OSVARIANTS_TEMPLATE, + verbose_name='OS Variants', + orderable=False, + attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1'}}, + ) + osrelease_hosts = tables.TemplateColumn( + OSRELEASE_HOSTS_TEMPLATE, + verbose_name='Hosts', + orderable=False, + attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1'}}, + ) + osrelease_errata = tables.TemplateColumn( + OSRELEASE_ERRATA_TEMPLATE, + verbose_name='Errata', + orderable=False, + attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1'}}, + ) + + class Meta(BaseTable.Meta): + model = OSRelease + fields = ( + 'selection', 'osrelease_name', 'cpe_name', 'osrelease_codename', 'osrelease_repos', + 'osvariants', 'osrelease_hosts', 'osrelease_errata', + ) + + +class OSVariantTable(BaseTable): + selection = tables.TemplateColumn( + CHECKBOX_TEMPLATE, + orderable=False, + verbose_name=SELECT_ALL_CHECKBOX, + attrs={'th': {'class': 'min-width-col centered'}, 'td': {'class': 'min-width-col centered'}}, + ) + osvariant_name = tables.TemplateColumn( + OSVARIANT_NAME_TEMPLATE, + order_by='name', + verbose_name='Name', + attrs={'th': {'class': 'col-sm-3'}, 'td': {'class': 'col-sm-3'}}, + ) + osvariant_arch = tables.Column( + accessor='arch__name', + verbose_name='Architecture', + attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1'}}, + ) + osvariant_codename = tables.TemplateColumn( + OSVARIANT_CODENAME_TEMPLATE, + order_by='codename', + verbose_name='Codename', + attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1'}}, + ) + osvariant_hosts = tables.TemplateColumn( + OSVARIANT_HOSTS_TEMPLATE, + verbose_name='Hosts', + order_by='hosts_count', + attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1'}}, + ) + osrelease = tables.TemplateColumn( + OSVARIANT_OSRELEASE_TEMPLATE, + order_by='osrelease__name', + verbose_name='OS Release', + attrs={'th': {'class': 'col-sm-4'}, 'td': {'class': 'col-sm-4'}}, + ) + repos_osrelease = tables.TemplateColumn( + REPOS_OSRELEASE_TEMPLATE, + verbose_name='Repos (OS Release)', + order_by='repos_count', + attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1'}}, + ) + + class Meta(BaseTable.Meta): + model = OSVariant + fields = ( + 'selection', 'osvariant_name', 'osvariant_arch', 'osvariant_codename', + 'osvariant_hosts', 'osrelease', 'repos_osrelease', + ) diff --git a/operatingsystems/templates/operatingsystems/operatingsystemrelease_table.html b/operatingsystems/templates/operatingsystems/operatingsystemrelease_table.html deleted file mode 100644 index 6a7eae13..00000000 --- a/operatingsystems/templates/operatingsystems/operatingsystemrelease_table.html +++ /dev/null @@ -1,27 +0,0 @@ -{% load common %} - - - - - - - - - - - - - - {% for osrelease in object_list %} - - - - - - - - - - {% endfor %} - -
    OS ReleaseCPE NameCodenameReposOS VariantsHostsErrata
    {{ osrelease.name }}{% if osrelease.cpe_name %}{{ osrelease.cpe_name }}{% endif %}{% if osrelease.codename %}{{ osrelease.codename }}{% endif %}{{ osrelease.repos.count }}{{ osrelease.osvariant_set.count }}{% host_count osrelease %}{{ osrelease.erratum_set.count }}
    diff --git a/operatingsystems/templates/operatingsystems/operatingsystemvariant_table.html b/operatingsystems/templates/operatingsystems/operatingsystemvariant_table.html deleted file mode 100644 index 3ef8403f..00000000 --- a/operatingsystems/templates/operatingsystems/operatingsystemvariant_table.html +++ /dev/null @@ -1,25 +0,0 @@ -{% load common %} - - - - - - - - - - - - - {% for osvariant in object_list %} - - - - - - - - - {% endfor %} - -
    NameArchitectureCodenameHostsOS ReleaseRepos (OS Release)
    {{ osvariant }}{{ osvariant.arch }}{% if osvariant.codename %}{{ osvariant.codename }}{% else %}{% if osvariant.osrelease %}{{ osvariant.osrelease.codename }}{% endif %}{% endif %}{{ osvariant.host_set.count }}{% if osvariant.osrelease %}{{ osvariant.osrelease }}{% endif %}{% if osvariant.osrelease.repos.count != None %}{{ osvariant.osrelease.repos.count }}{% else %}0{% endif %}
    diff --git a/operatingsystems/templates/operatingsystems/osrelease_list.html b/operatingsystems/templates/operatingsystems/osrelease_list.html index 1dfc80e1..95b570e0 100644 --- a/operatingsystems/templates/operatingsystems/osrelease_list.html +++ b/operatingsystems/templates/operatingsystems/osrelease_list.html @@ -1,7 +1,26 @@ -{% extends "objectlist.html" %} +{% extends "base.html" %} +{% load django_tables2 common %} {% block page_title %}OS Releases{% endblock %} {% block breadcrumbs %} {{ block.super }}
  • Operating Systems
  • OS Releases
    • {% endblock %} {% block content_title %} OS Releases {% endblock %} + +{% block content %} +
    +
    + {% get_querydict request as querydict %} + {% searchform terms querydict %} + +
    + {% csrf_token %} + + + {% include "bulk_actions.html" %} + + {% render_table table %} +
    +
    +
    +{% endblock %} diff --git a/operatingsystems/templates/operatingsystems/osvariant_list.html b/operatingsystems/templates/operatingsystems/osvariant_list.html index b83ede5f..f72274d4 100644 --- a/operatingsystems/templates/operatingsystems/osvariant_list.html +++ b/operatingsystems/templates/operatingsystems/osvariant_list.html @@ -1,6 +1,5 @@ -{% extends "objectlist.html" %} - -{% load common bootstrap3 %} +{% extends "base.html" %} +{% load django_tables2 common bootstrap3 %} {% block page_title %}OS Variants{% endblock %} @@ -8,12 +7,26 @@ {% block breadcrumbs %} {{ block.super }}
  • Operating Systems
  • OS Variants
    • {% endblock %} -{% block objectlist_actions %} +{% block content %} +
    +
    + {% if user.is_authenticated and perms.is_admin and nohost_osvariants %} +
    + {% bootstrap_icon "trash" %} Delete OS Variants with no Hosts +
    + {% endif %} -{% if user.is_authenticated and perms.is_admin and nohost_osvariants %} -
    - {% bootstrap_icon "trash" %} Delete OS Variants with no Hosts -
    -{% endif %} + {% get_querydict request as querydict %} + {% searchform terms querydict %} + +
    + {% csrf_token %} + + {% include "bulk_actions.html" %} + + {% render_table table %} +
    +
    +
    {% endblock %} diff --git a/operatingsystems/urls.py b/operatingsystems/urls.py index df194c9d..f58ed5ec 100644 --- a/operatingsystems/urls.py +++ b/operatingsystems/urls.py @@ -24,10 +24,12 @@ urlpatterns = [ path('', views.os_landing, name='os_landing'), path('variants/', views.osvariant_list, name='osvariant_list'), + path('variants/bulk_action/', views.osvariant_bulk_action, name='osvariant_bulk_action'), path('variants//', views.osvariant_detail, name='osvariant_detail'), path('variants//delete/', views.osvariant_delete, name='osvariant_delete'), path('variants/no_host/delete/', views.delete_nohost_osvariants, name='delete_nohost_osvariants'), path('releases/', views.osrelease_list, name='osrelease_list'), + path('releases/bulk_action/', views.osrelease_bulk_action, name='osrelease_bulk_action'), path('releases//', views.osrelease_detail, name='osrelease_detail'), path('releases//delete/', views.osrelease_delete, name='osrelease_delete'), ] diff --git a/operatingsystems/views.py b/operatingsystems/views.py index 6009f119..7dd4fcb5 100644 --- a/operatingsystems/views.py +++ b/operatingsystems/views.py @@ -17,10 +17,10 @@ from django.contrib import messages from django.contrib.auth.decorators import login_required -from django.core.paginator import EmptyPage, PageNotAnInteger, Paginator -from django.db.models import Q +from django.db.models import Count, Q from django.shortcuts import get_object_or_404, redirect, render from django.urls import reverse +from django_tables2 import RequestConfig from rest_framework import viewsets from hosts.models import Host @@ -31,11 +31,56 @@ from operatingsystems.serializers import ( OSReleaseSerializer, OSVariantSerializer, ) +from operatingsystems.tables import OSReleaseTable, OSVariantTable +from util import sanitize_filter_params + + +def _get_filtered_osvariants(filter_params): + """Helper to reconstruct filtered queryset from filter params.""" + from urllib.parse import parse_qs + params = parse_qs(filter_params) + + osvariants = OSVariant.objects.select_related() + + if 'osrelease_id' in params: + osvariants = osvariants.filter(osrelease=params['osrelease_id'][0]) + if 'search' in params: + terms = params['search'][0].lower() + query = Q() + for term in terms.split(' '): + q = Q(name__icontains=term) + query = query & q + osvariants = osvariants.filter(query) + + return osvariants + + +def _get_filtered_osreleases(filter_params): + """Helper to reconstruct filtered queryset from filter params.""" + from urllib.parse import parse_qs + params = parse_qs(filter_params) + + osreleases = OSRelease.objects.select_related() + + if 'erratum_id' in params: + osreleases = osreleases.filter(erratum=params['erratum_id'][0]) + if 'search' in params: + terms = params['search'][0].lower() + query = Q() + for term in terms.split(' '): + q = Q(name__icontains=term) + query = query & q + osreleases = osreleases.filter(query) + + return osreleases @login_required def osvariant_list(request): - osvariants = OSVariant.objects.select_related() + osvariants = OSVariant.objects.select_related().annotate( + hosts_count=Count('host'), + repos_count=Count('osrelease__repos'), + ) if 'osrelease_id' in request.GET: osvariants = osvariants.filter(osrelease=request.GET['osrelease_id']) @@ -50,23 +95,24 @@ def osvariant_list(request): else: terms = '' - page_no = request.GET.get('page') - paginator = Paginator(osvariants, 50) + nohost_osvariants = OSVariant.objects.filter(host__isnull=True).exists() - try: - page = paginator.page(page_no) - except PageNotAnInteger: - page = paginator.page(1) - except EmptyPage: - page = paginator.page(paginator.num_pages) + table = OSVariantTable(osvariants) + RequestConfig(request, paginate={'per_page': 50}).configure(table) - nohost_osvariants = OSVariant.objects.filter(host__isnull=True).exists() + filter_params = sanitize_filter_params(request.GET.urlencode()) + bulk_actions = [ + {'value': 'delete', 'label': 'Delete'}, + ] return render(request, 'operatingsystems/osvariant_list.html', - {'page': page, + {'table': table, 'terms': terms, - 'nohost_osvariants': nohost_osvariants}) + 'nohost_osvariants': nohost_osvariants, + 'total_count': osvariants.count(), + 'filter_params': filter_params, + 'bulk_actions': bulk_actions}) @login_required @@ -151,20 +197,21 @@ def osrelease_list(request): else: terms = '' - page_no = request.GET.get('page') - paginator = Paginator(osreleases, 50) + table = OSReleaseTable(osreleases) + RequestConfig(request, paginate={'per_page': 50}).configure(table) - try: - page = paginator.page(page_no) - except PageNotAnInteger: - page = paginator.page(1) - except EmptyPage: - page = paginator.page(paginator.num_pages) + filter_params = sanitize_filter_params(request.GET.urlencode()) + bulk_actions = [ + {'value': 'delete', 'label': 'Delete'}, + ] return render(request, 'operatingsystems/osrelease_list.html', - {'page': page, - 'terms': terms}) + {'table': table, + 'terms': terms, + 'total_count': osreleases.count(), + 'filter_params': filter_params, + 'bulk_actions': bulk_actions}) @login_required @@ -214,6 +261,88 @@ def os_landing(request): return render(request, 'operatingsystems/os_landing.html') +@login_required +def osvariant_bulk_action(request): + """Handle bulk actions on OS variants.""" + if request.method != 'POST': + return redirect('operatingsystems:osvariant_list') + + action = request.POST.get('action', '') + select_all_filtered = request.POST.get('select_all_filtered') == '1' + filter_params = request.POST.get('filter_params', '') + + if not action: + messages.warning(request, 'Please select an action') + if filter_params: + return redirect(f"{reverse('operatingsystems:osvariant_list')}?{filter_params}") + return redirect('operatingsystems:osvariant_list') + + if select_all_filtered: + osvariants = _get_filtered_osvariants(filter_params) + else: + selected_ids = request.POST.getlist('selected_ids') + if not selected_ids: + messages.warning(request, 'No OS Variants selected') + if filter_params: + return redirect(f"{reverse('operatingsystems:osvariant_list')}?{filter_params}") + return redirect('operatingsystems:osvariant_list') + osvariants = OSVariant.objects.filter(id__in=selected_ids) + + count = osvariants.count() + name = OSVariant._meta.verbose_name if count == 1 else OSVariant._meta.verbose_name_plural + + if action == 'delete': + osvariants.delete() + messages.success(request, f'Deleted {count} {name}') + else: + messages.warning(request, 'Invalid action') + + if filter_params: + return redirect(f"{reverse('operatingsystems:osvariant_list')}?{filter_params}") + return redirect('operatingsystems:osvariant_list') + + +@login_required +def osrelease_bulk_action(request): + """Handle bulk actions on OS releases.""" + if request.method != 'POST': + return redirect('operatingsystems:osrelease_list') + + action = request.POST.get('action', '') + select_all_filtered = request.POST.get('select_all_filtered') == '1' + filter_params = request.POST.get('filter_params', '') + + if not action: + messages.warning(request, 'Please select an action') + if filter_params: + return redirect(f"{reverse('operatingsystems:osrelease_list')}?{filter_params}") + return redirect('operatingsystems:osrelease_list') + + if select_all_filtered: + osreleases = _get_filtered_osreleases(filter_params) + else: + selected_ids = request.POST.getlist('selected_ids') + if not selected_ids: + messages.warning(request, 'No OS Releases selected') + if filter_params: + return redirect(f"{reverse('operatingsystems:osrelease_list')}?{filter_params}") + return redirect('operatingsystems:osrelease_list') + osreleases = OSRelease.objects.filter(id__in=selected_ids) + + count = osreleases.count() + name = OSRelease._meta.verbose_name if count == 1 else OSRelease._meta.verbose_name_plural + + if action == 'delete': + osreleases.delete() + messages.success(request, f'Deleted {count} {name}') + else: + messages.warning(request, 'Invalid action') + + if filter_params: + return redirect(f"{reverse('operatingsystems:osrelease_list')}?{filter_params}") + return redirect('operatingsystems:osrelease_list') + + class OSVariantViewSet(viewsets.ModelViewSet): """ API endpoint that allows operating system variants to be viewed or edited. diff --git a/packages/tables.py b/packages/tables.py new file mode 100644 index 00000000..633c79a2 --- /dev/null +++ b/packages/tables.py @@ -0,0 +1,120 @@ +# This file is part of Patchman. +# +# Patchman is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, version 3 only. +# +# Patchman is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Patchman. If not, see + +import django_tables2 as tables + +from packages.models import Package, PackageName +from util.tables import BaseTable + +PACKAGE_NAME_TEMPLATE = '{{ record }}' +PACKAGE_REPOS_TEMPLATE = ( + '' + 'Available from {{ record.repo_count }} Repositories' +) +PACKAGE_HOSTS_TEMPLATE = ( + '' + 'Installed on {{ record.host_set.count }} Hosts' +) +AFFECTED_TEMPLATE = ( + '' + 'Affected by {{ record.affected_by_erratum.count }} Errata' +) +FIXED_TEMPLATE = ( + '' + 'Provides fix in {{ record.provides_fix_in_erratum.count }} Errata' +) + + +class PackageTable(BaseTable): + package_name = tables.TemplateColumn( + PACKAGE_NAME_TEMPLATE, + order_by='name__name', + verbose_name='Package', + attrs={'th': {'class': 'col-sm-auto'}, 'td': {'class': 'col-sm-auto'}}, + ) + epoch = tables.Column( + verbose_name='Epoch', + default='', + attrs={'th': {'class': 'col-sm-auto'}, 'td': {'class': 'col-sm-auto'}}, + ) + package_version = tables.Column( + accessor='version', + verbose_name='Version', + attrs={'th': {'class': 'col-sm-auto'}, 'td': {'class': 'col-sm-auto'}}, + ) + release = tables.Column( + verbose_name='Release', + default='', + attrs={'th': {'class': 'col-sm-auto'}, 'td': {'class': 'col-sm-auto'}}, + ) + package_arch = tables.Column( + accessor='arch__name', + verbose_name='Arch', + attrs={'th': {'class': 'col-sm-auto'}, 'td': {'class': 'col-sm-auto'}}, + ) + packagetype = tables.Column( + accessor='packagetype', + verbose_name='Type', + attrs={'th': {'class': 'col-sm-auto'}, 'td': {'class': 'col-sm-auto'}}, + ) + package_repos = tables.TemplateColumn( + PACKAGE_REPOS_TEMPLATE, + verbose_name='Repositories', + orderable=False, + attrs={'th': {'class': 'col-sm-auto'}, 'td': {'class': 'col-sm-auto'}}, + ) + package_hosts = tables.TemplateColumn( + PACKAGE_HOSTS_TEMPLATE, + verbose_name='Hosts', + orderable=False, + attrs={'th': {'class': 'col-sm-auto'}, 'td': {'class': 'col-sm-auto'}}, + ) + affected = tables.TemplateColumn( + AFFECTED_TEMPLATE, + verbose_name='Affected', + orderable=False, + attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1'}}, + ) + fixed = tables.TemplateColumn( + FIXED_TEMPLATE, + verbose_name='Fixed', + orderable=False, + attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1'}}, + ) + + class Meta(BaseTable.Meta): + model = Package + fields = ( + 'package_name', 'epoch', 'package_version', 'release', 'package_arch', + 'packagetype', 'package_repos', 'package_hosts', 'affected', 'fixed', + ) + + +class PackageNameTable(BaseTable): + packagename_name = tables.TemplateColumn( + PACKAGE_NAME_TEMPLATE, + order_by='name', + verbose_name='Package', + attrs={'th': {'class': 'col-sm-6'}, 'td': {'class': 'col-sm-6'}}, + ) + versions = tables.TemplateColumn( + '{{ record.package_set.count }}', + orderable=False, + verbose_name='Versions available', + attrs={'th': {'class': 'col-sm-6'}, 'td': {'class': 'col-sm-6'}}, + ) + + class Meta(BaseTable.Meta): + model = PackageName + fields = ('packagename_name', 'versions') diff --git a/packages/templates/packages/package_name_table.html b/packages/templates/packages/package_name_table.html deleted file mode 100644 index 39977d96..00000000 --- a/packages/templates/packages/package_name_table.html +++ /dev/null @@ -1,17 +0,0 @@ -{% load common %} - - - - - - - - - {% for packagename in object_list %} - - - - - {% endfor %} - -
    PackageVersions available
    {{ packagename }}{{ packagename.package_set.count }}
    diff --git a/packages/templates/packages/package_table.html b/packages/templates/packages/package_table.html deleted file mode 100644 index 06316521..00000000 --- a/packages/templates/packages/package_table.html +++ /dev/null @@ -1,30 +0,0 @@ -{% load common %} - - - - - - - - - - - - - - - - {% for package in object_list %} - - - - - - - - - - - {% endfor %} - -
    PackageEpochVersionReleaseArchTypeRepositoriesHostsErrata
    {{ package }} {{ package.epoch }} {{ package.version }} {{ package.release }} {{ package.arch }} {{ package.get_packagetype_display }} Available from {{ package.repo_count }} Repositories Installed on {{ package.host_set.count }} Hosts Affected by {{ package.affected_by_erratum.count }} Errata Provides fix in {{ package.provides_fix_in_erratum.count }} Errata
    diff --git a/packages/views.py b/packages/views.py index 413faee0..287c033d 100644 --- a/packages/views.py +++ b/packages/views.py @@ -16,9 +16,9 @@ # along with Patchman. If not, see from django.contrib.auth.decorators import login_required -from django.core.paginator import EmptyPage, PageNotAnInteger, Paginator from django.db.models import Q from django.shortcuts import get_object_or_404, render +from django_tables2 import RequestConfig from rest_framework import viewsets from arch.models import PackageArchitecture @@ -26,6 +26,7 @@ from packages.serializers import ( PackageNameSerializer, PackageSerializer, PackageUpdateSerializer, ) +from packages.tables import PackageNameTable, PackageTable from util.filterspecs import Filter, FilterBar @@ -98,16 +99,6 @@ def package_list(request): else: terms = '' - page_no = request.GET.get('page') - paginator = Paginator(packages, 50) - - try: - page = paginator.page(page_no) - except PageNotAnInteger: - page = paginator.page(1) - except EmptyPage: - page = paginator.page(paginator.num_pages) - filter_list = [] filter_list.append(Filter(request, 'Affected by Errata', 'affected_by_errata', {'true': 'Yes', 'false': 'No'})) filter_list.append(Filter(request, 'Provides Fix in Errata', 'provides_fix_in_erratum', @@ -118,9 +109,12 @@ def package_list(request): filter_list.append(Filter(request, 'Architecture', 'arch_id', PackageArchitecture.objects.all())) filter_bar = FilterBar(request, filter_list) + table = PackageTable(packages) + RequestConfig(request, paginate={'per_page': 50}).configure(table) + return render(request, 'packages/package_list.html', - {'page': page, + {'table': table, 'filter_bar': filter_bar, 'terms': terms}) @@ -145,27 +139,19 @@ def package_name_list(request): else: terms = '' - page_no = request.GET.get('page') - paginator = Paginator(packages, 50) - - try: - page = paginator.page(page_no) - except PageNotAnInteger: - page = paginator.page(1) - except EmptyPage: - page = paginator.page(paginator.num_pages) - filter_list = [] filter_list.append(Filter(request, 'Package Type', 'packagetype', Package.PACKAGE_TYPES)) filter_list.append(Filter(request, 'Architecture', 'arch_id', PackageArchitecture.objects.all())) filter_bar = FilterBar(request, filter_list) + table = PackageNameTable(packages) + RequestConfig(request, paginate={'per_page': 50}).configure(table) + return render(request, 'packages/package_name_list.html', - {'page': page, + {'table': table, 'filter_bar': filter_bar, - 'terms': terms, - 'table_template': 'packages/package_name_table.html'}) + 'terms': terms}) @login_required diff --git a/patchman/settings.py b/patchman/settings.py index c3089caa..1553b247 100644 --- a/patchman/settings.py +++ b/patchman/settings.py @@ -48,6 +48,7 @@ 'django.template.context_processors.static', 'django.template.context_processors.tz', 'django.contrib.messages.context_processors.messages', + 'util.context_processors.issues_count', ], 'debug': DEBUG, }, @@ -79,6 +80,7 @@ 'django_extensions', 'taggit', 'bootstrap3', + 'django_tables2', 'rest_framework', 'django_filters', 'celery', @@ -108,6 +110,8 @@ TAGGIT_CASE_INSENSITIVE = True +DJANGO_TABLES2_TEMPLATE = 'table.html' + CELERY_BROKER_URL = 'redis://127.0.0.1:6379/0' CELERY_BROKER_TRANSPORT_OPTIONS = { 'queue_order_strategy': 'priority', diff --git a/patchman/sqlite3/base.py b/patchman/sqlite3/base.py index 308e0563..c7ba0c6f 100644 --- a/patchman/sqlite3/base.py +++ b/patchman/sqlite3/base.py @@ -1,3 +1,17 @@ +# This file is part of Patchman. +# +# Patchman is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, version 3 only. +# +# Patchman is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Patchman. If not, see + # temporary fix for 'database is locked' error on sqlite3 # can be removed when using django 5.1 and BEGIN IMMEDIATE in OPTIONS # see https://blog.pecar.me/django-sqlite-dblock for more details diff --git a/patchman/static/css/base.css b/patchman/static/css/base.css index 2ae0e5e1..0db76909 100644 --- a/patchman/static/css/base.css +++ b/patchman/static/css/base.css @@ -9,5 +9,21 @@ .label-brick { font-size: 12px; font-weight: normal; margin-bottom: 3px; padding-top: 5px; border-radius: 4px; height:25px; border-style:solid; border-width:thin; border-color: #222; white-space: normal; display: inline-block; } .breadcrumb { font-size: 12px; background-color: #222; border-radius: 0; margin-bottom: 3px; } .navbar { margin-bottom: 0; padding-bottom: 0; border-radius: 0; } +.navbar-inverse .dropdown-menu { background-color: #222; } +.navbar-inverse .dropdown-menu > li > a { color: #9d9d9d; } +.navbar-inverse .dropdown-menu > li > a:hover { background-color: #333; color: #fff; } .well-sm { margin-bottom: 0; } .centered { text-align: center; } +th.min-width-col, td.min-width-col { width: 1%; white-space: nowrap; padding: 5px !important; } +.table td { vertical-align: bottom !important; } + +/* Center pagination controls produced by django-tables2 without centering table cell contents */ +.django-tables2 .pagination { + text-align: center; +} +/* Ensure pagination lists are centered and horizontally aligned in all contexts */ +.pagination { + display: flex; + justify-content: center; + align-items: center; +} diff --git a/patchman/static/img/icon-alert.gif b/patchman/static/img/icon-alert.gif deleted file mode 100644 index a1dde2625445b76d041ae02ccfcb83481ca63c5e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 145 zcmV;C0B-+BNk%w1VGsZi0J9GO|G@+Q!3O`;RR7pu|IkAJ%Ps%YPXF0v|INcdJ{u&=}=IXLDhr+J%S1nrq(gCL;wIgri4F* diff --git a/patchman/static/img/icon-no.gif b/patchman/static/img/icon-no.gif deleted file mode 100644 index 1b4ee5814570885705399533f1182f8b0491c5fb..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 176 zcmZ?wbhEHb`H-TFR%!C^)o_GDj!gPtK1gc27Dv@$SQ0{~`FJvsmY diff --git a/patchman/static/img/icon-yes.gif b/patchman/static/img/icon-yes.gif deleted file mode 100644 index 73992827403791d6c1a75a079880e41dce7e0214..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 299 zcmZ?wbhEHbb?NhTQ$x_deWPc4O)NkN2|oXRf%p{M+wuUw(Z# z`TWGXJ8Mf07p=Or^7yl3mtJ2C+~V)C-fh~&DX}}E_C4PF@Y93ee}B)tGUw-?pC_Il zZ#vO%{oS?y|Nqw=uUUR`+4?){5_iQh&Q{xM6OkFieY2o T4)tf0@^WEj=4)bdWUvMRbX#E6 diff --git a/patchman/static/js/ajax-jquery.js b/patchman/static/js/ajax-jquery.js deleted file mode 100644 index af4c0729..00000000 --- a/patchman/static/js/ajax-jquery.js +++ /dev/null @@ -1,29 +0,0 @@ -function getCookie(name) { - var cookieValue = null; - if (document.cookie && document.cookie != '') { - var cookies = document.cookie.split(';'); - for (var i = 0; i < cookies.length; i++) { - var cookie = jQuery.trim(cookies[i]); - // Does this cookie string begin with the name we want? - if (cookie.substring(0, name.length + 1) == (name + '=')) { - cookieValue = decodeURIComponent(cookie.substring(name.length + 1)); - break; - } - } - } - return cookieValue; -} - -var csrftoken = getCookie('csrftoken'); - -function csrfSafeMethod(method) { - // these HTTP methods do not require CSRF protection - return (/^(GET|HEAD|OPTIONS|TRACE)$/.test(method)); -} -$.ajaxSetup({ - beforeSend: function(xhr, settings) { - if (!csrfSafeMethod(settings.type) && !this.crossDomain) { - xhr.setRequestHeader("X-CSRFToken", csrftoken); - } - } -}); diff --git a/patchman/static/js/button-post.js b/patchman/static/js/button-post.js deleted file mode 100644 index ad19698f..00000000 --- a/patchman/static/js/button-post.js +++ /dev/null @@ -1,23 +0,0 @@ -function repo_toggle_enabled(id, element, e) { - e.preventDefault(); - var url = id + "toggle_enabled/"; - $.post(url); - if (element.innerHTML.indexOf("icon-no.gif") > -1) { - var newHTML = element.innerHTML.replace("icon-no.gif", "icon-yes.gif").replace("Disabled", "Enabled"); - } else { - var newHTML = element.innerHTML.replace("icon-yes.gif", "icon-no.gif").replace("Enabled", "Disabled"); - } - element.innerHTML = newHTML; -} - -function repo_toggle_security(id, element, e) { - e.preventDefault(); - var url = id + "toggle_security/"; - $.post(url); - if (element.innerHTML.indexOf("icon-no.gif") > -1) { - var newHTML = element.innerHTML.replace("icon-no.gif", "icon-yes.gif").replace("Non-Security", "Security"); - } else { - var newHTML = element.innerHTML.replace("icon-yes.gif", "icon-no.gif").replace("Security", "Non-Security"); - } - element.innerHTML = newHTML; -} diff --git a/patchman/urls.py b/patchman/urls.py index 2ae64f56..2b9a9787 100644 --- a/patchman/urls.py +++ b/patchman/urls.py @@ -1,7 +1,7 @@ # Copyright 2012 VPAC, http://www.vpac.org # Copyright 2013-2021 Marcus Furlong # -# This file is part of patchman +# This file is part of Patchman. # # Patchman is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/reports/tables.py b/reports/tables.py new file mode 100644 index 00000000..52f077e0 --- /dev/null +++ b/reports/tables.py @@ -0,0 +1,60 @@ +# This file is part of Patchman. +# +# Patchman is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, version 3 only. +# +# Patchman is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Patchman. If not, see + +import django_tables2 as tables + +from reports.models import Report +from util.tables import BaseTable + +CHECKBOX_TEMPLATE = '' +SELECT_ALL_CHECKBOX = '' +REPORT_ID_TEMPLATE = '{{ record.id }}' +PROCESSED_TEMPLATE = '{% load common %}{% yes_no_img record.processed %}' + + +class ReportTable(BaseTable): + selection = tables.TemplateColumn( + CHECKBOX_TEMPLATE, + orderable=False, + verbose_name=SELECT_ALL_CHECKBOX, + attrs={'th': {'class': 'min-width-col centered'}, 'td': {'class': 'min-width-col centered'}}, + ) + report_id = tables.TemplateColumn( + REPORT_ID_TEMPLATE, + order_by='id', + verbose_name='ID', + attrs={'th': {'class': 'min-width-col'}, 'td': {'class': 'min-width-col'}}, + ) + host = tables.Column( + accessor='host', + verbose_name='Host', + attrs={'th': {'class': 'col-sm-3'}, 'td': {'class': 'col-sm-3'}}, + ) + created = tables.Column( + verbose_name='Created', + attrs={'th': {'class': 'col-sm-3'}, 'td': {'class': 'col-sm-3'}}, + ) + report_ip = tables.Column( + verbose_name='IP Address', + attrs={'th': {'class': 'col-sm-2'}, 'td': {'class': 'col-sm-2'}}, + ) + processed = tables.TemplateColumn( + PROCESSED_TEMPLATE, + verbose_name='Processed', + attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'centered col-sm-1'}}, + ) + + class Meta(BaseTable.Meta): + model = Report + fields = ('selection', 'report_id', 'host', 'created', 'report_ip', 'processed') diff --git a/reports/templates/reports/report_detail.html b/reports/templates/reports/report_detail.html index 247128bb..08d5005f 100644 --- a/reports/templates/reports/report_detail.html +++ b/reports/templates/reports/report_detail.html @@ -1,6 +1,6 @@ {% extends "base.html" %} -{% load bootstrap3 %} +{% load bootstrap3 common %} {% block page_title %}Report - {{ report }} {% endblock %} @@ -39,7 +39,7 @@ Tags {{ report.tags }} Client Protocol {{ report.protocol }} User Agent {{ report.useragent }} - Has Been Processed? {{ report.processed }} + Has Been Processed? {% yes_no_img report.processed %} {% if user.is_authenticated and perms.is_admin %} {% bootstrap_icon "trash" %} Delete this Report diff --git a/reports/templates/reports/report_list.html b/reports/templates/reports/report_list.html index 8ca6b1a6..a66464b2 100644 --- a/reports/templates/reports/report_list.html +++ b/reports/templates/reports/report_list.html @@ -1,7 +1,37 @@ -{% extends "objectlist.html" %} +{% extends "base.html" %} +{% load django_tables2 common %} {% block page_title %}Reports{% endblock %} {% block breadcrumbs %} {{ block.super }}
  • Reports
    • {% endblock %} {% block content_title %} Reports {% endblock %} + +{% block content %} +
    +
    + {% get_querydict request as querydict %} + {% searchform terms querydict %} + +
    + {% csrf_token %} + + + {% include "bulk_actions.html" %} + + {% render_table table %} +
    +
    + + {% if filter_bar %} +
    +
    +
    Filter by...
    +
    + {{ filter_bar|safe }} +
    +
    +
    + {% endif %} +
    +{% endblock %} diff --git a/reports/templates/reports/report_table.html b/reports/templates/reports/report_table.html deleted file mode 100644 index 747c8613..00000000 --- a/reports/templates/reports/report_table.html +++ /dev/null @@ -1,23 +0,0 @@ -{% load common %} - - - - - - - - - - - - {% for report in object_list %} - - - - - - - - {% endfor %} - -
    IDHostTimeIP AddressProcessed
    {{ report.id }} {{ report.host }} {{ report.created }} {{ report.report_ip }} {% yes_no_img report.processed 'Processed' 'Not Processed' %}
    diff --git a/reports/urls.py b/reports/urls.py index 8826cc82..b419317d 100644 --- a/reports/urls.py +++ b/reports/urls.py @@ -23,6 +23,7 @@ urlpatterns = [ path('', views.report_list, name='report_list'), + path('bulk_action/', views.report_bulk_action, name='report_bulk_action'), path('upload/', views.upload), path('/', views.report_detail, name='report_detail'), path('/delete/', views.report_delete, name='report_delete'), diff --git a/reports/views.py b/reports/views.py index f247deb2..c46b7a41 100644 --- a/reports/views.py +++ b/reports/views.py @@ -17,21 +17,46 @@ from django.contrib import messages from django.contrib.auth.decorators import login_required -from django.core.paginator import EmptyPage, PageNotAnInteger, Paginator from django.db.models import Q from django.db.utils import OperationalError from django.http import Http404, HttpResponse from django.shortcuts import get_object_or_404, redirect, render from django.urls import reverse from django.views.decorators.csrf import csrf_exempt +from django_tables2 import RequestConfig from tenacity import ( retry, retry_if_exception_type, stop_after_attempt, wait_exponential, ) from reports.models import Report +from reports.tables import ReportTable +from util import sanitize_filter_params from util.filterspecs import Filter, FilterBar +def _get_filtered_reports(filter_params): + """Helper to reconstruct filtered queryset from filter params.""" + from urllib.parse import parse_qs + params = parse_qs(filter_params) + + reports = Report.objects.select_related() + + if 'host_id' in params: + reports = reports.filter(hostname=params['host_id'][0]) + if 'processed' in params: + processed = params['processed'][0] == 'true' + reports = reports.filter(processed=processed) + if 'search' in params: + terms = params['search'][0].lower() + query = Q() + for term in terms.split(' '): + q = Q(host__icontains=term) + query = query & q + reports = reports.filter(query) + + return reports + + @retry( retry=retry_if_exception_type(OperationalError), stop=stop_after_attempt(5), @@ -96,25 +121,27 @@ def report_list(request): else: terms = '' - page_no = request.GET.get('page') - paginator = Paginator(reports, 50) - - try: - page = paginator.page(page_no) - except PageNotAnInteger: - page = paginator.page(1) - except EmptyPage: - page = paginator.page(paginator.num_pages) - filter_list = [] filter_list.append(Filter(request, 'Processed', 'processed', {'true': 'Yes', 'false': 'No'})) filter_bar = FilterBar(request, filter_list) + table = ReportTable(reports) + RequestConfig(request, paginate={'per_page': 50}).configure(table) + + filter_params = sanitize_filter_params(request.GET.urlencode()) + bulk_actions = [ + {'value': 'process', 'label': 'Process'}, + {'value': 'delete', 'label': 'Delete'}, + ] + return render(request, 'reports/report_list.html', - {'page': page, + {'table': table, 'filter_bar': filter_bar, - 'terms': terms}) + 'terms': terms, + 'total_count': reports.count(), + 'filter_params': filter_params, + 'bulk_actions': bulk_actions}) @login_required @@ -158,3 +185,51 @@ def report_delete(request, report_id): return render(request, 'reports/report_delete.html', {'report': report}) + + +@login_required +def report_bulk_action(request): + """Handle bulk actions on reports.""" + if request.method != 'POST': + return redirect('reports:report_list') + + action = request.POST.get('action', '') + select_all_filtered = request.POST.get('select_all_filtered') == '1' + filter_params = request.POST.get('filter_params', '') + + if not action: + messages.warning(request, 'Please select an action') + if filter_params: + return redirect(f"{reverse('reports:report_list')}?{filter_params}") + return redirect('reports:report_list') + + if select_all_filtered: + reports = _get_filtered_reports(filter_params) + else: + selected_ids = request.POST.getlist('selected_ids') + if not selected_ids: + messages.warning(request, 'No reports selected') + if filter_params: + return redirect(f"{reverse('reports:report_list')}?{filter_params}") + return redirect('reports:report_list') + reports = Report.objects.filter(id__in=selected_ids) + + count = reports.count() + name = Report._meta.verbose_name if count == 1 else Report._meta.verbose_name_plural + + if action == 'process': + from reports.tasks import process_report + for report in reports: + report.processed = False + report.save() + process_report.delay(report.id) + messages.success(request, f'Queued {count} {name} for processing') + elif action == 'delete': + reports.delete() + messages.success(request, f'Deleted {count} {name}') + else: + messages.warning(request, 'Invalid action') + + if filter_params: + return redirect(f"{reverse('reports:report_list')}?{filter_params}") + return redirect('reports:report_list') diff --git a/repos/tables.py b/repos/tables.py new file mode 100644 index 00000000..8551822f --- /dev/null +++ b/repos/tables.py @@ -0,0 +1,165 @@ +# This file is part of Patchman. +# +# Patchman is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, version 3 only. +# +# Patchman is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Patchman. If not, see + +import django_tables2 as tables + +from repos.models import Mirror, Repository +from util.tables import BaseTable + +# RepositoryTable templates +CHECKBOX_TEMPLATE = '' +SELECT_ALL_CHECKBOX = '' +REPO_NAME_TEMPLATE = '{{ record }}' +MIRRORS_TEMPLATE = ( + '' + '{{ record.mirror_set.count }}' +) +REPO_ENABLED_TEMPLATE = '{% load common %}{% yes_no_img record.enabled %}' +SECURITY_TEMPLATE = '{% load common %}{% yes_no_img record.security %}' +AUTH_REQUIRED_TEMPLATE = '{% if record.auth_required %}Yes{% else %}No{% endif %}' + +# MirrorTable templates +MIRROR_CHECKBOX_TEMPLATE = '' +MIRROR_ID_TEMPLATE = '{{ record.id }}' +MIRROR_URL_TEMPLATE = '{{ record.url|truncatechars:25 }}' +MIRROR_PACKAGES_TEMPLATE = ( + '{% if not record.mirrorlist %}' + '' + '{{ record.packages.count }}{% endif %}' +) +MIRROR_ENABLED_TEMPLATE = '{% load common %}{% yes_no_img record.enabled %}' +REFRESH_TEMPLATE = '{% load common %}{% yes_no_img record.refresh %}' +MIRRORLIST_TEMPLATE = '{% load common %}{% yes_no_img record.mirrorlist %}' +LAST_ACCESS_OK_TEMPLATE = '{% load common %}{% yes_no_img record.last_access_ok %}' +CHECKSUM_TEMPLATE = '{% if not record.mirrorlist %}{{ record.packages_checksum|truncatechars:16 }}{% endif %}' + + +class RepositoryTable(BaseTable): + selection = tables.TemplateColumn( + CHECKBOX_TEMPLATE, + orderable=False, + verbose_name=SELECT_ALL_CHECKBOX, + attrs={'th': {'class': 'min-width-col centered'}, 'td': {'class': 'min-width-col centered'}}, + ) + repo_name = tables.TemplateColumn( + REPO_NAME_TEMPLATE, + order_by='name', + verbose_name='Repo Name', + attrs={'th': {'class': 'col-sm-4'}, 'td': {'class': 'col-sm-4'}}, + ) + repo_id = tables.Column( + verbose_name='Repo ID', + default='', + attrs={'th': {'class': 'col-sm-3'}, 'td': {'class': 'col-sm-3'}}, + ) + mirrors = tables.TemplateColumn( + MIRRORS_TEMPLATE, + orderable=False, + verbose_name='Mirrors', + attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1 centered'}}, + ) + repo_enabled = tables.TemplateColumn( + REPO_ENABLED_TEMPLATE, + orderable=False, + verbose_name='Enabled', + attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1 centered'}}, + ) + security = tables.TemplateColumn( + SECURITY_TEMPLATE, + orderable=False, + verbose_name='Security', + attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1 centered'}}, + ) + auth_required = tables.TemplateColumn( + AUTH_REQUIRED_TEMPLATE, + orderable=False, + verbose_name='Auth Required', + attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1 centered'}}, + ) + + class Meta(BaseTable.Meta): + model = Repository + fields = ( + 'selection', 'repo_name', 'repo_id', 'mirrors', + 'repo_enabled', 'security', 'auth_required', + ) + + +class MirrorTable(BaseTable): + selection = tables.TemplateColumn( + MIRROR_CHECKBOX_TEMPLATE, + orderable=False, + verbose_name=SELECT_ALL_CHECKBOX, + attrs={'th': {'class': 'min-width-col centered'}, 'td': {'class': 'min-width-col centered'}}, + ) + mirror_id = tables.TemplateColumn( + MIRROR_ID_TEMPLATE, + order_by='id', + verbose_name='ID', + attrs={'th': {'class': 'min-width-col'}, 'td': {'class': 'min-width-col'}}, + ) + mirror_url = tables.TemplateColumn( + MIRROR_URL_TEMPLATE, + orderable=False, + verbose_name='URL', + attrs={'th': {'class': 'col-sm-2'}, 'td': {'class': 'col-sm-2'}}, + ) + mirror_packages = tables.TemplateColumn( + MIRROR_PACKAGES_TEMPLATE, + order_by='packages_count', + verbose_name='Packages', + attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1 centered'}}, + ) + mirror_enabled = tables.TemplateColumn( + MIRROR_ENABLED_TEMPLATE, + orderable=False, + verbose_name='Enabled', + attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1 centered'}}, + ) + refresh = tables.TemplateColumn( + REFRESH_TEMPLATE, + orderable=False, + verbose_name='Refresh', + attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1 centered'}}, + ) + mirrorlist = tables.TemplateColumn( + MIRRORLIST_TEMPLATE, + orderable=False, + verbose_name='Mirrorlist/Metalink', + attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1 centered'}}, + ) + last_access_ok = tables.TemplateColumn( + LAST_ACCESS_OK_TEMPLATE, + orderable=False, + verbose_name='Last Access OK', + attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1 centered'}}, + ) + timestamp = tables.Column( + order_by='timestamp', + verbose_name='Timestamp', + attrs={'th': {'class': 'col-sm-2'}, 'td': {'class': 'col-sm-2'}}, + ) + checksum = tables.TemplateColumn( + CHECKSUM_TEMPLATE, + order_by='packages_checksum', + verbose_name='Checksum', + attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1'}}, + ) + + class Meta(BaseTable.Meta): + model = Mirror + fields = ( + 'selection', 'mirror_id', 'mirror_url', 'mirror_packages', 'mirror_enabled', 'refresh', + 'mirrorlist', 'last_access_ok', 'timestamp', 'checksum', + ) diff --git a/repos/templates/repos/mirror_edit_repo.html b/repos/templates/repos/mirror_edit_repo.html deleted file mode 100644 index 1a785538..00000000 --- a/repos/templates/repos/mirror_edit_repo.html +++ /dev/null @@ -1,33 +0,0 @@ -{% load common %} - - - - - - - - - - - - - - - - - {% for mirror in object_list %} - - - - - - - - - - - - - {% endfor %} - -
    RepoIDURLPackagesEnabledRefreshMirrorlist/MetalinkLast Access OKTimestampChecksum
    {{ mirror.repo }}{{ mirror.id }}{{ mirror.url|truncatechars:25 }}{{ mirror.packages.count }}{% yes_no_img mirror.enabled 'Enabled' 'Not Enabled' %}{% yes_no_img mirror.refresh 'Yes' 'No' %}{% yes_no_img mirror.mirrorlist 'Yes' 'No' %}{% yes_no_img mirror.last_access_ok 'Yes' 'No' %}{{ mirror.timestamp }}{{ mirror.packages_checksum|truncatechars:16 }}
    diff --git a/repos/templates/repos/mirror_list.html b/repos/templates/repos/mirror_list.html index 5f6c32d4..e5143a2b 100644 --- a/repos/templates/repos/mirror_list.html +++ b/repos/templates/repos/mirror_list.html @@ -1,7 +1,26 @@ -{% extends "objectlist.html" %} +{% extends "base.html" %} +{% load django_tables2 common %} {% block page_title %}Mirrors{% endblock %} {% block breadcrumbs %} {{ block.super }}
  • Repositories
  • Mirrors
    • {% endblock %} {% block content_title %} Mirrors {% endblock %} + +{% block content %} +
    +
    + {% get_querydict request as querydict %} + {% searchform terms querydict %} + +
    + {% csrf_token %} + + + {% include "bulk_actions.html" %} + + {% render_table table %} +
    +
    +
    +{% endblock %} diff --git a/repos/templates/repos/mirror_table.html b/repos/templates/repos/mirror_table.html deleted file mode 100644 index e5b40129..00000000 --- a/repos/templates/repos/mirror_table.html +++ /dev/null @@ -1,39 +0,0 @@ -{% load common bootstrap3 %} - - - - - - - - - - - - - - - - - - {% for mirror in object_list %} - - - - - - - - - - - - - - {% endfor %} - -
    IDURLPackagesEnabledRefreshMirrorlist/MetalinkLast Access OKTimestampChecksumDeleteEdit
    {{ mirror.id }}{{ mirror.url|truncatechars:25 }} - {% if not mirror.mirrorlist %} - {{ mirror.packages.count }} - {% endif %} - {% yes_no_img mirror.enabled 'Enabled' 'Not Enabled' %}{% yes_no_img mirror.refresh 'Yes' 'No' %}{% yes_no_img mirror.mirrorlist 'Yes' 'No' %}{% yes_no_img mirror.last_access_ok 'Yes' 'No' %}{{ mirror.timestamp }}{% if not mirror.mirrorlist %}{{ mirror.packages_checksum|truncatechars:16 }}{% endif %}{% bootstrap_icon "trash" %} Delete this Mirror{% bootstrap_icon "edit" %} Edit this Mirror
    diff --git a/repos/templates/repos/mirror_with_repo_list.html b/repos/templates/repos/mirror_with_repo_list.html index ac3b5edf..352ecb3b 100644 --- a/repos/templates/repos/mirror_with_repo_list.html +++ b/repos/templates/repos/mirror_with_repo_list.html @@ -1,10 +1,10 @@ {% extends "repos/mirror_list.html" %} -{% load common bootstrap3 %} +{% load common bootstrap3 django_tables2 %} {% block content %} -{% gen_table page.object_list 'repos/mirror_edit_repo.html' %} +{% render_table table %} {% if user.is_authenticated and perms.is_admin %} {% if link_form and create_form %} diff --git a/repos/templates/repos/repo_list.html b/repos/templates/repos/repo_list.html index e3eab847..f32900b6 100644 --- a/repos/templates/repos/repo_list.html +++ b/repos/templates/repos/repo_list.html @@ -1,7 +1,37 @@ -{% extends "objectlist.html" %} +{% extends "base.html" %} +{% load django_tables2 common %} {% block page_title %}Repositories{% endblock %} {% block breadcrumbs %} {{ block.super }}
  • Repositories
    • {% endblock %} {% block content_title %} Repositories {% endblock %} + +{% block content %} +
    +
    + {% get_querydict request as querydict %} + {% searchform terms querydict %} + +
    + {% csrf_token %} + + + {% include "bulk_actions.html" %} + + {% render_table table %} +
    +
    + + {% if filter_bar %} +
    +
    +
    Filter by...
    +
    + {{ filter_bar|safe }} +
    +
    +
    + {% endif %} +
    +{% endblock %} diff --git a/repos/templates/repos/repository_table.html b/repos/templates/repos/repository_table.html deleted file mode 100644 index bcd7e721..00000000 --- a/repos/templates/repos/repository_table.html +++ /dev/null @@ -1,25 +0,0 @@ -{% load common repo_buttons %} - - - - - - - - - - - - - {% for repo in object_list %} - - - - - - - - - {% endfor %} - -
    Repo NameRepo IDMirrorsEnabledSecurityAuth Required
    {{ repo }}{% if repo.repo_id %} {{ repo.repo_id }} {% endif %} {{ repo.mirror_set.count }}
    {% yes_no_button_repo_en repo %}
    {% yes_no_button_repo_sec repo %}
    		{% yes_no_img repo.auth_required %}
    diff --git a/repos/templatetags/repo_buttons.py b/repos/templatetags/repo_buttons.py deleted file mode 100644 index 3689c8b7..00000000 --- a/repos/templatetags/repo_buttons.py +++ /dev/null @@ -1,54 +0,0 @@ -# Copyright 2012 VPAC, http://www.vpac.org -# Copyright 2013-2021 Marcus Furlong -# -# This file is part of Patchman. -# -# Patchman is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, version 3 only. -# -# Patchman is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Patchman. If not, see - -from django.template import Library -from django.templatetags.static import static -from django.utils.html import format_html - -register = Library() - - -@register.simple_tag -def yes_no_button_repo_en(repo): - - repo_url = repo.get_absolute_url() - yes_icon = static('img/icon-yes.gif') - no_icon = static('img/icon-no.gif') - html = '' - return format_html(html) - - -@register.simple_tag -def yes_no_button_repo_sec(repo): - - repo_url = repo.get_absolute_url() - yes_icon = static('img/icon-yes.gif') - no_icon = static('img/icon-no.gif') - html = '' - return format_html(html) diff --git a/repos/urls.py b/repos/urls.py index 176f9a13..b736e5be 100644 --- a/repos/urls.py +++ b/repos/urls.py @@ -23,6 +23,7 @@ urlpatterns = [ path('', views.repo_list, name='repo_list'), + path('bulk_action/', views.repo_bulk_action, name='repo_bulk_action'), path('/', views.repo_detail, name='repo_detail'), path('/toggle_enabled/', views.repo_toggle_enabled, name='repo_toggle_enabled'), path('/toggle_security/', views.repo_toggle_security, name='repo_toggle_security'), @@ -30,6 +31,7 @@ path('/delete/', views.repo_delete, name='repo_delete'), path('/refresh/', views.repo_refresh, name='repo_refresh'), path('mirrors/', views.mirror_list, name='mirror_list'), + path('mirrors/bulk_action/', views.mirror_bulk_action, name='mirror_bulk_action'), path('mirrors/mirror//', views.mirror_detail, name='mirror_detail'), path('mirrors/mirror//edit/', views.mirror_edit, name='mirror_edit'), path('mirrors/mirror//delete/', views.mirror_delete, name='mirror_delete'), diff --git a/repos/views.py b/repos/views.py index 1f0c2bfa..2a45b668 100644 --- a/repos/views.py +++ b/repos/views.py @@ -17,12 +17,12 @@ from django.contrib import messages from django.contrib.auth.decorators import login_required -from django.core.paginator import EmptyPage, PageNotAnInteger, Paginator from django.db import IntegrityError -from django.db.models import Q +from django.db.models import Count, Q from django.http import HttpResponse from django.shortcuts import get_object_or_404, redirect, render from django.urls import reverse +from django_tables2 import RequestConfig from rest_framework import viewsets from arch.models import MachineArchitecture @@ -35,6 +35,8 @@ from repos.serializers import ( MirrorPackageSerializer, MirrorSerializer, RepositorySerializer, ) +from repos.tables import MirrorTable, RepositoryTable +from util import sanitize_filter_params from util.filterspecs import Filter, FilterBar @@ -75,16 +77,6 @@ def repo_list(request): repos = repos.distinct() - page_no = request.GET.get('page') - paginator = Paginator(repos, 50) - - try: - page = paginator.page(page_no) - except PageNotAnInteger: - page = paginator.page(1) - except EmptyPage: - page = paginator.page(paginator.num_pages) - filter_list = [] filter_list.append(Filter(request, 'OS Release', 'osrelease_id', OSRelease.objects.filter(repos__in=repos))) filter_list.append(Filter(request, 'Enabled', 'enabled', {'true': 'Yes', 'false': 'No'})) @@ -94,11 +86,29 @@ def repo_list(request): MachineArchitecture.objects.filter(repository__in=repos))) filter_bar = FilterBar(request, filter_list) + table = RepositoryTable(repos) + RequestConfig(request, paginate={'per_page': 50}).configure(table) + + # Build filter params string for "select all filtered" option + filter_params = sanitize_filter_params(request.GET.urlencode()) + + bulk_actions = [ + {'value': 'enable', 'label': 'Enable'}, + {'value': 'disable', 'label': 'Disable'}, + {'value': 'mark_security', 'label': 'Mark as Security'}, + {'value': 'mark_non_security', 'label': 'Mark as Non-Security'}, + {'value': 'refresh', 'label': 'Refresh'}, + {'value': 'delete', 'label': 'Delete'}, + ] + return render(request, 'repos/repo_list.html', - {'page': page, + {'table': table, 'filter_bar': filter_bar, - 'terms': terms}) + 'terms': terms, + 'total_count': repos.count(), + 'filter_params': filter_params, + 'bulk_actions': bulk_actions}) @login_required @@ -110,15 +120,19 @@ def pre_reqs(arch, repotype): text = 'Not all mirror architectures are the same,' text += ' cannot link to or create repos' messages.info(request, text) - return render(request, 'repos/mirror_with_repo_list.html', {'page': page, 'checksum': checksum}) + table = MirrorTable(mirrors) + RequestConfig(request, paginate={'per_page': 50}).configure(table) + return render(request, 'repos/mirror_with_repo_list.html', {'table': table, 'checksum': checksum}) if mirror.repo.repotype != repotype: text = 'Not all mirror repotypes are the same,' text += ' cannot link to or create repos' messages.info(request, text) + table = MirrorTable(mirrors) + RequestConfig(request, paginate={'per_page': 50}).configure(table) return render(request, 'repos/mirror_with_repo_list.html', - {'page': page, 'checksum': checksum}) + {'table': table, 'checksum': checksum}) return True def move_mirrors(repo): @@ -135,7 +149,9 @@ def move_mirrors(repo): if oldrepo.mirror_set.count() == 0: oldrepo.delete() - mirrors = Mirror.objects.select_related().order_by('packages_checksum') + mirrors = Mirror.objects.select_related().annotate( + packages_count=Count('packages'), + ).order_by('packages_checksum') checksum = None if 'checksum' in request.GET: @@ -160,16 +176,6 @@ def move_mirrors(repo): mirrors = mirrors.distinct() - page_no = request.GET.get('page') - paginator = Paginator(mirrors, 50) - - try: - page = paginator.page(page_no) - except PageNotAnInteger: - page = paginator.page(1) - except EmptyPage: - page = paginator.page(paginator.num_pages) - if request.method == 'POST': arch = mirrors[0].repo.arch repotype = mirrors[0].repo.repotype @@ -208,15 +214,35 @@ def move_mirrors(repo): else: link_form = LinkRepoForm(prefix='link') create_form = CreateRepoForm(prefix='create') + table = MirrorTable(mirrors) + RequestConfig(request, paginate={'per_page': 50}).configure(table) return render(request, 'repos/mirror_with_repo_list.html', - {'page': page, + {'table': table, 'link_form': link_form, 'create_form': create_form, 'checksum': checksum}) + + table = MirrorTable(mirrors) + RequestConfig(request, paginate={'per_page': 50}).configure(table) + + filter_params = sanitize_filter_params(request.GET.urlencode()) + bulk_actions = [ + {'value': 'edit', 'label': 'Edit'}, + {'value': 'enable', 'label': 'Enable'}, + {'value': 'disable', 'label': 'Disable'}, + {'value': 'enable_refresh', 'label': 'Enable Refresh'}, + {'value': 'disable_refresh', 'label': 'Disable Refresh'}, + {'value': 'delete', 'label': 'Delete'}, + ] + return render(request, 'repos/mirror_list.html', - {'page': page}) + {'table': table, + 'terms': terms, + 'total_count': mirrors.count(), + 'filter_params': filter_params, + 'bulk_actions': bulk_actions}) @login_required @@ -348,7 +374,7 @@ def repo_toggle_enabled(request, repo_id): repo.enabled = True status = 'enabled' repo.save() - if request.is_ajax(): + if request.headers.get('X-Requested-With') == 'XMLHttpRequest': return HttpResponse(status=204) else: text = f'Repository {repo} has been {status}' @@ -367,7 +393,7 @@ def repo_toggle_security(request, repo_id): repo.security = True sectype = 'security' repo.save() - if request.is_ajax(): + if request.headers.get('X-Requested-With') == 'XMLHttpRequest': return HttpResponse(status=204) else: text = f'Repository {repo} has been marked' @@ -388,6 +414,180 @@ def repo_refresh(request, repo_id): return redirect(repo.get_absolute_url()) +def _get_filtered_repos(filter_params): + """Helper to reconstruct filtered queryset from filter params.""" + from urllib.parse import parse_qs + params = parse_qs(filter_params) + + repos = Repository.objects.select_related().order_by('name') + + if 'repotype' in params: + repos = repos.filter(repotype=params['repotype'][0]) + if 'arch_id' in params: + repos = repos.filter(arch=params['arch_id'][0]) + if 'osrelease_id' in params: + repos = repos.filter(osrelease=params['osrelease_id'][0]) + if 'security' in params: + security = params['security'][0] == 'true' + repos = repos.filter(security=security) + if 'enabled' in params: + enabled = params['enabled'][0] == 'true' + repos = repos.filter(enabled=enabled) + if 'package_id' in params: + repos = repos.filter(mirror__packages=params['package_id'][0]) + if 'search' in params: + terms = params['search'][0].lower() + query = Q() + for term in terms.split(' '): + q = Q(name__icontains=term) + query = query & q + repos = repos.filter(query) + + return repos.distinct() + + +@login_required +def repo_bulk_action(request): + """Handle bulk actions on repositories.""" + if request.method != 'POST': + return redirect('repos:repo_list') + + action = request.POST.get('action', '') + select_all_filtered = request.POST.get('select_all_filtered') == '1' + filter_params = request.POST.get('filter_params', '') + + if not action: + messages.warning(request, 'Please select an action') + if filter_params: + return redirect(f"{reverse('repos:repo_list')}?{filter_params}") + return redirect('repos:repo_list') + + if select_all_filtered: + repos = _get_filtered_repos(filter_params) + else: + selected_ids = request.POST.getlist('selected_ids') + if not selected_ids: + messages.warning(request, 'No repositories selected') + if filter_params: + return redirect(f"{reverse('repos:repo_list')}?{filter_params}") + return redirect('repos:repo_list') + repos = Repository.objects.filter(id__in=selected_ids) + + count = repos.count() + name = Repository._meta.verbose_name if count == 1 else Repository._meta.verbose_name_plural + + if action == 'enable': + repos.update(enabled=True) + messages.success(request, f'Enabled {count} {name}') + elif action == 'disable': + repos.update(enabled=False) + messages.success(request, f'Disabled {count} {name}') + elif action == 'mark_security': + repos.update(security=True) + messages.success(request, f'Marked {count} {name} as security') + elif action == 'mark_non_security': + repos.update(security=False) + messages.success(request, f'Marked {count} {name} as non-security') + elif action == 'refresh': + from repos.tasks import refresh_repo + for repo in repos: + refresh_repo.delay(repo.id) + messages.success(request, f'Queued {count} {name} for refresh') + elif action == 'delete': + repos.delete() + messages.success(request, f'Deleted {count} {name}') + else: + messages.warning(request, 'Invalid action') + + # Preserve filter params when redirecting + if filter_params: + return redirect(f"{reverse('repos:repo_list')}?{filter_params}") + return redirect('repos:repo_list') + + +def _get_filtered_mirrors(filter_params): + """Helper to reconstruct filtered queryset from filter params.""" + from urllib.parse import parse_qs + params = parse_qs(filter_params) + + mirrors = Mirror.objects.select_related().order_by('packages_checksum') + + if 'checksum' in params: + mirrors = mirrors.filter(packages_checksum=params['checksum'][0]) + if 'repo_id' in params: + mirrors = mirrors.filter(repo=params['repo_id'][0]) + if 'search' in params: + terms = params['search'][0].lower() + query = Q() + for term in terms.split(' '): + q = Q(url__icontains=term) + query = query & q + mirrors = mirrors.filter(query) + + return mirrors.distinct() + + +@login_required +def mirror_bulk_action(request): + """Handle bulk actions on mirrors.""" + if request.method != 'POST': + return redirect('repos:mirror_list') + + action = request.POST.get('action', '') + select_all_filtered = request.POST.get('select_all_filtered') == '1' + filter_params = request.POST.get('filter_params', '') + + if not action: + messages.warning(request, 'Please select an action') + if filter_params: + return redirect(f"{reverse('repos:mirror_list')}?{filter_params}") + return redirect('repos:mirror_list') + + if select_all_filtered: + mirrors = _get_filtered_mirrors(filter_params) + else: + selected_ids = request.POST.getlist('selected_ids') + if not selected_ids: + messages.warning(request, 'No mirrors selected') + if filter_params: + return redirect(f"{reverse('repos:mirror_list')}?{filter_params}") + return redirect('repos:mirror_list') + mirrors = Mirror.objects.filter(id__in=selected_ids) + + count = mirrors.count() + name = Mirror._meta.verbose_name if count == 1 else Mirror._meta.verbose_name_plural + + if action == 'edit': + if count != 1: + messages.warning(request, 'Please select exactly one mirror to edit') + if filter_params: + return redirect(f"{reverse('repos:mirror_list')}?{filter_params}") + return redirect('repos:mirror_list') + mirror = mirrors.first() + return redirect('repos:mirror_edit', mirror_id=mirror.id) + elif action == 'enable': + mirrors.update(enabled=True) + messages.success(request, f'Enabled {count} {name}') + elif action == 'disable': + mirrors.update(enabled=False) + messages.success(request, f'Disabled {count} {name}') + elif action == 'enable_refresh': + mirrors.update(refresh=True) + messages.success(request, f'Enabled refresh for {count} {name}') + elif action == 'disable_refresh': + mirrors.update(refresh=False) + messages.success(request, f'Disabled refresh for {count} {name}') + elif action == 'delete': + mirrors.delete() + messages.success(request, f'Deleted {count} {name}') + else: + messages.warning(request, 'Invalid action') + + if filter_params: + return redirect(f"{reverse('repos:mirror_list')}?{filter_params}") + return redirect('repos:mirror_list') + + class RepositoryViewSet(viewsets.ModelViewSet): """ API endpoint that allows repositories to be viewed or edited. diff --git a/requirements.txt b/requirements.txt index 08ce4573..92e421e1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,3 +20,4 @@ django-celery-beat==2.7.0 tqdm==4.67.1 cvss==3.4 zstandard==0.25.0 +django-tables2==2.8.0 diff --git a/security/tables.py b/security/tables.py new file mode 100644 index 00000000..4d7272af --- /dev/null +++ b/security/tables.py @@ -0,0 +1,183 @@ +# This file is part of Patchman. +# +# Patchman is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, version 3 only. +# +# Patchman is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Patchman. If not, see + +import django_tables2 as tables + +from security.models import CVE, CWE, Reference +from util.tables import BaseTable + +# CVETable templates +CVE_ID_TEMPLATE = '{{ record.cve_id }}' +CVE_LINKS_TEMPLATE = ( + '{% load bootstrap3 %}' + 'NIST {% bootstrap_icon "link" %}' + '  ' + 'MITRE {% bootstrap_icon "link" %}' + '  ' + 'osv.dev {% bootstrap_icon "link" %}' +) +CVE_DESCRIPTION_TEMPLATE = ( + '' + '{{ record.description|truncatechars:60 }}' +) +CVSS_SCORES_TEMPLATE = '{% for score in record.cvss_scores.all %} {{ score.score }} {% endfor %}' +CWES_TEMPLATE = '{% for cwe in record.cwes.all %} {{ cwe.cwe_id }} {% endfor %}' +CVE_ERRATA_TEMPLATE = ( + '' + '{{ record.erratum_set.count }}' +) + +# CWETable templates +CWE_ID_TEMPLATE = '{{ record.cwe_id }}' +CWE_DESCRIPTION_TEMPLATE = ( + '' + '{{ record.description|truncatechars:120 }}' +) +CWE_CVES_TEMPLATE = ( + '' + '{{ record.cve_set.count }}' +) + +# ReferenceTable templates +REFERENCE_URL_TEMPLATE = '{{ record.url }}' +LINKED_ERRATA_TEMPLATE = ( + '' + '{{ record.erratum_set.count }}' +) + + +class CVETable(BaseTable): + cve_id = tables.TemplateColumn( + CVE_ID_TEMPLATE, + order_by='cve_id', + verbose_name='CVE ID', + attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1'}}, + ) + links = tables.TemplateColumn( + CVE_LINKS_TEMPLATE, + orderable=False, + verbose_name='Links', + attrs={'th': {'class': 'col-sm-2'}, 'td': {'class': 'col-sm-2'}}, + ) + cve_description = tables.TemplateColumn( + CVE_DESCRIPTION_TEMPLATE, + orderable=False, + verbose_name='Description', + attrs={'th': {'class': 'col-sm-3'}, 'td': {'class': 'col-sm-3'}}, + ) + cvss_scores = tables.TemplateColumn( + CVSS_SCORES_TEMPLATE, + orderable=False, + verbose_name='CVSS Scores', + attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1'}}, + ) + cwes = tables.TemplateColumn( + CWES_TEMPLATE, + orderable=False, + verbose_name='CWEs', + attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1'}}, + ) + reserved_date = tables.DateColumn( + order_by='reserved_date', + verbose_name='Reserved', + default='', + attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1'}}, + ) + rejected_date = tables.DateColumn( + order_by='rejected_date', + verbose_name='Rejected', + default='', + attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1'}}, + ) + published_date = tables.DateColumn( + order_by='published_date', + verbose_name='Published', + default='', + attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1'}}, + ) + updated_date = tables.DateColumn( + order_by='updated_date', + verbose_name='Updated', + default='', + attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1'}}, + ) + cve_errata = tables.TemplateColumn( + CVE_ERRATA_TEMPLATE, + orderable=False, + verbose_name='Errata', + attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1'}}, + ) + + class Meta(BaseTable.Meta): + model = CVE + fields = ( + 'cve_id', 'links', 'cve_description', 'cvss_scores', 'cwes', + 'reserved_date', 'rejected_date', 'published_date', 'updated_date', 'cve_errata', + ) + + +class CWETable(BaseTable): + cwe_id = tables.TemplateColumn( + CWE_ID_TEMPLATE, + order_by='cwe_id', + verbose_name='CWE ID', + attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1'}}, + ) + cwe_name = tables.Column( + accessor='name', + order_by='name', + verbose_name='Name', + default='', + attrs={'th': {'class': 'col-sm-4'}, 'td': {'class': 'col-sm-4'}}, + ) + cwe_description = tables.TemplateColumn( + CWE_DESCRIPTION_TEMPLATE, + orderable=False, + verbose_name='Description', + attrs={'th': {'class': 'col-sm-6'}, 'td': {'class': 'col-sm-6'}}, + ) + cwe_cves = tables.TemplateColumn( + CWE_CVES_TEMPLATE, + orderable=False, + verbose_name='CVEs', + attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1'}}, + ) + + class Meta(BaseTable.Meta): + model = CWE + fields = ('cwe_id', 'cwe_name', 'cwe_description', 'cwe_cves') + + +class ReferenceTable(BaseTable): + ref_type = tables.Column( + order_by='ref_type', + verbose_name='Type', + attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1'}}, + ) + reference_url = tables.TemplateColumn( + REFERENCE_URL_TEMPLATE, + orderable=False, + verbose_name='URL', + attrs={'th': {'class': 'col-sm-10'}, 'td': {'class': 'col-sm-10'}}, + ) + linked_errata = tables.TemplateColumn( + LINKED_ERRATA_TEMPLATE, + orderable=False, + verbose_name='Linked Errata', + attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1'}}, + ) + + class Meta(BaseTable.Meta): + model = Reference + fields = ('ref_type', 'reference_url', 'linked_errata') diff --git a/security/templates/security/cve_table.html b/security/templates/security/cve_table.html deleted file mode 100644 index 63347b1e..00000000 --- a/security/templates/security/cve_table.html +++ /dev/null @@ -1,37 +0,0 @@ -{% load common bootstrap3 %} - - - - - - - - - - - - - - - - - {% for cve in object_list %} - - - - - - - - - - - - - {% endfor %} - -
    CVE IDLinksDescriptionCVSS ScoresCWEsReservedRejectedPublishedUpdatedErrata
    {{ cve.cve_id }} - NIST {% bootstrap_icon "link" %}   - MITRE {% bootstrap_icon "link" %}   - osv.dev {% bootstrap_icon "link" %} - {{ cve.description|truncatechars:60 }}{% for score in cve.cvss_scores.all %} {{ score.score }} {% endfor %}{% for cwe in cve.cwes.all %} {{ cwe.cwe_id }} {% endfor %}{{ cve.reserved_date|date|default_if_none:'' }}{{ cve.rejected_date|date|default_if_none:'' }}{{ cve.published_date|date|default_if_none:'' }}{{ cve.updated_date|date|default_if_none:'' }}{{ cve.erratum_set.count }}
    diff --git a/security/templates/security/cwe_table.html b/security/templates/security/cwe_table.html deleted file mode 100644 index 85ccd118..00000000 --- a/security/templates/security/cwe_table.html +++ /dev/null @@ -1,21 +0,0 @@ -{% load common %} - - - - - - - - - - - {% for cwe in object_list %} - - - - - - - {% endfor %} - -
    CWE IDNameDescriptionCVEs
    {{ cwe.cwe_id }}{{ cwe.name }}{{ cwe.description|truncatechars:120 }}{{ cwe.cve_set.count }}
    diff --git a/security/templates/security/reference_table.html b/security/templates/security/reference_table.html deleted file mode 100644 index a28ff719..00000000 --- a/security/templates/security/reference_table.html +++ /dev/null @@ -1,19 +0,0 @@ -{% load common %} - - - - - - - - - - {% for eref in object_list %} - - - - - - {% endfor %} - -
    TypeURLLinked Errata
    {{ eref.ref_type }}{{ eref.url }}{{ eref.erratum_set.count }}
    diff --git a/security/views.py b/security/views.py index c9e606a6..ae56a82b 100644 --- a/security/views.py +++ b/security/views.py @@ -15,9 +15,9 @@ # along with Patchman. If not, see from django.contrib.auth.decorators import login_required -from django.core.paginator import EmptyPage, PageNotAnInteger, Paginator from django.db.models import Q from django.shortcuts import get_object_or_404, render +from django_tables2 import RequestConfig from rest_framework import viewsets from operatingsystems.models import OSRelease @@ -26,6 +26,7 @@ from security.serializers import ( CVESerializer, CWESerializer, ReferenceSerializer, ) +from security.tables import CVETable, CWETable, ReferenceTable from util.filterspecs import Filter, FilterBar @@ -45,19 +46,12 @@ def cwe_list(request): else: terms = '' - page_no = request.GET.get('page') - paginator = Paginator(cwes, 50) - - try: - page = paginator.page(page_no) - except PageNotAnInteger: - page = paginator.page(1) - except EmptyPage: - page = paginator.page(paginator.num_pages) + table = CWETable(cwes) + RequestConfig(request, paginate={'per_page': 50}).configure(table) return render(request, 'security/cwe_list.html', - {'page': page, + {'table': table, 'terms': terms}) @@ -95,19 +89,12 @@ def cve_list(request): else: terms = '' - page_no = request.GET.get('page') - paginator = Paginator(cves, 50) - - try: - page = paginator.page(page_no) - except PageNotAnInteger: - page = paginator.page(1) - except EmptyPage: - page = paginator.page(paginator.num_pages) + table = CVETable(cves) + RequestConfig(request, paginate={'per_page': 50}).configure(table) return render(request, 'security/cve_list.html', - {'page': page, + {'table': table, 'terms': terms}) @@ -148,24 +135,17 @@ def reference_list(request): else: terms = '' - page_no = request.GET.get('page') - paginator = Paginator(refs, 50) - - try: - page = paginator.page(page_no) - except PageNotAnInteger: - page = paginator.page(1) - except EmptyPage: - page = paginator.page(paginator.num_pages) - filter_list = [] filter_list.append(Filter(request, 'Reference Type', 'ref_type', Reference.objects.values_list('ref_type', flat=True).distinct())) filter_bar = FilterBar(request, filter_list) + table = ReferenceTable(refs) + RequestConfig(request, paginate={'per_page': 50}).configure(table) + return render(request, 'security/reference_list.html', - {'page': page, + {'table': table, 'filter_bar': filter_bar, 'terms': terms}) diff --git a/setup.cfg b/setup.cfg index b1d5ee4e..a523584d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -9,6 +9,7 @@ requires = /usr/bin/python3 python3-django-bootstrap3 python3-django-rest-framework python3-django-filter + python3-django-tables2 python3-debian python3-rpm python3-tqdm diff --git a/util/__init__.py b/util/__init__.py index b85e5e37..4261804d 100644 --- a/util/__init__.py +++ b/util/__init__.py @@ -33,6 +33,7 @@ from enum import Enum from hashlib import md5, sha1, sha256, sha512 from time import time +from urllib.parse import parse_qs, urlencode from django.conf import settings from django.utils.dateparse import parse_datetime @@ -59,6 +60,14 @@ } +def sanitize_filter_params(filter_params): + """Sanitize filter_params to prevent query string injection.""" + if not filter_params: + return '' + parsed = parse_qs(filter_params) + return urlencode(parsed, doseq=True) + + def fetch_content(response, text='', ljust=35): """ Display a progress bar to fetch the request content if verbose is True. Otherwise, just return the request content diff --git a/util/context_processors.py b/util/context_processors.py new file mode 100644 index 00000000..f26c2748 --- /dev/null +++ b/util/context_processors.py @@ -0,0 +1,105 @@ +# Copyright 2013-2025 Marcus Furlong +# +# This file is part of Patchman. +# +# Patchman is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, version 3 only. +# +# Patchman is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Patchman. If not, see + +from datetime import datetime, timedelta + +from django.db.models import F + +from hosts.models import Host +from operatingsystems.models import OSRelease, OSVariant +from reports.models import Report +from repos.models import Repository +from util import get_setting_of_type + + +def issues_count(request): + """Context processor to provide issues count for navbar.""" + if not request.user.is_authenticated: + return {'issues_count': 0} + + hosts = Host.objects.all() + osvariants = OSVariant.objects.all() + osreleases = OSRelease.objects.all() + repos = Repository.objects.all() + + # host issues + days = get_setting_of_type( + setting_name='DAYS_WITHOUT_REPORT', + setting_type=int, + default=14, + ) + last_report_delta = datetime.now() - timedelta(days=days) + stale_hosts = hosts.filter(lastreport__lt=last_report_delta) + norepo_hosts = hosts.filter(repos__isnull=True, osvariant__osrelease__repos__isnull=True) + reboot_hosts = hosts.filter(reboot_required=True) + secupdate_hosts = hosts.filter(updates__security=True, updates__isnull=False).distinct() + bugupdate_hosts = hosts.exclude( + updates__security=True, updates__isnull=False + ).distinct().filter( + updates__security=False, updates__isnull=False + ).distinct() + diff_rdns_hosts = hosts.exclude(reversedns=F('hostname')).filter(check_dns=True) + + # os variant issues + noosrelease_osvariants = osvariants.filter(osrelease__isnull=True) + nohost_osvariants = osvariants.filter(host__isnull=True) + + # os release issues + norepo_osreleases_count = 0 + if hosts.filter(host_repos_only=False).exists(): + norepo_osreleases_count = osreleases.filter(repos__isnull=True).count() + + # mirror issues + failed_mirrors = repos.filter( + auth_required=False, mirror__last_access_ok=False + ).filter(mirror__last_access_ok=True).distinct() + disabled_mirrors = repos.filter( + auth_required=False, mirror__enabled=False, mirror__mirrorlist=False + ).distinct() + norefresh_mirrors = repos.filter(auth_required=False, mirror__refresh=False).distinct() + + # repo issues + failed_repos = repos.filter( + auth_required=False, mirror__last_access_ok=False + ).exclude(id__in=[x.id for x in failed_mirrors]).distinct() + unused_repos = repos.filter(host__isnull=True, osrelease__isnull=True) + nomirror_repos = repos.filter(mirror__isnull=True) + nohost_repos = repos.filter(host__isnull=True) + + # report issues + unprocessed_reports = Report.objects.filter(processed=False) + + count = ( + (1 if stale_hosts.exists() else 0) + + (1 if norepo_hosts.exists() else 0) + + (1 if reboot_hosts.exists() else 0) + + (1 if secupdate_hosts.exists() else 0) + + (1 if bugupdate_hosts.exists() else 0) + + (1 if diff_rdns_hosts.exists() else 0) + + (1 if noosrelease_osvariants.exists() else 0) + + (1 if nohost_osvariants.exists() else 0) + + (1 if norepo_osreleases_count > 0 else 0) + + (1 if failed_mirrors.exists() else 0) + + (1 if disabled_mirrors.exists() else 0) + + (1 if norefresh_mirrors.exists() else 0) + + (1 if failed_repos.exists() else 0) + + (1 if unused_repos.exists() else 0) + + (1 if nomirror_repos.exists() else 0) + + (1 if nohost_repos.exists() else 0) + + (1 if unprocessed_reports.exists() else 0) + ) + + return {'issues_count': count} diff --git a/util/tables.py b/util/tables.py new file mode 100644 index 00000000..f755bb4b --- /dev/null +++ b/util/tables.py @@ -0,0 +1,26 @@ +# This file is part of Patchman. +# +# Patchman is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, version 3 only. +# +# Patchman is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Patchman. If not, see + +import django_tables2 as tables + + +class BaseTable(tables.Table): + """Base table class with common settings for all patchman tables.""" + + class Meta: + abstract = True + template_name = 'table.html' + attrs = { + "class": "table table-striped table-bordered table-hover table-condensed table-responsive", + } diff --git a/util/templates/base.html b/util/templates/base.html index d732263c..17e952f3 100644 --- a/util/templates/base.html +++ b/util/templates/base.html @@ -8,8 +8,6 @@ {% load static %} {% block page_title %}{% endblock %} - - {% block extrahead %}{% endblock %} diff --git a/util/templates/bulk_actions.html b/util/templates/bulk_actions.html new file mode 100644 index 00000000..5dea6de9 --- /dev/null +++ b/util/templates/bulk_actions.html @@ -0,0 +1,90 @@ +{% load bootstrap3 %} +{% load common %} +{# Include this in list templates that support bulk actions #} +{# Required context: table, total_count, filter_params, bulk_actions (list of dicts with 'value' and 'label') #} + +
    +
    +
    + + +
    +
    + +
    +
    + + +
    +
    +
    + + diff --git a/util/templates/dashboard.html b/util/templates/dashboard.html index 631fea36..b1dea384 100644 --- a/util/templates/dashboard.html +++ b/util/templates/dashboard.html @@ -4,7 +4,7 @@ {% block page_title %}Patchman Dashboard{% endblock %} -{% block content_title %} Patch Management Dashboard for {{ site.name }} {% endblock %} +{% block content_title %} Issues Dashboard for {{ site.name }} {% endblock %} {% block content %} diff --git a/util/templates/navbar.html b/util/templates/navbar.html index 2a2edc0b..206f09fd 100644 --- a/util/templates/navbar.html +++ b/util/templates/navbar.html @@ -10,31 +10,46 @@ diff --git a/util/templates/objectlist.html b/util/templates/objectlist.html index f2b4fcf9..7d0601e7 100644 --- a/util/templates/objectlist.html +++ b/util/templates/objectlist.html @@ -1,24 +1,14 @@ {% extends "base.html" %} -{% load common bootstrap3 static %} +{% load common bootstrap3 static django_tables2 %} {% block content %}
    -
    +
    {% get_querydict request as querydict %} {% searchform terms querydict %} - {% gen_table page.object_list table_template %} -
    - {% object_count page %} -
    -
    - {% get_querystring request as querystring %} - {% bootstrap_pagination page size='small' extra=querystring %} -
    -
    - Page {{ page.number }} of {{ page.paginator.num_pages }} -
    + {% render_table table %}
    {% if filter_bar %} diff --git a/util/templates/table.html b/util/templates/table.html new file mode 100644 index 00000000..6084c681 --- /dev/null +++ b/util/templates/table.html @@ -0,0 +1,118 @@ +{% load django_tables2 %} +{% load i18n l10n %} +{% load common %} +{% block table-wrapper %} +
    + {% block table %} + + {% block table.thead %} + {% if table.show_header %} + + + {% for column in table.columns %} + + {% endfor %} + + + {% endif %} + {% endblock table.thead %} + {% block table.tbody %} + + {% for row in table.paginated_rows %} + {% block table.tbody.row %} + + {% for column, cell in row.items %} + + {% endfor %} + + {% endblock table.tbody.row %} + {% empty %} + {% if table.empty_text %} + {% block table.tbody.empty_text %} + + {% endblock table.tbody.empty_text %} + {% endif %} + {% endfor %} + + {% endblock table.tbody %} + {% block table.tfoot %} + {% if table.has_footer %} + + + {% for column in table.columns %} + + {% endfor %} + + + {% endif %} + {% endblock table.tfoot %} +
    + {% if column.orderable %} + {{ column.header }}{% if column.is_ordered %}{% if column.order_by_alias.is_descending %} {% else %} {% endif %}{% endif %} + {% else %} + {{ column.header|safe }} + {% endif %} +
    {% if column.localize == None %}{{ cell }}{% else %}{% if column.localize %}{{ cell|localize }}{% else %}{{ cell|unlocalize }}{% endif %}{% endif %}
    {{ table.empty_text }}
    {{ column.footer }}
    + {% endblock table %} + + {% block object_count %} + {% if table.page %} +
    + {% object_count table %} +
    + {% endif %} + {% endblock object_count %} + + {% block pagination %} + {% if table.page %} +
    + +
    + {% endif %} + {% endblock pagination %} + + {% block page_info %} + {% if table.page %} +
    + Page {{ table.page.number }} of {{ table.paginator.num_pages }} +
    + {% endif %} + {% endblock page_info %} +
    +{% endblock table-wrapper %} diff --git a/util/templatetags/common.py b/util/templatetags/common.py index 674e1721..38d478d6 100644 --- a/util/templatetags/common.py +++ b/util/templatetags/common.py @@ -14,15 +14,14 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -import re +import importlib from datetime import datetime, timedelta from urllib.parse import urlencode -from django.core.paginator import Paginator from django.template import Library from django.template.loader import get_template -from django.templatetags.static import static from django.utils.html import format_html +from django_tables2 import RequestConfig from humanize import naturaltime from util import get_setting_of_type @@ -30,56 +29,75 @@ register = Library() -@register.simple_tag -def active(request, pattern): - if re.search(fr"^{request.META['SCRIPT_NAME']}/{pattern}", request.path): - return 'active' - return '' - - @register.simple_tag def yes_no_img(boolean, alt_yes='Active', alt_no='Not Active'): - yes_icon = static('img/icon-yes.gif') - no_icon = static('img/icon-no.gif') if boolean: - html = f'{alt_yes}' + html = f'' else: - html = f'{alt_no}' + html = f'' return format_html(html) @register.simple_tag def no_yes_img(boolean, alt_yes='Not Required', alt_no='Required'): - yes_icon = static('img/icon-yes.gif') - no_icon = static('img/icon-no.gif') if not boolean: - html = f'{alt_yes}' + html = f'' else: - html = f'{alt_no}' + html = f'' return format_html(html) -@register.simple_tag -def gen_table(object_list, template_name=None): +@register.simple_tag(takes_context=True) +def gen_table(context, object_list, template_name=None): + """Generate a django-tables2 table for non-paginated contexts (e.g., dashboard).""" if not object_list: return '' - if not template_name: - app_label = object_list.model._meta.app_label - model_name = object_list.model._meta.verbose_name.replace(' ', '') - template_name = f'{app_label}/{model_name.lower()}_table.html' - template = get_template(template_name) - html = template.render({'object_list': object_list}) - return html + + request = context.get('request') + + app_label = object_list.model._meta.app_label + model_name = object_list.model.__name__ + + app_mod = importlib.import_module(f"{app_label}.tables") + TableClass = getattr(app_mod, f"{model_name}Table") + + table = TableClass(object_list) + + # Exclude selection column for embedded tables (dashboard, detail pages) + if 'selection' in table.columns: + table.columns.hide('selection') + + # No pagination for dashboard/detail page tables + if request: + RequestConfig(request, paginate=False).configure(table) + + # Render using the table's configured template + from django.template import engines + django_engine = engines['django'] + template = django_engine.from_string('{% load django_tables2 %}{% render_table table %}') + return template.render({'table': table, 'request': request}) @register.simple_tag -def object_count(page): - if isinstance(page.paginator, Paginator): - if page.paginator.count == 1: - name = page.paginator.object_list.model._meta.verbose_name +def object_count(table): + """Return object count string for django-tables2 table.""" + if hasattr(table, 'paginator') and table.paginator: + count = table.paginator.count + if count == 1: + name = table.data.data.model._meta.verbose_name.title() else: - name = page.paginator.object_list.model._meta.verbose_name_plural - return f'{page.paginator.count} {name}' + name = table.data.data.model._meta.verbose_name_plural.title() + return f'{count} {name}' + return '' + + +@register.filter +def verbose_name_plural(table): + """Return the verbose_name_plural from a django-tables2 table's model.""" + try: + return table.data.data.model._meta.verbose_name_plural.title() + except AttributeError: + return '' @register.simple_tag diff --git a/util/urls.py b/util/urls.py index ea0b77fc..f5ceaed3 100644 --- a/util/urls.py +++ b/util/urls.py @@ -23,6 +23,6 @@ app_name = 'util' urlpatterns = [ - path('', RedirectView.as_view(pattern_name='util:dashboard', permanent=True)), # noqa + path('', RedirectView.as_view(pattern_name='hosts:host_list', permanent=True)), # noqa path('dashboard/', views.dashboard, name='dashboard'), ] From cd173bc30f4be2b72bf0206d2e82a2e87421404a Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Tue, 13 Jan 2026 19:02:03 -0500 Subject: [PATCH 045/146] add django2-select2 functionality --- debian/control | 2 +- patchman/settings.py | 1 + patchman/static/css/base.css | 1 + patchman/urls.py | 1 + repos/forms.py | 25 +++++++++++++++++-------- repos/templates/repos/repo_edit.html | 4 ---- requirements.txt | 1 + setup.cfg | 1 + 8 files changed, 23 insertions(+), 13 deletions(-) diff --git a/debian/control b/debian/control index cd19139c..34610549 100644 --- a/debian/control +++ b/debian/control @@ -21,7 +21,7 @@ Depends: ${misc:Depends}, python3 (>= 3.11), python3-django (>= 4.2), python3-yaml, libapache2-mod-wsgi-py3, apache2, sqlite3, celery, python3-celery, python3-django-celery-beat, redis-server, python3-redis, python3-git, python3-django-taggit, python3-zstandard, - python3-django-tables2 + python3-django-tables2, python3-django-select2 Suggests: python3-mysqldb, python3-psycopg2, python3-pymemcache, memcached Description: Django-based patch status monitoring tool for linux systems. . diff --git a/patchman/settings.py b/patchman/settings.py index 1553b247..76c28b3f 100644 --- a/patchman/settings.py +++ b/patchman/settings.py @@ -81,6 +81,7 @@ 'taggit', 'bootstrap3', 'django_tables2', + 'django_select2', 'rest_framework', 'django_filters', 'celery', diff --git a/patchman/static/css/base.css b/patchman/static/css/base.css index 0db76909..45ed77c7 100644 --- a/patchman/static/css/base.css +++ b/patchman/static/css/base.css @@ -16,6 +16,7 @@ .centered { text-align: center; } th.min-width-col, td.min-width-col { width: 1%; white-space: nowrap; padding: 5px !important; } .table td { vertical-align: bottom !important; } +.select2-results__options { max-height: 400px !important; } /* Center pagination controls produced by django-tables2 without centering table cell contents */ .django-tables2 .pagination { diff --git a/patchman/urls.py b/patchman/urls.py index 2b9a9787..b66a0de3 100644 --- a/patchman/urls.py +++ b/patchman/urls.py @@ -56,6 +56,7 @@ path('admin/', admin.site.urls), path('api/', include(router.urls)), path('api-auth/', include('rest_framework.urls', namespace='rest_framework')), # noqa + path('select2/', include('django_select2.urls')), path('', include('util.urls', namespace='util')), path('errata/', include('errata.urls', namespace='errata')), path('reports/', include('reports.urls', namespace='reports')), diff --git a/repos/forms.py b/repos/forms.py index 9cb66897..d776cd00 100644 --- a/repos/forms.py +++ b/repos/forms.py @@ -15,26 +15,35 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -from django.contrib.admin.widgets import FilteredSelectMultiple from django.forms import ( Form, ModelChoiceField, ModelForm, ModelMultipleChoiceField, TextInput, ValidationError, ) +from django_select2.forms import ModelSelect2MultipleWidget from repos.models import Mirror, Repository -class EditRepoForm(ModelForm): - class Media: - css = { - 'all': ('admin/css/widgets.css',) - } +class MirrorSelect2Widget(ModelSelect2MultipleWidget): + model = Mirror + search_fields = ['url__icontains', 'repo__name__icontains'] + max_results = 50 + def __init__(self, *args, **kwargs): + kwargs.setdefault('attrs', {}) + kwargs['attrs'].setdefault('data-minimum-input-length', 0) + super().__init__(*args, **kwargs) + + def label_from_instance(self, obj): + return f"{obj.repo.name} - {obj.url}" + + +class EditRepoForm(ModelForm): mirrors = ModelMultipleChoiceField( queryset=Mirror.objects.select_related(), required=False, - label=None, - widget=FilteredSelectMultiple('Mirrors', is_stacked=False)) + widget=MirrorSelect2Widget(attrs={'style': 'width: 100%'}), + ) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) diff --git a/repos/templates/repos/repo_edit.html b/repos/templates/repos/repo_edit.html index 0a160d8e..5a968896 100644 --- a/repos/templates/repos/repo_edit.html +++ b/repos/templates/repos/repo_edit.html @@ -3,10 +3,6 @@ {% load common bootstrap3 static %} {% block extrahead %} - - - - {{ edit_form.media }} {% endblock %} diff --git a/requirements.txt b/requirements.txt index 92e421e1..1bad175b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,3 +21,4 @@ tqdm==4.67.1 cvss==3.4 zstandard==0.25.0 django-tables2==2.8.0 +django-select2==8.3.0 diff --git a/setup.cfg b/setup.cfg index a523584d..48415b86 100644 --- a/setup.cfg +++ b/setup.cfg @@ -10,6 +10,7 @@ requires = /usr/bin/python3 python3-django-rest-framework python3-django-filter python3-django-tables2 + python3-django-select2 python3-debian python3-rpm python3-tqdm From 83f96f3995a48e145179ee1b893940b46c631089 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Tue, 13 Jan 2026 19:16:31 -0500 Subject: [PATCH 046/146] update host and package templates --- errata/templates/errata/erratum_detail.html | 16 ++-- hosts/tables.py | 18 ++++- hosts/templates/hosts/host_delete.html | 15 ++-- hosts/templates/hosts/host_detail.html | 85 ++++++++++++--------- hosts/views.py | 18 ++++- patchman/static/css/base.css | 6 +- repos/forms.py | 9 +-- security/templates/security/cve_detail.html | 20 ++--- util/context_processors.py | 5 +- util/templates/dashboard.html | 28 ++++--- 10 files changed, 123 insertions(+), 97 deletions(-) diff --git a/errata/templates/errata/erratum_detail.html b/errata/templates/errata/erratum_detail.html index 4738154e..2dbfceaf 100644 --- a/errata/templates/errata/erratum_detail.html +++ b/errata/templates/errata/erratum_detail.html @@ -61,24 +61,20 @@
    -
    +
    +
    -
    +
    +
    diff --git a/hosts/tables.py b/hosts/tables.py index 835d8195..e527af7b 100644 --- a/hosts/tables.py +++ b/hosts/tables.py @@ -41,6 +41,10 @@ '{{ record.osvariant }}' '{% endif %}' ) +PACKAGES_TEMPLATE = ( + '' + '{{ record.packages_count }}' +) LASTREPORT_TEMPLATE = ( '{% load report_alert %}' '{{ record.lastreport }} {% report_alert record.lastreport %}' @@ -87,13 +91,19 @@ class HostTable(BaseTable): OSVARIANT_TEMPLATE, order_by='osvariant__name', verbose_name='OS Variant', - attrs={'th': {'class': 'col-sm-2'}, 'td': {'class': 'col-sm-2'}}, + attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1'}}, + ) + packages_installed = tables.TemplateColumn( + PACKAGES_TEMPLATE, + order_by='packages_count', + verbose_name='Packages', + attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1 centered'}}, ) lastreport = tables.TemplateColumn( LASTREPORT_TEMPLATE, order_by='lastreport', verbose_name='Last Report', - attrs={'th': {'class': 'col-sm-2'}, 'td': {'class': 'col-sm-2'}}, + attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1'}}, ) reboot_required = tables.TemplateColumn( REBOOT_TEMPLATE, @@ -105,6 +115,6 @@ class HostTable(BaseTable): class Meta(BaseTable.Meta): model = Host fields = ( - 'selection', 'hostname', 'sec_updates', 'bug_updates', 'affected_errata', - 'kernel', 'osvariant', 'lastreport', 'reboot_required', + 'selection', 'hostname', 'packages_installed', 'sec_updates', 'bug_updates', + 'affected_errata', 'kernel', 'osvariant', 'lastreport', 'reboot_required', ) diff --git a/hosts/templates/hosts/host_delete.html b/hosts/templates/hosts/host_delete.html index 5f37d8ab..2b665f24 100644 --- a/hosts/templates/hosts/host_delete.html +++ b/hosts/templates/hosts/host_delete.html @@ -30,18 +30,19 @@ Updated {{ host.updated_at }} Last Report {{ host.lastreport }} - Updates Available {{ host.updates.count }} + Packages Installed {{ host.packages.count}} + Updates Available {{ host.updates.count }} + Errata{{ host.errata.count }} Reboot Required {{ host.reboot_required }} - Packages Installed {{ host.packages.count}} Repos In Use{% if host.host_repos_only %}Host Repos{% else %}Host and OS Release Repos{% endif %} Last 3 reports - {% for report in reports %} - - {{ report.created }} - - {% endfor %} + diff --git a/hosts/templates/hosts/host_detail.html b/hosts/templates/hosts/host_detail.html index f12bf22b..01a5c546 100644 --- a/hosts/templates/hosts/host_detail.html +++ b/hosts/templates/hosts/host_detail.html @@ -12,10 +12,9 @@
    @@ -41,18 +40,18 @@ Updated {{ host.updated_at }} Last Report {{ host.lastreport }} Packages Installed {{ host.packages.count}} - Updates Available {{ host.updates.count }} + Updates Available {{ host.updates.count }} Errata{{ host.errata.count }} Reboot Required {{ host.reboot_required }} Repos In Use{% if host.host_repos_only %}Host Repos{% else %}Host and OS Release Repos{% endif %} Last 3 reports - {% for report in reports %} - - {{ report.created }} - - {% endfor %} + @@ -64,30 +63,38 @@
    -
    +
    - +
    + +
    +
    - + - {% for update in host.updates.select_related %} - - + {% endfor %} @@ -141,21 +148,27 @@ {% gen_table host.modules.all %} - -
    -
    - -
    -
    - {% for package in host.packages.select_related %} - - {{ package }} - - {% endfor %} -
    -
    -
    -
    + + {% endblock %} diff --git a/hosts/views.py b/hosts/views.py index 7d969888..b50f021c 100644 --- a/hosts/views.py +++ b/hosts/views.py @@ -76,9 +76,10 @@ def _get_filtered_hosts(filter_params): @login_required def host_list(request): hosts = Host.objects.select_related().annotate( - sec_updates_count=Count('updates', filter=Q(updates__security=True)), - bug_updates_count=Count('updates', filter=Q(updates__security=False)), - errata_count=Count('errata'), + sec_updates_count=Count('updates', filter=Q(updates__security=True), distinct=True), + bug_updates_count=Count('updates', filter=Q(updates__security=False), distinct=True), + errata_count=Count('errata', distinct=True), + packages_count=Count('packages', distinct=True), ) if 'domain_id' in request.GET: @@ -156,11 +157,20 @@ def host_detail(request, hostname): host = get_object_or_404(Host, hostname=hostname) reports = Report.objects.filter(host=hostname).order_by('-created')[:3] hostrepos = HostRepo.objects.filter(host=host) + + # Build packages list with update info + updates_by_package = {u.oldpackage_id: u for u in host.updates.select_related()} + packages_with_updates = [] + for package in host.packages.select_related('name', 'arch').order_by('name__name'): + package.update = updates_by_package.get(package.id) + packages_with_updates.append(package) + return render(request, 'hosts/host_detail.html', {'host': host, 'reports': reports, - 'hostrepos': hostrepos}) + 'hostrepos': hostrepos, + 'packages_with_updates': packages_with_updates}) @login_required diff --git a/patchman/static/css/base.css b/patchman/static/css/base.css index 45ed77c7..6c54a642 100644 --- a/patchman/static/css/base.css +++ b/patchman/static/css/base.css @@ -5,8 +5,6 @@ .panel-body { padding: 5px; font-size: 12px; } .panel { margin-bottom: 5px; font-size: 12px; } .panel-heading { padding: 5px; font-size: 13px; } -.brick { border-radius: 5px; margin: 3px 3px; padding: 2px 5px; height: 25px; border-style:solid; border-width:thin; } -.label-brick { font-size: 12px; font-weight: normal; margin-bottom: 3px; padding-top: 5px; border-radius: 4px; height:25px; border-style:solid; border-width:thin; border-color: #222; white-space: normal; display: inline-block; } .breadcrumb { font-size: 12px; background-color: #222; border-radius: 0; margin-bottom: 3px; } .navbar { margin-bottom: 0; padding-bottom: 0; border-radius: 0; } .navbar-inverse .dropdown-menu { background-color: #222; } @@ -17,6 +15,10 @@ th.min-width-col, td.min-width-col { width: 1%; white-space: nowrap; padding: 5px !important; } .table td { vertical-align: bottom !important; } .select2-results__options { max-height: 400px !important; } +.package-list { display: flex; flex-wrap: wrap; gap: 6px; list-style: none; padding: 10px; margin: 0; } +.package-list li { padding: 4px 8px; white-space: nowrap; border: 1px solid #ddd; border-radius: 4px; } +.package-list li:nth-child(odd) { background: #f0f7ff; } +.package-list li:nth-child(even) { background: #f5f0ff; } /* Center pagination controls produced by django-tables2 without centering table cell contents */ .django-tables2 .pagination { diff --git a/repos/forms.py b/repos/forms.py index d776cd00..f9795b51 100644 --- a/repos/forms.py +++ b/repos/forms.py @@ -28,6 +28,7 @@ class MirrorSelect2Widget(ModelSelect2MultipleWidget): model = Mirror search_fields = ['url__icontains', 'repo__name__icontains'] max_results = 50 + queryset = Mirror.objects.select_related().order_by('repo__name', 'url') def __init__(self, *args, **kwargs): kwargs.setdefault('attrs', {}) @@ -40,7 +41,7 @@ def label_from_instance(self, obj): class EditRepoForm(ModelForm): mirrors = ModelMultipleChoiceField( - queryset=Mirror.objects.select_related(), + queryset=Mirror.objects.select_related().order_by('repo__name', 'url'), required=False, widget=MirrorSelect2Widget(attrs={'style': 'width: 100%'}), ) @@ -93,12 +94,6 @@ def clean_repotype(self): class EditMirrorForm(ModelForm): - class Media: - css = { - 'all': ('admin/css/widgets.css',) - } - js = ('animations.js', 'actions.js') - def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.fields['url'].widget = TextInput(attrs={'size': 150},) diff --git a/security/templates/security/cve_detail.html b/security/templates/security/cve_detail.html index 6c86197a..a41ea6c7 100644 --- a/security/templates/security/cve_detail.html +++ b/security/templates/security/cve_detail.html @@ -82,20 +82,20 @@
    - {% for package in affected_packages %} - - {{ package }} - - {% endfor %} +
      + {% for package in affected_packages %} +
    • {{ package }}
      • + {% endfor %} +
    - {% for package in fixed_packages %} - - {{ package }} - - {% endfor %} +
      + {% for package in fixed_packages %} +
    • {{ package }}
      • + {% endfor %} +
    diff --git a/util/context_processors.py b/util/context_processors.py index f26c2748..ca3fa427 100644 --- a/util/context_processors.py +++ b/util/context_processors.py @@ -14,9 +14,10 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -from datetime import datetime, timedelta +from datetime import timedelta from django.db.models import F +from django.utils import timezone from hosts.models import Host from operatingsystems.models import OSRelease, OSVariant @@ -41,7 +42,7 @@ def issues_count(request): setting_type=int, default=14, ) - last_report_delta = datetime.now() - timedelta(days=days) + last_report_delta = timezone.now() - timedelta(days=days) stale_hosts = hosts.filter(lastreport__lt=last_report_delta) norepo_hosts = hosts.filter(repos__isnull=True, osvariant__osrelease__repos__isnull=True) reboot_hosts = hosts.filter(reboot_required=True) diff --git a/util/templates/dashboard.html b/util/templates/dashboard.html index b1dea384..699ecc56 100644 --- a/util/templates/dashboard.html +++ b/util/templates/dashboard.html @@ -189,11 +189,11 @@
    - {% for checksum in possible_mirrors %} - - {{ checksum }} - - {% endfor %} +
      + {% for checksum in possible_mirrors %} +
    • {{ checksum }}
      • + {% endfor %} +
    {% endif %} @@ -204,13 +204,11 @@
    -
    +
    +
    {% endif %} @@ -221,11 +219,11 @@
    - {% for package in orphaned_packages %} - - {{ package }} - - {% endfor %} +
      + {% for package in orphaned_packages %} +
    • {{ package }}
      • + {% endfor %} +
    {% endif %} From fa2c3e8a0cd2a0ceb17679f29d36db1e67e7e77a Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Tue, 13 Jan 2026 19:18:02 -0500 Subject: [PATCH 047/146] prefer triangles over chevrons --- util/templates/table.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/templates/table.html b/util/templates/table.html index 6084c681..bc63d651 100644 --- a/util/templates/table.html +++ b/util/templates/table.html @@ -12,7 +12,7 @@ {% for column in table.columns %}     - +
    InstalledAvailableInstalled PackageUpdate Available
    - {% if update.security %} - - {% else %} - + {% for package in packages_with_updates %} +
    + {% if package.update %} + {% if package.update.security %} + + {% else %} + + {% endif %} {% endif %} -   - {{ update.oldpackage }} + {{ package }} - {{ update.newpackage }} + {% if package.update %} + {{ package.update.newpackage }} + {% endif %}
    {% if column.orderable %} - {{ column.header }}{% if column.is_ordered %}{% if column.order_by_alias.is_descending %} {% else %} {% endif %}{% endif %} + {{ column.header }}{% if column.is_ordered %}{% if column.order_by_alias.is_descending %} {% else %} {% endif %}{% endif %} {% else %} {{ column.header|safe }} {% endif %} From 26c74a31bc7f9011437535fb91c2252920c13f52 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Tue, 13 Jan 2026 19:23:44 -0500 Subject: [PATCH 048/146] fix table column alignments and cell contents --- errata/tables.py | 14 ++++++------- patchman/static/css/base.css | 4 ++++ patchman/static/js/expandable-text.js | 2 +- repos/tables.py | 4 ++-- security/tables.py | 30 +++++++++++---------------- 5 files changed, 26 insertions(+), 28 deletions(-) diff --git a/errata/tables.py b/errata/tables.py index c23d25be..d70be2cd 100644 --- a/errata/tables.py +++ b/errata/tables.py @@ -55,7 +55,7 @@ class ErratumTable(BaseTable): ERRATUM_NAME_TEMPLATE, order_by='name', verbose_name='ID', - attrs={'th': {'class': 'col-sm-2'}, 'td': {'class': 'col-sm-2'}}, + attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1'}}, ) e_type = tables.Column( order_by='e_type', @@ -65,7 +65,7 @@ class ErratumTable(BaseTable): issue_date = tables.DateColumn( order_by='issue_date', verbose_name='Published Date', - attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1'}}, + attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1 centered'}}, ) synopsis = tables.Column( orderable=False, @@ -76,31 +76,31 @@ class ErratumTable(BaseTable): PACKAGES_AFFECTED_TEMPLATE, orderable=False, verbose_name='Packages Affected', - attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1'}}, + attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1 centered'}}, ) packages_fixed = tables.TemplateColumn( PACKAGES_FIXED_TEMPLATE, orderable=False, verbose_name='Packages Fixed', - attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1'}}, + attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1 centered'}}, ) osreleases = tables.TemplateColumn( OSRELEASES_TEMPLATE, orderable=False, verbose_name='OS Releases Affected', - attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1'}}, + attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1 centered'}}, ) erratum_cves = tables.TemplateColumn( ERRATUM_CVES_TEMPLATE, orderable=False, verbose_name='CVEs', - attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1'}}, + attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1 centered'}}, ) references = tables.TemplateColumn( REFERENCES_TEMPLATE, orderable=False, verbose_name='References', - attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1'}}, + attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1 centered'}}, ) class Meta(BaseTable.Meta): diff --git a/patchman/static/css/base.css b/patchman/static/css/base.css index 6c54a642..76c7dbc4 100644 --- a/patchman/static/css/base.css +++ b/patchman/static/css/base.css @@ -19,6 +19,10 @@ th.min-width-col, td.min-width-col { width: 1%; white-space: nowrap; padding: 5p .package-list li { padding: 4px 8px; white-space: nowrap; border: 1px solid #ddd; border-radius: 4px; } .package-list li:nth-child(odd) { background: #f0f7ff; } .package-list li:nth-child(even) { background: #f5f0ff; } +td.truncate-cell { max-width: 0; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +td.truncate-cell a { display: block; overflow: hidden; text-overflow: ellipsis; } +.expandable-text { cursor: pointer; display: block; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +.expandable-text.expanded { white-space: normal; overflow: visible; } /* Center pagination controls produced by django-tables2 without centering table cell contents */ .django-tables2 .pagination { diff --git a/patchman/static/js/expandable-text.js b/patchman/static/js/expandable-text.js index 0f5861ce..d881e73c 100644 --- a/patchman/static/js/expandable-text.js +++ b/patchman/static/js/expandable-text.js @@ -2,7 +2,7 @@ document.addEventListener('DOMContentLoaded', function() { const expandableTexts = document.querySelectorAll('.expandable-text'); expandableTexts.forEach(text => { text.addEventListener('click', function() { - this.textContent = this.dataset.fullText; + this.classList.toggle('expanded'); }); }); }); diff --git a/repos/tables.py b/repos/tables.py index 8551822f..d4c8bb9b 100644 --- a/repos/tables.py +++ b/repos/tables.py @@ -32,7 +32,7 @@ # MirrorTable templates MIRROR_CHECKBOX_TEMPLATE = '' MIRROR_ID_TEMPLATE = '{{ record.id }}' -MIRROR_URL_TEMPLATE = '{{ record.url|truncatechars:25 }}' +MIRROR_URL_TEMPLATE = '{{ record.url }}' MIRROR_PACKAGES_TEMPLATE = ( '{% if not record.mirrorlist %}' '' @@ -113,7 +113,7 @@ class MirrorTable(BaseTable): MIRROR_URL_TEMPLATE, orderable=False, verbose_name='URL', - attrs={'th': {'class': 'col-sm-2'}, 'td': {'class': 'col-sm-2'}}, + attrs={'th': {'class': 'col-sm-2'}, 'td': {'class': 'col-sm-2 truncate-cell'}}, ) mirror_packages = tables.TemplateColumn( MIRROR_PACKAGES_TEMPLATE, diff --git a/security/tables.py b/security/tables.py index 4d7272af..ba209829 100644 --- a/security/tables.py +++ b/security/tables.py @@ -27,10 +27,7 @@ '  ' 'osv.dev {% bootstrap_icon "link" %}' ) -CVE_DESCRIPTION_TEMPLATE = ( - '' - '{{ record.description|truncatechars:60 }}' -) +CVE_DESCRIPTION_TEMPLATE = '{{ record.description }}' CVSS_SCORES_TEMPLATE = '{% for score in record.cvss_scores.all %} {{ score.score }} {% endfor %}' CWES_TEMPLATE = '{% for cwe in record.cwes.all %} {{ cwe.cwe_id }} {% endfor %}' CVE_ERRATA_TEMPLATE = ( @@ -40,10 +37,7 @@ # CWETable templates CWE_ID_TEMPLATE = '{{ record.cwe_id }}' -CWE_DESCRIPTION_TEMPLATE = ( - '' - '{{ record.description|truncatechars:120 }}' -) +CWE_DESCRIPTION_TEMPLATE = '{{ record.description }}' CWE_CVES_TEMPLATE = ( '' '{{ record.cve_set.count }}' @@ -68,13 +62,13 @@ class CVETable(BaseTable): CVE_LINKS_TEMPLATE, orderable=False, verbose_name='Links', - attrs={'th': {'class': 'col-sm-2'}, 'td': {'class': 'col-sm-2'}}, + attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1'}}, ) cve_description = tables.TemplateColumn( CVE_DESCRIPTION_TEMPLATE, orderable=False, verbose_name='Description', - attrs={'th': {'class': 'col-sm-3'}, 'td': {'class': 'col-sm-3'}}, + attrs={'th': {'class': 'col-sm-3'}, 'td': {'class': 'col-sm-3 truncate-cell'}}, ) cvss_scores = tables.TemplateColumn( CVSS_SCORES_TEMPLATE, @@ -92,25 +86,25 @@ class CVETable(BaseTable): order_by='reserved_date', verbose_name='Reserved', default='', - attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1'}}, + attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1 centered'}}, ) rejected_date = tables.DateColumn( order_by='rejected_date', verbose_name='Rejected', default='', - attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1'}}, + attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1 centered'}}, ) published_date = tables.DateColumn( order_by='published_date', verbose_name='Published', default='', - attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1'}}, + attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1 centered'}}, ) updated_date = tables.DateColumn( order_by='updated_date', verbose_name='Updated', default='', - attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1'}}, + attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1 centered'}}, ) cve_errata = tables.TemplateColumn( CVE_ERRATA_TEMPLATE, @@ -145,13 +139,13 @@ class CWETable(BaseTable): CWE_DESCRIPTION_TEMPLATE, orderable=False, verbose_name='Description', - attrs={'th': {'class': 'col-sm-6'}, 'td': {'class': 'col-sm-6'}}, + attrs={'th': {'class': 'col-sm-6'}, 'td': {'class': 'col-sm-6 truncate-cell'}}, ) cwe_cves = tables.TemplateColumn( CWE_CVES_TEMPLATE, orderable=False, verbose_name='CVEs', - attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1'}}, + attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1 centered'}}, ) class Meta(BaseTable.Meta): @@ -169,13 +163,13 @@ class ReferenceTable(BaseTable): REFERENCE_URL_TEMPLATE, orderable=False, verbose_name='URL', - attrs={'th': {'class': 'col-sm-10'}, 'td': {'class': 'col-sm-10'}}, + attrs={'th': {'class': 'col-sm-8'}, 'td': {'class': 'col-sm-8'}}, ) linked_errata = tables.TemplateColumn( LINKED_ERRATA_TEMPLATE, orderable=False, verbose_name='Linked Errata', - attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1'}}, + attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1 centered'}}, ) class Meta(BaseTable.Meta): From a730a4fa77e4b92202146cc4607b292361727f29 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Tue, 13 Jan 2026 19:29:58 -0500 Subject: [PATCH 049/146] fix OS tables and add dropdown menu --- operatingsystems/tables.py | 12 ++++++------ util/templates/navbar.html | 8 +++++++- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/operatingsystems/tables.py b/operatingsystems/tables.py index 0be5b77d..8e3acb8c 100644 --- a/operatingsystems/tables.py +++ b/operatingsystems/tables.py @@ -87,25 +87,25 @@ class OSReleaseTable(BaseTable): OSRELEASE_REPOS_TEMPLATE, verbose_name='Repos', orderable=False, - attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1'}}, + attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1 centered'}}, ) osvariants = tables.TemplateColumn( OSVARIANTS_TEMPLATE, verbose_name='OS Variants', orderable=False, - attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1'}}, + attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1 centered'}}, ) osrelease_hosts = tables.TemplateColumn( OSRELEASE_HOSTS_TEMPLATE, verbose_name='Hosts', orderable=False, - attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1'}}, + attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1 centered'}}, ) osrelease_errata = tables.TemplateColumn( OSRELEASE_ERRATA_TEMPLATE, verbose_name='Errata', orderable=False, - attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1'}}, + attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1 centered'}}, ) class Meta(BaseTable.Meta): @@ -144,7 +144,7 @@ class OSVariantTable(BaseTable): OSVARIANT_HOSTS_TEMPLATE, verbose_name='Hosts', order_by='hosts_count', - attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1'}}, + attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1 centered'}}, ) osrelease = tables.TemplateColumn( OSVARIANT_OSRELEASE_TEMPLATE, @@ -156,7 +156,7 @@ class OSVariantTable(BaseTable): REPOS_OSRELEASE_TEMPLATE, verbose_name='Repos (OS Release)', order_by='repos_count', - attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1'}}, + attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1 centered'}}, ) class Meta(BaseTable.Meta): diff --git a/util/templates/navbar.html b/util/templates/navbar.html index 206f09fd..6effcaad 100644 --- a/util/templates/navbar.html +++ b/util/templates/navbar.html @@ -29,7 +29,13 @@
  • Security References
    • -
  • Operating Systems
    • +
  • + Operating Systems + +
  • Reports
  • Issues{% if issues_count %} {{ issues_count }}{% endif %}
    • From 7bdf1f21164ab33117ff30bb40ff91e330cd8b5c Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Tue, 13 Jan 2026 19:37:26 -0500 Subject: [PATCH 050/146] fix broken rocky links --- errata/sources/distros/rocky.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/errata/sources/distros/rocky.py b/errata/sources/distros/rocky.py index 2d805985..272ac2a7 100644 --- a/errata/sources/distros/rocky.py +++ b/errata/sources/distros/rocky.py @@ -203,8 +203,8 @@ def process_rocky_erratum(advisory): def add_rocky_erratum_references(e, advisory): """ Add Rocky Linux errata references """ - e.add_reference('Rocky Advisory', 'https://apollo.build.resf.org/{e.name}') - e.add_reference('Rocky Advisory', 'https://errata.rockylinux.org/{e.name}') + e.add_reference('Rocky Advisory', f'https://apollo.build.resf.org/{e.name}') + e.add_reference('Rocky Advisory', f'https://errata.rockylinux.org/{e.name}') advisory_cves = advisory.get('cves') for a_cve in advisory_cves: cve_id = a_cve.get('cve') From 4c8af9e778f927a7c1197c811b57375d282444a7 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Tue, 13 Jan 2026 19:44:05 -0500 Subject: [PATCH 051/146] move navbar to side --- patchman/static/css/base.css | 20 ++++- patchman/static/js/expandable-text.js | 32 ++++++++ util/templates/base.html | 28 ++++--- util/templates/dashboard.html | 4 + util/templates/navbar.html | 105 ++++++++++++-------------- 5 files changed, 115 insertions(+), 74 deletions(-) diff --git a/patchman/static/css/base.css b/patchman/static/css/base.css index 76c7dbc4..3acb3ff5 100644 --- a/patchman/static/css/base.css +++ b/patchman/static/css/base.css @@ -5,7 +5,7 @@ .panel-body { padding: 5px; font-size: 12px; } .panel { margin-bottom: 5px; font-size: 12px; } .panel-heading { padding: 5px; font-size: 13px; } -.breadcrumb { font-size: 12px; background-color: #222; border-radius: 0; margin-bottom: 3px; } +.breadcrumb { font-size: 14px; background-color: #222; border-radius: 0; margin-bottom: 3px; position: sticky; top: 0; z-index: 100; } .navbar { margin-bottom: 0; padding-bottom: 0; border-radius: 0; } .navbar-inverse .dropdown-menu { background-color: #222; } .navbar-inverse .dropdown-menu > li > a { color: #9d9d9d; } @@ -24,6 +24,24 @@ td.truncate-cell a { display: block; overflow: hidden; text-overflow: ellipsis; .expandable-text { cursor: pointer; display: block; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } .expandable-text.expanded { white-space: normal; overflow: visible; } +/* Sidebar layout */ +.sidebar-layout { display: flex; min-height: 100vh; } +.sidebar-nav { width: 200px; background-color: #222; display: flex; flex-direction: column; position: fixed; height: 100vh; } +.sidebar-header { padding: 15px; border-bottom: 1px solid #333; } +.sidebar-brand { color: #9d9d9d; font-size: 18px; font-weight: bold; text-decoration: none; } +.sidebar-brand:hover { color: #fff; text-decoration: none; } +.sidebar-menu { list-style: none; padding: 0; margin: 0; } +.sidebar-menu li a { display: block; padding: 10px 15px; color: #9d9d9d; text-decoration: none; font-size: 14px; } +.sidebar-menu li a:hover { background-color: #333; color: #fff; } +.sidebar-menu li.active > a { background-color: #333; color: #fff; } +.sidebar-menu .submenu { list-style: none; padding: 0; margin: 0; display: none; background-color: #1a1a1a; } +.sidebar-menu .submenu li a { padding-left: 30px; font-size: 13px; } +.sidebar-menu .has-submenu.open > .submenu { display: block; } +.sidebar-menu .has-submenu > a .caret { float: right; margin-top: 6px; transition: transform 0.2s; } +.sidebar-menu .has-submenu.open > a .caret { transform: rotate(180deg); } +.sidebar-bottom { margin-top: auto; border-top: 1px solid #333; } +.main-content { margin-left: 200px; flex: 1; } + /* Center pagination controls produced by django-tables2 without centering table cell contents */ .django-tables2 .pagination { text-align: center; diff --git a/patchman/static/js/expandable-text.js b/patchman/static/js/expandable-text.js index d881e73c..fa803106 100644 --- a/patchman/static/js/expandable-text.js +++ b/patchman/static/js/expandable-text.js @@ -5,4 +5,36 @@ document.addEventListener('DOMContentLoaded', function() { this.classList.toggle('expanded'); }); }); + + // Sidebar submenu state from localStorage + const STORAGE_KEY = 'patchman_sidebar_state'; + const savedState = JSON.parse(localStorage.getItem(STORAGE_KEY) || '{}'); + + // Apply saved state to submenus + const submenuItems = document.querySelectorAll('.has-submenu'); + submenuItems.forEach((item, index) => { + const menuId = item.querySelector('a').textContent.trim(); + if (savedState[menuId] !== undefined) { + if (savedState[menuId]) { + item.classList.add('open'); + } else { + item.classList.remove('open'); + } + } + }); + + // Toggle submenu and save state + const submenuLinks = document.querySelectorAll('.has-submenu > a'); + submenuLinks.forEach(link => { + link.addEventListener('click', function(e) { + e.preventDefault(); + const parent = this.parentElement; + parent.classList.toggle('open'); + // Save state to localStorage + const menuId = this.textContent.trim(); + const currentState = JSON.parse(localStorage.getItem(STORAGE_KEY) || '{}'); + currentState[menuId] = parent.classList.contains('open'); + localStorage.setItem(STORAGE_KEY, JSON.stringify(currentState)); + }); + }); }); diff --git a/util/templates/base.html b/util/templates/base.html index 17e952f3..3e780fcc 100644 --- a/util/templates/base.html +++ b/util/templates/base.html @@ -12,23 +12,21 @@ {% block extrahead %}{% endblock %} -
    +
    {% include "navbar.html" with user=user %} -
      - {% block breadcrumbs %} -
    • Home
      • - {% endblock %} -
    - {% bootstrap_messages %} -
    -
    -
    - {% block content_title %}{% endblock %} +
    +
      + {% block breadcrumbs %} + {% endblock %} +
    + {% bootstrap_messages %} +
    +
    +
    + {% block content %}{{ content }}{% endblock %} +
    + {% block footer %}{% endblock %}
    -
    - {% block content %}{{ content }}{% endblock %} -
    - {% block footer %}{% endblock %}
    diff --git a/util/templates/dashboard.html b/util/templates/dashboard.html index 699ecc56..5976adcb 100644 --- a/util/templates/dashboard.html +++ b/util/templates/dashboard.html @@ -4,6 +4,10 @@ {% block page_title %}Patchman Dashboard{% endblock %} +{% block breadcrumbs %} +
  • Issues
    • +{% endblock %} + {% block content_title %} Issues Dashboard for {{ site.name }} {% endblock %} {% block content %} diff --git a/util/templates/navbar.html b/util/templates/navbar.html index 6effcaad..5e7d233e 100644 --- a/util/templates/navbar.html +++ b/util/templates/navbar.html @@ -1,61 +1,50 @@ - From 14b7fd0e95990668425e656ac86b7e3c8a4907cb Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Tue, 13 Jan 2026 19:49:17 -0500 Subject: [PATCH 052/146] fix datetime issues using timezone --- util/templatetags/common.py | 5 +++-- util/views.py | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/util/templatetags/common.py b/util/templatetags/common.py index 38d478d6..0fdeaff1 100644 --- a/util/templatetags/common.py +++ b/util/templatetags/common.py @@ -15,11 +15,12 @@ # along with Patchman. If not, see import importlib -from datetime import datetime, timedelta +from datetime import timedelta from urllib.parse import urlencode from django.template import Library from django.template.loader import get_template +from django.utils import timezone from django.utils.html import format_html from django_tables2 import RequestConfig from humanize import naturaltime @@ -132,7 +133,7 @@ def reports_timedelta(): setting_type=int, default=14, ) - return naturaltime(datetime.now() - timedelta(days=days)) + return naturaltime(timezone.now() - timedelta(days=days)) @register.simple_tag diff --git a/util/views.py b/util/views.py index fd003bca..6d88f28d 100644 --- a/util/views.py +++ b/util/views.py @@ -15,12 +15,13 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -from datetime import datetime, timedelta +from datetime import timedelta from django.contrib.auth.decorators import login_required from django.contrib.sites.models import Site from django.db.models import F from django.shortcuts import render +from django.utils import timezone from hosts.models import Host from operatingsystems.models import OSRelease, OSVariant @@ -50,7 +51,7 @@ def dashboard(request): setting_type=int, default=14, ) - last_report_delta = datetime.now() - timedelta(days=days) + last_report_delta = timezone.now() - timedelta(days=days) stale_hosts = hosts.filter(lastreport__lt=last_report_delta) norepo_hosts = hosts.filter(repos__isnull=True, osvariant__osrelease__repos__isnull=True) # noqa reboot_hosts = hosts.filter(reboot_required=True) From 7024c33cec720f78af1c254eb5e8300a93f2a125 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Tue, 13 Jan 2026 21:09:20 -0500 Subject: [PATCH 053/146] add collapsible sidebar --- patchman/static/css/base.css | 18 +++++++--- patchman/static/js/expandable-text.js | 32 ----------------- patchman/static/js/sidebar.js | 52 +++++++++++++++++++++++++++ util/templates/base.html | 6 ++++ 4 files changed, 72 insertions(+), 36 deletions(-) create mode 100644 patchman/static/js/sidebar.js diff --git a/patchman/static/css/base.css b/patchman/static/css/base.css index 3acb3ff5..86927756 100644 --- a/patchman/static/css/base.css +++ b/patchman/static/css/base.css @@ -6,6 +6,7 @@ .panel { margin-bottom: 5px; font-size: 12px; } .panel-heading { padding: 5px; font-size: 13px; } .breadcrumb { font-size: 14px; background-color: #222; border-radius: 0; margin-bottom: 3px; position: sticky; top: 0; z-index: 100; } +.breadcrumb > li:nth-child(2):before { content: none; } .navbar { margin-bottom: 0; padding-bottom: 0; border-radius: 0; } .navbar-inverse .dropdown-menu { background-color: #222; } .navbar-inverse .dropdown-menu > li > a { color: #9d9d9d; } @@ -26,12 +27,16 @@ td.truncate-cell a { display: block; overflow: hidden; text-overflow: ellipsis; /* Sidebar layout */ .sidebar-layout { display: flex; min-height: 100vh; } -.sidebar-nav { width: 200px; background-color: #222; display: flex; flex-direction: column; position: fixed; height: 100vh; } +.sidebar-nav { width: 200px; background-color: #222; display: flex; flex-direction: column; position: fixed; height: 100vh; overflow: hidden; } .sidebar-header { padding: 15px; border-bottom: 1px solid #333; } -.sidebar-brand { color: #9d9d9d; font-size: 18px; font-weight: bold; text-decoration: none; } +.sidebar-brand { color: #9d9d9d; font-size: 18px; font-weight: bold; text-decoration: none; white-space: nowrap; } .sidebar-brand:hover { color: #fff; text-decoration: none; } -.sidebar-menu { list-style: none; padding: 0; margin: 0; } -.sidebar-menu li a { display: block; padding: 10px 15px; color: #9d9d9d; text-decoration: none; font-size: 14px; } +.sidebar-toggle { background: none; border: none; color: #666; cursor: pointer; padding: 0 10px 0 0; font-size: 14px; } +.sidebar-toggle:hover { color: #333; } +.sidebar-toggle .glyphicon { transition: transform 0.2s; } +.sidebar-toggle-container { list-style: none; display: flex; align-items: center; } +.sidebar-menu { list-style: none; padding: 0; margin: 0; overflow-y: auto; overflow-x: hidden; } +.sidebar-menu li a { display: block; padding: 10px 15px; color: #9d9d9d; text-decoration: none; font-size: 14px; white-space: nowrap; } .sidebar-menu li a:hover { background-color: #333; color: #fff; } .sidebar-menu li.active > a { background-color: #333; color: #fff; } .sidebar-menu .submenu { list-style: none; padding: 0; margin: 0; display: none; background-color: #1a1a1a; } @@ -42,6 +47,11 @@ td.truncate-cell a { display: block; overflow: hidden; text-overflow: ellipsis; .sidebar-bottom { margin-top: auto; border-top: 1px solid #333; } .main-content { margin-left: 200px; flex: 1; } +/* Collapsed sidebar */ +.sidebar-nav.collapsed { width: 0; } +.sidebar-layout.collapsed .main-content { margin-left: 0; } +.sidebar-layout.collapsed .sidebar-toggle .glyphicon { transform: rotate(180deg); } + /* Center pagination controls produced by django-tables2 without centering table cell contents */ .django-tables2 .pagination { text-align: center; diff --git a/patchman/static/js/expandable-text.js b/patchman/static/js/expandable-text.js index fa803106..d881e73c 100644 --- a/patchman/static/js/expandable-text.js +++ b/patchman/static/js/expandable-text.js @@ -5,36 +5,4 @@ document.addEventListener('DOMContentLoaded', function() { this.classList.toggle('expanded'); }); }); - - // Sidebar submenu state from localStorage - const STORAGE_KEY = 'patchman_sidebar_state'; - const savedState = JSON.parse(localStorage.getItem(STORAGE_KEY) || '{}'); - - // Apply saved state to submenus - const submenuItems = document.querySelectorAll('.has-submenu'); - submenuItems.forEach((item, index) => { - const menuId = item.querySelector('a').textContent.trim(); - if (savedState[menuId] !== undefined) { - if (savedState[menuId]) { - item.classList.add('open'); - } else { - item.classList.remove('open'); - } - } - }); - - // Toggle submenu and save state - const submenuLinks = document.querySelectorAll('.has-submenu > a'); - submenuLinks.forEach(link => { - link.addEventListener('click', function(e) { - e.preventDefault(); - const parent = this.parentElement; - parent.classList.toggle('open'); - // Save state to localStorage - const menuId = this.textContent.trim(); - const currentState = JSON.parse(localStorage.getItem(STORAGE_KEY) || '{}'); - currentState[menuId] = parent.classList.contains('open'); - localStorage.setItem(STORAGE_KEY, JSON.stringify(currentState)); - }); - }); }); diff --git a/patchman/static/js/sidebar.js b/patchman/static/js/sidebar.js new file mode 100644 index 00000000..a134ab36 --- /dev/null +++ b/patchman/static/js/sidebar.js @@ -0,0 +1,52 @@ +document.addEventListener('DOMContentLoaded', function() { + const STORAGE_KEY = 'patchman_sidebar_state'; + const COLLAPSED_KEY = 'patchman_sidebar_collapsed'; + const savedState = JSON.parse(localStorage.getItem(STORAGE_KEY) || '{}'); + + const sidebarNav = document.querySelector('.sidebar-nav'); + const sidebarLayout = document.querySelector('.sidebar-layout'); + + // Apply saved collapsed state + if (localStorage.getItem(COLLAPSED_KEY) === 'true') { + sidebarNav.classList.add('collapsed'); + sidebarLayout.classList.add('collapsed'); + } + + // Sidebar collapse toggle + const sidebarToggle = document.querySelector('.sidebar-toggle'); + if (sidebarToggle) { + sidebarToggle.addEventListener('click', function() { + sidebarNav.classList.toggle('collapsed'); + sidebarLayout.classList.toggle('collapsed'); + localStorage.setItem(COLLAPSED_KEY, sidebarNav.classList.contains('collapsed')); + }); + } + + // Apply saved state to submenus + const submenuItems = document.querySelectorAll('.has-submenu'); + submenuItems.forEach((item, index) => { + const menuId = item.querySelector('a').textContent.trim(); + if (savedState[menuId] !== undefined) { + if (savedState[menuId]) { + item.classList.add('open'); + } else { + item.classList.remove('open'); + } + } + }); + + // Toggle submenu and save state + const submenuLinks = document.querySelectorAll('.has-submenu > a'); + submenuLinks.forEach(link => { + link.addEventListener('click', function(e) { + e.preventDefault(); + const parent = this.parentElement; + parent.classList.toggle('open'); + // Save state to localStorage + const menuId = this.textContent.trim(); + const currentState = JSON.parse(localStorage.getItem(STORAGE_KEY) || '{}'); + currentState[menuId] = parent.classList.contains('open'); + localStorage.setItem(STORAGE_KEY, JSON.stringify(currentState)); + }); + }); +}); diff --git a/util/templates/base.html b/util/templates/base.html index 3e780fcc..e29b228f 100644 --- a/util/templates/base.html +++ b/util/templates/base.html @@ -9,6 +9,7 @@ {% block page_title %}{% endblock %} + {% block extrahead %}{% endblock %} @@ -16,6 +17,11 @@ {% include "navbar.html" with user=user %}
      +
    • + +
      • {% block breadcrumbs %} {% endblock %}
    From 257686f786217657a28cf71d9d1f601b6382b62c Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Thu, 22 Jan 2026 23:14:02 -0500 Subject: [PATCH 054/146] table tweaks --- hosts/tables.py | 16 ++++++++-------- operatingsystems/tables.py | 1 + repos/tables.py | 2 +- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/hosts/tables.py b/hosts/tables.py index e527af7b..715a29ae 100644 --- a/hosts/tables.py +++ b/hosts/tables.py @@ -63,25 +63,25 @@ class HostTable(BaseTable): HOSTNAME_TEMPLATE, order_by='hostname', verbose_name='Hostname', - attrs={'th': {'class': 'col-sm-3'}, 'td': {'class': 'col-sm-3'}}, + attrs={'th': {'class': 'col-sm-2'}, 'td': {'class': 'col-sm-2'}}, ) sec_updates = tables.TemplateColumn( SEC_UPDATES_TEMPLATE, order_by='sec_updates_count', verbose_name='Security Updates', - attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1 centered'}}, + attrs={'th': {'class': 'min-width-col'}, 'td': {'class': 'min-width-col centered'}}, ) bug_updates = tables.TemplateColumn( BUG_UPDATES_TEMPLATE, order_by='bug_updates_count', verbose_name='Bugfix Updates', - attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1 centered'}}, + attrs={'th': {'class': 'min-width-col'}, 'td': {'class': 'min-width-col centered'}}, ) affected_errata = tables.TemplateColumn( AFFECTED_ERRATA_TEMPLATE, order_by='errata_count', verbose_name='Affected by Errata', - attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1 centered'}}, + attrs={'th': {'class': 'min-width-col'}, 'td': {'class': 'min-width-col centered'}}, ) kernel = tables.Column( verbose_name='Running Kernel', @@ -91,25 +91,25 @@ class HostTable(BaseTable): OSVARIANT_TEMPLATE, order_by='osvariant__name', verbose_name='OS Variant', - attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1'}}, + attrs={'th': {'class': 'col-sm-2'}, 'td': {'class': 'col-sm-2'}}, ) packages_installed = tables.TemplateColumn( PACKAGES_TEMPLATE, order_by='packages_count', verbose_name='Packages', - attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1 centered'}}, + attrs={'th': {'class': 'min-width-col'}, 'td': {'class': 'min-width-col centered'}}, ) lastreport = tables.TemplateColumn( LASTREPORT_TEMPLATE, order_by='lastreport', verbose_name='Last Report', - attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1'}}, + attrs={'th': {'class': 'col-sm-2'}, 'td': {'class': 'col-sm-2'}}, ) reboot_required = tables.TemplateColumn( REBOOT_TEMPLATE, order_by='reboot_required', verbose_name='Reboot Status', - attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1 centered'}}, + attrs={'th': {'class': 'min-width-col'}, 'td': {'class': 'min-width-col centered'}}, ) class Meta(BaseTable.Meta): diff --git a/operatingsystems/tables.py b/operatingsystems/tables.py index 8e3acb8c..14437094 100644 --- a/operatingsystems/tables.py +++ b/operatingsystems/tables.py @@ -131,6 +131,7 @@ class OSVariantTable(BaseTable): ) osvariant_arch = tables.Column( accessor='arch__name', + default='', verbose_name='Architecture', attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1'}}, ) diff --git a/repos/tables.py b/repos/tables.py index d4c8bb9b..cff76c08 100644 --- a/repos/tables.py +++ b/repos/tables.py @@ -148,7 +148,7 @@ class MirrorTable(BaseTable): timestamp = tables.Column( order_by='timestamp', verbose_name='Timestamp', - attrs={'th': {'class': 'col-sm-2'}, 'td': {'class': 'col-sm-2'}}, + attrs={'th': {'class': 'col-sm-1'}, 'td': {'class': 'col-sm-1'}}, ) checksum = tables.TemplateColumn( CHECKSUM_TEMPLATE, From d5971533a36ca0c548ce74bac3477849750fe796 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Thu, 22 Jan 2026 23:23:57 -0500 Subject: [PATCH 055/146] add more generic approach to handling redhat references --- security/utils.py | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/security/utils.py b/security/utils.py index 343442a7..127f2c73 100644 --- a/security/utils.py +++ b/security/utils.py @@ -17,6 +17,7 @@ from urllib.parse import urlparse from security.models import CVE, CWE, Reference +from util.logging import warning_message def get_cve_reference(cve_id): @@ -111,22 +112,32 @@ def fixup_reference(ref): url = fixup_bugzilla_url(url) url = fixup_rhn_url(url) if url.hostname == 'access.redhat.com': - if 'l1d-cache-eviction-and-vector-register-sampling' in url.path or \ - 'security/vulnerabilities/speculativeexecution' in url.path or \ - 'security/vulnerabilities/stackguard' in url.path: - ref_type = 'Link' - elif 'security/cve' in url.path: + processed = False + old_ref = url.path.split('/')[-1] + if 'security/cve' in url.path: return - else: - old_ref = url.path.split('/')[-1] + elif url.path.startswith('/solutions/') or url.path.startswith('/articles/'): + ref_type = 'Link' + processed = True + elif url.path.startswith('/security/vulnerabilities/'): + if old_ref.isdigit() or old_ref.isalpha(): + ref_type = 'Link' + processed = True + elif old_ref.startswith('cve'): + ref_type = 'Link' + processed = True + if not processed: refs = old_ref.split('-') if ':' not in url.path: try: - new_ref = f'{refs[0]}-{refs[1]}:{refs[2]}' + if refs[0].upper() == 'RHSB': + new_ref = f'{refs[0]}-{refs[1]}-{refs[2]}' + else: + new_ref = f'{refs[0]}-{refs[1]}:{refs[2]}' path = url.path.replace(old_ref, new_ref) url = url._replace(path=path) except IndexError: - pass + warning_message(f'Unable to process reference URL: {url}') ref_type = refs[0].upper() final_url = url.geturl() if final_url in ['https://launchpad.net/bugs/', 'https://launchpad.net/bugs/XXXXXX']: From 4afc05d0c6c4ef17552525687aba4f54c4915c4f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 23 Jan 2026 04:40:42 +0000 Subject: [PATCH 056/146] Bump django-select2 from 8.3.0 to 8.4.1 Bumps [django-select2](https://github.com/codingjoe/django-select2) from 8.3.0 to 8.4.1. - [Release notes](https://github.com/codingjoe/django-select2/releases) - [Commits](https://github.com/codingjoe/django-select2/compare/8.3.0...8.4.1) --- updated-dependencies: - dependency-name: django-select2 dependency-version: 8.4.1 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 1bad175b..3101fac5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,4 +21,4 @@ tqdm==4.67.1 cvss==3.4 zstandard==0.25.0 django-tables2==2.8.0 -django-select2==8.3.0 +django-select2==8.4.1 From f5271450e5a8bc3a0e66830bbc6c39611489cdef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Thu, 29 Jan 2026 10:20:42 +0000 Subject: [PATCH 057/146] Branch cleanup --- util/templatetags/common.py | 1 - 1 file changed, 1 deletion(-) diff --git a/util/templatetags/common.py b/util/templatetags/common.py index 8f8bc792..0fdeaff1 100644 --- a/util/templatetags/common.py +++ b/util/templatetags/common.py @@ -18,7 +18,6 @@ from datetime import timedelta from urllib.parse import urlencode -from django.core.paginator import Paginator from django.template import Library from django.template.loader import get_template from django.utils import timezone From 83d4a920b7eb8d0dc912bd6fcc00befffaca7b1f Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Sun, 1 Feb 2026 13:22:38 -0500 Subject: [PATCH 058/146] rest api report uploads --- INSTALL.md | 87 +++++ README.md | 3 + client/patchman-client | 351 +++++++++++++++++- client/patchman-client.conf | 7 + debian/control | 5 +- patchman-client.spec | 2 +- patchman/settings.py | 9 + patchman/urls.py | 2 + reports/models.py | 122 ++++++- reports/serializers.py | 91 +++++ reports/tables.py | 54 +++ reports/templates/reports/report_detail.html | 128 ++++--- reports/tests.py | 291 +++++++++++++++ reports/utils.py | 354 ++++++++++++++----- reports/views.py | 113 +++++- requirements.txt | 1 + util/management/__init__.py | 0 util/management/commands/__init__.py | 0 util/management/commands/create_api_key.py | 43 +++ util/management/commands/list_api_keys.py | 52 +++ util/management/commands/revoke_api_key.py | 63 ++++ 21 files changed, 1634 insertions(+), 144 deletions(-) create mode 100644 reports/serializers.py create mode 100644 reports/tests.py create mode 100644 util/management/__init__.py create mode 100644 util/management/commands/__init__.py create mode 100644 util/management/commands/create_api_key.py create mode 100644 util/management/commands/list_api_keys.py create mode 100644 util/management/commands/revoke_api_key.py diff --git a/INSTALL.md b/INSTALL.md index cf669297..2cd83308 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -140,6 +140,93 @@ is running +## Report Protocols + +Patchman supports two report protocols: + +### Protocol 1 (Text) +The original form-based protocol. Uses multipart form data to upload package +and repository information. No additional dependencies required on the client. + +### Protocol 2 (JSON) +A JSON-based REST API. Provides better error handling, structured validation, +and easier debugging. Requires `jq` on the client. + +To use Protocol 2, update your `patchman-client.conf`: + +```shell +protocol=2 +``` + +Or use the `-p 2` command line option: + +```shell +$ patchman-client -s http://patchman.example.org -p 2 +``` + + +## API Key Authentication + +For Protocol 2, API key authentication is available using +[djangorestframework-api-key](https://florimondmanca.github.io/djangorestframework-api-key/). +Keys are hashed in the database and cannot be retrieved after creation. + +### Server-side setup + +1. Run migrations (first time only): + +```shell +$ patchman-manage migrate +``` + +2. Create an API key: + +```shell +$ patchman-manage create_api_key "clients" +Created API key: clients + + Key: abc123... + +Add this to your patchman-client.conf: + api_key=abc123... + +Save this key as it cannot be retrieved later. +``` + +3. List existing keys: + +```shell +$ patchman-manage list_api_keys +``` + +4. Revoke a key: + +```shell +$ patchman-manage revoke_api_key +``` + +5. To require API keys for all Protocol 2 uploads, set in `local_settings.py`: + +```python +REQUIRE_API_KEY = True +``` + +API keys can also be managed via the Django admin interface. + +### Client-side setup + +Add the API key to `patchman-client.conf`: + +```shell +protocol=2 +api_key=abc123... +``` + +Or use the `-k` command line option: + +```shell +$ patchman-client -s http://patchman.example.org -p 2 -k abc123... +``` ## Configure Database diff --git a/README.md b/README.md index d425c5fd..11088556 100644 --- a/README.md +++ b/README.md @@ -120,6 +120,9 @@ required to report packages, `yum`, `dnf`, `zypper` and/or `apt` are required to report repositories. These packages are normally installed by default on most systems. +For Protocol 2 (JSON-based reports), `jq` is required. If `jq` is not available, +the client will automatically fall back to Protocol 1 (text-based reports). + deb-based OS's do not always change the kernel version when a kernel update is installed, so the `update-notifier-common` package can optionally be installed to enable this functionality. rpm-based OS's can tell if a reboot is required diff --git a/client/patchman-client b/client/patchman-client index bf4bc5c9..29b3b869 100755 --- a/client/patchman-client +++ b/client/patchman-client @@ -12,9 +12,10 @@ report=false local_updates=false repo_check=true tags='' +api_key='' usage() { - echo "${0} [-v] [-d] [-n] [-u] [-r] [-s SERVER] [-c FILE] [-t TAGS] [-h HOSTNAME]" + echo "${0} [-v] [-d] [-n] [-u] [-r] [-s SERVER] [-c FILE] [-t TAGS] [-h HOSTNAME] [-p PROTOCOL] [-k API_KEY]" echo "-v: verbose output (default is silent)" echo "-d: debug output" echo "-n: no repo check (required when used as an apt or yum plugin)" @@ -24,13 +25,15 @@ usage() { echo "-c FILE: config file location (default is /etc/patchman/patchman-client.conf)" echo "-t TAGS: comma-separated list of tags, e.g. -t www,dev" echo "-h HOSTNAME: specify the hostname of the local host" + echo "-p PROTOCOL: protocol version (1 or 2, default is 1)" + echo "-k API_KEY: API key for protocol 2 authentication" echo echo "Command line options override config file options." exit 0 } parseopts() { - while getopts "vdnurs:c:t:h:" opt; do + while getopts "vdnurs:c:t:h:p:k:" opt; do case ${opt} in v) verbose=true @@ -60,6 +63,12 @@ parseopts() { h) cli_hostname=${OPTARG} ;; + p) + cli_protocol=${OPTARG} + ;; + k) + cli_api_key=${OPTARG} + ;; *) usage ;; @@ -74,6 +83,14 @@ cleanup() { echo "Debug: not deleting ${tmpfile_sec} (security updates)" echo "Debug: not deleting ${tmpfile_bug} (updates)" echo "Debug: not deleting ${tmpfile_mod} (modules)" + if [ ! -z "${tmpfile_packages_json}" ] ; then + echo "Debug: not deleting ${tmpfile_packages_json} (packages json)" + echo "Debug: not deleting ${tmpfile_repos_json} (repos json)" + echo "Debug: not deleting ${tmpfile_modules_json} (modules json)" + echo "Debug: not deleting ${tmpfile_sec_json} (security updates json)" + echo "Debug: not deleting ${tmpfile_bug_json} (bug updates json)" + echo "Debug: not deleting ${tmpfile_report_json} (full report json)" + fi elif ${verbose} && ! ${debug} ; then echo "Deleting ${tmpfile_pkg}" echo "Deleting ${tmpfile_rep}" @@ -87,6 +104,12 @@ cleanup() { rm -fr "${tmpfile_sec}" rm -fr "${tmpfile_bug}" rm -fr "${tmpfile_mod}" + rm -fr "${tmpfile_packages_json}" + rm -fr "${tmpfile_repos_json}" + rm -fr "${tmpfile_modules_json}" + rm -fr "${tmpfile_sec_json}" + rm -fr "${tmpfile_bug_json}" + rm -fr "${tmpfile_report_json}" fi flock -u 200 rm -fr "${lock_dir}/patchman.lock" @@ -130,6 +153,14 @@ check_conf() { tags="${cli_tags}" fi + if [ ! -z "${cli_protocol}" ] ; then + protocol=${cli_protocol} + fi + + if [ ! -z "${cli_api_key}" ] ; then + api_key=${cli_api_key} + fi + if [ -z "${hostname}" ] && [ -z "${cli_hostname}" ] ; then get_hostname else @@ -140,6 +171,14 @@ check_conf() { check_booleans + # Check if protocol 2 is requested but jq is not available + if [ "${protocol}" == "2" ] ; then + if ! check_command_exists jq ; then + echo "Warning: jq not found, falling back to protocol 1" + protocol=1 + fi + fi + if ${verbose} ; then echo "Patchman configuration seems OK:" if [ -f ${conf} ] ; then @@ -148,6 +187,10 @@ check_conf() { echo "Patchman Server: ${server}" echo "Hostname: ${hostname}" echo "Tags: ${tags}" + echo "Protocol: ${protocol}" + if [ ! -z "${api_key}" ] ; then + echo "API Key: ${api_key:0:12}..." + fi for var in report local_updates repo_check verbose debug ; do eval val=\$${var} echo "${var}: ${val}" @@ -642,6 +685,302 @@ reboot_required() { fi } +build_packages_json() { + # Convert packages file to JSON array, writing to temp file + local outfile="${1}" + echo "[" > "${outfile}" + local first=true + while IFS= read -r line ; do + if [ -z "${line}" ] ; then + continue + fi + # Parse: 'name' 'epoch' 'version' 'release' 'arch' 'type' ['category'] ['repo'] + # Extract quoted fields properly + local parts=() + while IFS= read -r part ; do + parts+=("${part}") + done < <(echo "${line}" | grep -oP "'[^']*'" | tr -d "'") + + local name="${parts[0]:-}" + local epoch="${parts[1]:-}" + local version="${parts[2]:-}" + local release="${parts[3]:-}" + local arch="${parts[4]:-}" + local ptype="${parts[5]:-}" + local category="${parts[6]:-}" + local repo="${parts[7]:-}" + + # Skip packages without a valid type + if [ -z "${ptype}" ] ; then + continue + fi + + # Skip gpg-pubkey entries (they're not real packages) + if [ "${name}" == "gpg-pubkey" ] ; then + continue + fi + + if ! ${first} ; then + echo "," >> "${outfile}" + fi + first=false + + jq -n -c \ + --arg name "${name}" \ + --arg epoch "${epoch}" \ + --arg version "${version}" \ + --arg release "${release}" \ + --arg arch "${arch}" \ + --arg type "${ptype}" \ + --arg category "${category}" \ + --arg repo "${repo}" \ + '{name: $name, epoch: $epoch, version: $version, release: $release, arch: $arch, type: $type, category: $category, repo: $repo}' >> "${outfile}" + done < "${tmpfile_pkg}" + echo "]" >> "${outfile}" +} + +build_repos_json() { + # Convert repos file to JSON array, writing to temp file + local outfile="${1}" + echo "[" > "${outfile}" + local first=true + while IFS= read -r line ; do + if [ -z "${line}" ] ; then + continue + fi + # Parse: 'type' 'name' 'id' 'priority' 'url1' ['url2' ...] + local parts=() + while IFS= read -r part ; do + parts+=("${part}") + done < <(echo "${line}" | grep -oP "'[^']*'" | tr -d "'") + + local rtype="${parts[0]}" + local name="${parts[1]}" + local rid="${parts[2]}" + local priority="${parts[3]}" + + # Collect URLs (all remaining parts) + local urls_json="[" + local url_first=true + for ((i=4; i<${#parts[@]}; i++)); do + if ! ${url_first} ; then + urls_json="${urls_json}," + fi + url_first=false + urls_json="${urls_json}\"${parts[i]}\"" + done + urls_json="${urls_json}]" + + if ! ${first} ; then + echo "," >> "${outfile}" + fi + first=false + + jq -n -c \ + --arg type "${rtype}" \ + --arg name "${name}" \ + --arg id "${rid}" \ + --argjson priority "${priority:-0}" \ + --argjson urls "${urls_json}" \ + '{type: $type, name: $name, id: $id, priority: $priority, urls: $urls}' >> "${outfile}" + done < "${tmpfile_rep}" + echo "]" >> "${outfile}" +} + +build_modules_json() { + # Convert modules file to JSON array, writing to temp file + local outfile="${1}" + echo "[" > "${outfile}" + local first=true + while IFS= read -r line ; do + if [ -z "${line}" ] ; then + continue + fi + # Parse: 'name' 'stream' 'version' 'context' 'arch' 'repo' ['pkg1' 'pkg2' ...] + local parts=() + while IFS= read -r part ; do + parts+=("${part}") + done < <(echo "${line}" | grep -oP "'[^']*'" | tr -d "'") + + local name="${parts[0]}" + local stream="${parts[1]}" + local version="${parts[2]}" + local context="${parts[3]}" + local arch="${parts[4]}" + local repo="${parts[5]}" + + # Collect packages (all remaining parts) + local pkgs_json="[" + local pkg_first=true + for ((i=6; i<${#parts[@]}; i++)); do + if ! ${pkg_first} ; then + pkgs_json="${pkgs_json}," + fi + pkg_first=false + pkgs_json="${pkgs_json}\"${parts[i]}\"" + done + pkgs_json="${pkgs_json}]" + + if ! ${first} ; then + echo "," >> "${outfile}" + fi + first=false + + jq -n -c \ + --arg name "${name}" \ + --arg stream "${stream}" \ + --arg version "${version}" \ + --arg context "${context}" \ + --arg arch "${arch}" \ + --arg repo "${repo}" \ + --argjson packages "${pkgs_json}" \ + '{name: $name, stream: $stream, version: $version, context: $context, arch: $arch, repo: $repo, packages: $packages}' >> "${outfile}" + done < "${tmpfile_mod}" + echo "]" >> "${outfile}" +} + +build_updates_json() { + # Convert updates file to JSON array, writing to temp file + local updates_file="${1}" + local outfile="${2}" + echo "[" > "${outfile}" + local first=true + while IFS= read -r line ; do + if [ -z "${line}" ] ; then + continue + fi + # Parse: name.arch version repo + local parts=(${line}) + local name_arch="${parts[0]}" + local version="${parts[1]}" + local repo="${parts[2]}" + + # Split name.arch + local name="${name_arch%.*}" + local arch="${name_arch##*.}" + + if ! ${first} ; then + echo "," >> "${outfile}" + fi + first=false + + jq -n -c \ + --arg name "${name}" \ + --arg version "${version}" \ + --arg arch "${arch}" \ + --arg repo "${repo}" \ + '{name: $name, version: $version, arch: $arch, repo: $repo}' >> "${outfile}" + done < "${updates_file}" + echo "]" >> "${outfile}" +} + +build_json_report() { + # Build complete JSON report using temp files + # Temp files are created by post_json_data before calling this function + + build_packages_json "${tmpfile_packages_json}" + build_repos_json "${tmpfile_repos_json}" + build_modules_json "${tmpfile_modules_json}" + build_updates_json "${tmpfile_sec}" "${tmpfile_sec_json}" + build_updates_json "${tmpfile_bug}" "${tmpfile_bug_json}" + + # Convert tags to JSON array + local tags_json="[]" + if [ ! -z "${tags}" ] && [ "${tags}" != "Default" ] ; then + tags_json=$(echo "${tags}" | tr ',' '\n' | jq -R . | jq -s .) + fi + + # Convert reboot to boolean + local reboot_bool="false" + if [ "${reboot}" == "True" ] ; then + reboot_bool="true" + fi + + jq -n \ + --argjson protocol 2 \ + --arg hostname "${hostname}" \ + --arg arch "${host_arch}" \ + --arg kernel "${host_kernel}" \ + --arg os "${os}" \ + --argjson tags "${tags_json}" \ + --argjson reboot_required "${reboot_bool}" \ + --slurpfile packages "${tmpfile_packages_json}" \ + --slurpfile repos "${tmpfile_repos_json}" \ + --slurpfile modules "${tmpfile_modules_json}" \ + --slurpfile sec_updates "${tmpfile_sec_json}" \ + --slurpfile bug_updates "${tmpfile_bug_json}" \ + '{ + protocol: $protocol, + hostname: $hostname, + arch: $arch, + kernel: $kernel, + os: $os, + tags: $tags, + reboot_required: $reboot_required, + packages: $packages[0], + repos: $repos[0], + modules: $modules[0], + sec_updates: $sec_updates[0], + bug_updates: $bug_updates[0] + }' +} + +post_json_data() { + # Post data using protocol 2 (JSON) + curl_opts=${curl_options} + + if ${verbose} ; then + curl_opts="${curl_opts} -i" + echo "Sending JSON data to ${server} with curl (protocol 2):" + else + curl_opts="${curl_opts} -s -S" + fi + + # Create temp files for JSON (global so cleanup can handle them) + tmpfile_packages_json=$(mktemp) + tmpfile_repos_json=$(mktemp) + tmpfile_modules_json=$(mktemp) + tmpfile_sec_json=$(mktemp) + tmpfile_bug_json=$(mktemp) + tmpfile_report_json=$(mktemp) + + # Build JSON report to temp file + build_json_report > "${tmpfile_report_json}" + + if ${debug} ; then + echo "JSON Report:" + jq . "${tmpfile_report_json}" + fi + + curl_opts="${curl_opts} -H 'Content-Type: application/json'" + if [ ! -z "${api_key}" ] ; then + curl_opts="${curl_opts} -H 'Authorization: Api-Key ${api_key}'" + fi + curl_opts="${curl_opts} -d @${tmpfile_report_json}" + + post_command="curl ${curl_opts} ${server%/}/api/report/" + + if ${verbose} ; then + echo "${post_command}" + fi + + result=$(eval "${post_command}") + retval=${?} + + if [ ! ${retval} -eq 0 ] ; then + echo 'Failed to upload report.' + exit ${retval} + fi + + if ${report} || ${verbose} ; then + if [ ! -z "${result}" ] ; then + echo "${result}" | jq . 2>/dev/null || echo "${result}" + else + echo "No output returned." + fi + fi +} + post_data() { curl_opts=${curl_options} @@ -736,4 +1075,10 @@ if ${repo_check} ; then get_repos fi reboot_required -post_data + +# Use protocol 2 (JSON) or protocol 1 (form data) based on config +if [ "${protocol}" == "2" ] ; then + post_json_data +else + post_data +fi diff --git a/client/patchman-client.conf b/client/patchman-client.conf index fdca5663..666dd54c 100644 --- a/client/patchman-client.conf +++ b/client/patchman-client.conf @@ -9,3 +9,10 @@ tags="Server" # Does the client output a report of the upload (e.g. for cronjob output) report=false + +# Protocol version (1 = text, 2 = json) +# Protocol 2 requires jq to be installed +protocol=1 + +# API key for protocol 2 authentication +#api_key=pm_your_api_key_here diff --git a/debian/control b/debian/control index 34610549..53121399 100644 --- a/debian/control +++ b/debian/control @@ -15,7 +15,8 @@ Architecture: all Homepage: https://github.com/furlongm/patchman Depends: ${misc:Depends}, python3 (>= 3.11), python3-django (>= 4.2), python3-django-extensions, python3-django-bootstrap3, python3-cvss, - python3-djangorestframework, python3-django-filters, python3-debian, + python3-djangorestframework, python3-djangorestframework-api-key, + python3-django-filters, python3-debian, python3-rpm, python3-tqdm, python3-defusedxml, python3-pip, python3-tenacity, python3-requests, python3-colorama, python3-magic, python3-humanize, python3-yaml, libapache2-mod-wsgi-py3, apache2, sqlite3, @@ -44,7 +45,7 @@ Description: Django-based patch status monitoring tool for linux systems. Package: patchman-client Architecture: all Homepage: https://github.com/furlongm/patchman -Depends: ${misc:Depends}, curl, debianutils, util-linux, coreutils, mawk +Depends: ${misc:Depends}, curl, debianutils, util-linux, coreutils, mawk, jq Description: Client for the patchman monitoring system. . The client will send a list of packages and repositories to the upstream diff --git a/patchman-client.spec b/patchman-client.spec index f2f8279a..ed04ac12 100644 --- a/patchman-client.spec +++ b/patchman-client.spec @@ -8,7 +8,7 @@ License: GPLv3 URL: http://patchman.openbytes.ie Source: %{expand:%%(pwd)} BuildArch: noarch -Requires: curl which coreutils util-linux gawk +Requires: curl which coreutils util-linux gawk jq %define _binary_payload w9.gzdio diff --git a/patchman/settings.py b/patchman/settings.py index 76c28b3f..20ac82e8 100644 --- a/patchman/settings.py +++ b/patchman/settings.py @@ -100,15 +100,24 @@ 'security.apps.SecurityConfig', 'reports.apps.ReportsConfig', 'util.apps.UtilConfig', + 'rest_framework_api_key', ] REST_FRAMEWORK = { 'DEFAULT_PERMISSION_CLASSES': ['rest_framework.permissions.IsAuthenticatedOrReadOnly'], # noqa + 'DEFAULT_AUTHENTICATION_CLASSES': [ + 'rest_framework.authentication.SessionAuthentication', + 'rest_framework.authentication.BasicAuthentication', + ], 'DEFAULT_FILTER_BACKENDS': ['django_filters.rest_framework.DjangoFilterBackend'], # noqa 'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.PageNumberPagination', # noqa 'PAGE_SIZE': 100, } +# API Key authentication settings +# Set to False to allow unauthenticated protocol 2 report uploads +REQUIRE_API_KEY = True + TAGGIT_CASE_INSENSITIVE = True DJANGO_TABLES2_TEMPLATE = 'table.html' diff --git a/patchman/urls.py b/patchman/urls.py index b66a0de3..4cc22900 100644 --- a/patchman/urls.py +++ b/patchman/urls.py @@ -28,6 +28,7 @@ from hosts import views as host_views from operatingsystems import views as os_views from packages import views as package_views +from reports import views as report_views from repos import views as repo_views from security import views as security_views @@ -48,6 +49,7 @@ router.register(r'repo', repo_views.RepositoryViewSet) router.register(r'mirror', repo_views.MirrorViewSet) router.register(r'mirror-package', repo_views.MirrorPackageViewSet) +router.register(r'report', report_views.ReportViewSet, basename='report') admin.autodiscover() diff --git a/reports/models.py b/reports/models.py index f1e0f6f3..bda18fed 100644 --- a/reports/models.py +++ b/reports/models.py @@ -15,6 +15,8 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see +import json + from django.db import models from django.urls import reverse @@ -53,6 +55,91 @@ def __str__(self): def get_absolute_url(self): return reverse('reports:report_detail', args=[str(self.id)]) + @property + def packages_parsed(self): + """Parse packages JSON for Protocol 2 reports.""" + if self.protocol == '2' and self.packages: + try: + return json.loads(self.packages) + except json.JSONDecodeError: + return [] + return [] + + @property + def repos_parsed(self): + """Parse repos JSON for Protocol 2 reports.""" + if self.protocol == '2' and self.repos: + try: + return json.loads(self.repos) + except json.JSONDecodeError: + return [] + return [] + + @property + def modules_parsed(self): + """Parse modules JSON for Protocol 2 reports.""" + if self.protocol == '2' and self.modules: + try: + return json.loads(self.modules) + except json.JSONDecodeError: + return [] + return [] + + @property + def sec_updates_parsed(self): + """Parse security updates JSON for Protocol 2 reports.""" + if self.protocol == '2' and self.sec_updates: + try: + return json.loads(self.sec_updates) + except json.JSONDecodeError: + return [] + return [] + + @property + def bug_updates_parsed(self): + """Parse bug updates JSON for Protocol 2 reports.""" + if self.protocol == '2' and self.bug_updates: + try: + return json.loads(self.bug_updates) + except json.JSONDecodeError: + return [] + return [] + + @property + def has_packages(self): + """Check if report has packages data.""" + if self.protocol == '2': + return bool(self.packages_parsed) + return bool(self.packages and self.packages.strip()) + + @property + def has_repos(self): + """Check if report has repos data.""" + if self.protocol == '2': + return bool(self.repos_parsed) + return bool(self.repos and self.repos.strip()) + + @property + def has_modules(self): + """Check if report has modules data.""" + if self.protocol == '2': + return bool(self.modules_parsed) + return bool(self.modules and self.modules.strip()) + + @property + def has_sec_updates(self): + """Check if report has security updates.""" + if self.protocol == '2': + return bool(self.sec_updates_parsed) + return bool(self.sec_updates and self.sec_updates.strip()) + + @property + def has_bug_updates(self): + """Check if report has bug updates.""" + if self.protocol == '2': + return bool(self.bug_updates_parsed) + return bool(self.bug_updates and self.bug_updates.strip()) + def parse(self, data, meta): """ Parse a report and save the object """ @@ -113,13 +200,34 @@ def process(self, find_updates=True, verbose=False): if verbose: info_message(text=f'Processing report {self.id} - {self.host}') - from reports.utils import ( - process_modules, process_packages, process_repos, process_updates, - ) - process_repos(report=self, host=host) - process_modules(report=self, host=host) - process_packages(report=self, host=host) - process_updates(report=self, host=host) + if self.protocol == '2': + # Protocol 2: JSON data + import json + + from reports.utils import ( + process_modules_json, process_packages_json, + process_repos_json, process_updates_json, + ) + packages_json = json.loads(self.packages) if self.packages else [] + repos_json = json.loads(self.repos) if self.repos else [] + modules_json = json.loads(self.modules) if self.modules else [] + sec_updates_json = json.loads(self.sec_updates) if self.sec_updates else [] + bug_updates_json = json.loads(self.bug_updates) if self.bug_updates else [] + + process_repos_json(repos_json, host, self.arch) + process_modules_json(modules_json, host) + process_packages_json(packages_json, host) + process_updates_json(sec_updates_json, bug_updates_json, host) + else: + # Protocol 1: Text data + from reports.utils import ( + process_modules, process_packages, process_repos, + process_updates, + ) + process_repos(report=self, host=host) + process_modules(report=self, host=host) + process_packages(report=self, host=host) + process_updates(report=self, host=host) self.processed = True self.save() diff --git a/reports/serializers.py b/reports/serializers.py new file mode 100644 index 00000000..6822afa7 --- /dev/null +++ b/reports/serializers.py @@ -0,0 +1,91 @@ +# Copyright 2013-2025 Marcus Furlong +# +# This file is part of Patchman. +# +# Patchman is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, version 3 only. +# +# Patchman is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Patchman. If not, see + +from rest_framework import serializers + + +class PackageSerializer(serializers.Serializer): + """Serializer for a single package in a report.""" + name = serializers.CharField(max_length=255) + epoch = serializers.CharField(max_length=255, required=False, allow_blank=True, default='') + version = serializers.CharField(max_length=255) + release = serializers.CharField(max_length=255, required=False, allow_blank=True, default='') + arch = serializers.CharField(max_length=255) + type = serializers.ChoiceField(choices=['deb', 'rpm', 'arch', 'gentoo']) + # Gentoo-specific fields + category = serializers.CharField(max_length=255, required=False, allow_blank=True, default='') + repo = serializers.CharField(max_length=255, required=False, allow_blank=True, default='') + + +class RepoSerializer(serializers.Serializer): + """Serializer for a single repository in a report.""" + type = serializers.ChoiceField(choices=['deb', 'rpm', 'arch', 'gentoo']) + name = serializers.CharField(max_length=255) + id = serializers.CharField(max_length=255, required=False, allow_blank=True, default='') + priority = serializers.IntegerField(required=False, default=0) + urls = serializers.ListField( + child=serializers.URLField(max_length=512), + required=False, + default=list + ) + + +class ModuleSerializer(serializers.Serializer): + """Serializer for a single module in a report.""" + name = serializers.CharField(max_length=255) + stream = serializers.CharField(max_length=255) + version = serializers.CharField(max_length=255) + context = serializers.CharField(max_length=255) + arch = serializers.CharField(max_length=255) + repo = serializers.CharField(max_length=255, required=False, allow_blank=True, default='') + packages = serializers.ListField( + child=serializers.CharField(max_length=255), + required=False, + default=list + ) + + +class UpdateSerializer(serializers.Serializer): + """Serializer for a single update (security or bugfix) in a report.""" + name = serializers.CharField(max_length=255) + version = serializers.CharField(max_length=255) + arch = serializers.CharField(max_length=255) + repo = serializers.CharField(max_length=255, required=False, allow_blank=True, default='') + + +class ReportUploadSerializer(serializers.Serializer): + """Serializer for protocol 2 JSON report uploads.""" + protocol = serializers.IntegerField(default=2) + hostname = serializers.CharField(max_length=255) + arch = serializers.CharField(max_length=255) + kernel = serializers.CharField(max_length=255) + os = serializers.CharField(max_length=255) + tags = serializers.ListField( + child=serializers.CharField(max_length=255), + required=False, + default=list + ) + reboot_required = serializers.BooleanField(required=False, default=False) + packages = PackageSerializer(many=True, required=False, default=list) + repos = RepoSerializer(many=True, required=False, default=list) + modules = ModuleSerializer(many=True, required=False, default=list) + sec_updates = UpdateSerializer(many=True, required=False, default=list) + bug_updates = UpdateSerializer(many=True, required=False, default=list) + + def validate_protocol(self, value): + if value != 2: + raise serializers.ValidationError('This endpoint only accepts protocol 2') + return value diff --git a/reports/tables.py b/reports/tables.py index 52f077e0..bc43e49e 100644 --- a/reports/tables.py +++ b/reports/tables.py @@ -58,3 +58,57 @@ class ReportTable(BaseTable): class Meta(BaseTable.Meta): model = Report fields = ('selection', 'report_id', 'host', 'created', 'report_ip', 'processed') + + +class ReportPackageTable(tables.Table): + """Table for displaying packages in Protocol 2 reports.""" + name = tables.Column() + epoch = tables.Column(default='') + version = tables.Column() + release = tables.Column() + arch = tables.Column() + type = tables.Column() + + class Meta: + attrs = {'class': 'table table-striped table-bordered table-hover table-condensed'} + orderable = True + + +class ReportRepoTable(tables.Table): + """Table for displaying repos in Protocol 2 reports.""" + type = tables.Column() + name = tables.Column() + id = tables.Column(verbose_name='ID') + priority = tables.Column() + + class Meta: + attrs = {'class': 'table table-striped table-bordered table-hover table-condensed'} + orderable = True + + +class ReportModuleTable(tables.Table): + """Table for displaying modules in Protocol 2 reports.""" + name = tables.Column() + stream = tables.Column() + version = tables.Column() + context = tables.Column() + arch = tables.Column() + repo = tables.Column(default='') + + class Meta: + attrs = {'class': 'table table-striped table-bordered table-hover table-condensed'} + orderable = True + + +class ReportUpdateTable(tables.Table): + """Table for displaying updates in Protocol 2 reports.""" + name = tables.Column() + epoch = tables.Column(default='') + version = tables.Column() + release = tables.Column() + arch = tables.Column() + repo = tables.Column(default='') + + class Meta: + attrs = {'class': 'table table-striped table-bordered table-hover table-condensed'} + orderable = True diff --git a/reports/templates/reports/report_detail.html b/reports/templates/reports/report_detail.html index 08d5005f..b68122bd 100644 --- a/reports/templates/reports/report_detail.html +++ b/reports/templates/reports/report_detail.html @@ -1,6 +1,6 @@ {% extends "base.html" %} -{% load bootstrap3 common %} +{% load bootstrap3 common django_tables2 %} {% block page_title %}Report - {{ report }} {% endblock %} @@ -12,17 +12,21 @@
    @@ -48,71 +52,95 @@
    -
    -
    - - {% for repo in report.repos.splitlines %} - - - - {% endfor %} -
    {{ repo }}
    + {% if report.has_repos %} +
    +
    + {% if repos_table %} + {% render_table repos_table %} + {% else %} + + {% for repo in report.repos.splitlines %} + + + + {% endfor %} +
    {{ repo }}
    + {% endif %} +
    -
    + {% endif %} - {% if report.modules %} + {% if report.has_modules %}
    - - {% for module in report.modules.splitlines %} - - - - {% endfor %} -
    {{ module }}
    + {% if modules_table %} + {% render_table modules_table %} + {% else %} + + {% for module in report.modules.splitlines %} + + + + {% endfor %} +
    {{ module }}
    + {% endif %}
    {% endif %} - {% if report.sec_updates %} + {% if report.has_sec_updates %}
    - - {% for update in report.sec_updates.splitlines %} - - - - {% endfor %} -
    {{ update }}
    + {% if sec_updates_table %} + {% render_table sec_updates_table %} + {% else %} + + {% for update in report.sec_updates.splitlines %} + + + + {% endfor %} +
    {{ update }}
    + {% endif %}
    {% endif %} - {% if report.bug_updates %} + {% if report.has_bug_updates %}
    - - {% for update in report.bug_updates.splitlines %} - - - - {% endfor %} -
    {{ update }}
    + {% if bug_updates_table %} + {% render_table bug_updates_table %} + {% else %} + + {% for update in report.bug_updates.splitlines %} + + + + {% endfor %} +
    {{ update }}
    + {% endif %}
    {% endif %} -
    -
    - - {% for package in report.packages.splitlines %} - - - - {% endfor %} -
    {{ package }}
    + {% if report.has_packages %} +
    +
    + {% if packages_table %} + {% render_table packages_table %} + {% else %} + + {% for package in report.packages.splitlines %} + + + + {% endfor %} +
    {{ package }}
    + {% endif %} +
    -
    + {% endif %}
    {% endblock %} diff --git a/reports/tests.py b/reports/tests.py new file mode 100644 index 00000000..c75fe305 --- /dev/null +++ b/reports/tests.py @@ -0,0 +1,291 @@ +# Copyright 2025 Marcus Furlong +# +# This file is part of Patchman. +# +# Patchman is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, version 3 only. +# +# Patchman is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Patchman. If not, see + +from django.test import TestCase, override_settings +from rest_framework import status +from rest_framework.test import APITestCase +from rest_framework_api_key.models import APIKey + +from reports.models import Report + + +@override_settings( + REQUIRE_API_KEY=False, + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class ReportAPITests(APITestCase): + """Tests for the Protocol 2 JSON report API.""" + + def setUp(self): + self.url = '/api/report/' + + def test_upload_minimal_report(self): + """Test uploading a minimal valid report.""" + data = { + 'protocol': 2, + 'hostname': 'testhost.example.com', + 'arch': 'x86_64', + 'kernel': '5.15.0-91-generic', + 'os': 'Ubuntu 22.04.3 LTS', + } + response = self.client.post(self.url, data, format='json') + self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED) + self.assertEqual(response.data['status'], 'accepted') + self.assertIn('report_id', response.data) + + # Verify report was created + report = Report.objects.get(id=response.data['report_id']) + self.assertEqual(report.host, 'testhost.example.com') + self.assertEqual(report.protocol, '2') + + def test_upload_full_report(self): + """Test uploading a report with all fields.""" + data = { + 'protocol': 2, + 'hostname': 'server1.example.com', + 'arch': 'x86_64', + 'kernel': '5.15.0-91-generic', + 'os': 'Ubuntu 22.04.3 LTS', + 'tags': ['web', 'production'], + 'reboot_required': True, + 'packages': [ + { + 'name': 'nginx', + 'epoch': '', + 'version': '1.18.0', + 'release': '6ubuntu14', + 'arch': 'amd64', + 'type': 'deb' + }, + { + 'name': 'curl', + 'epoch': '', + 'version': '7.81.0', + 'release': '1ubuntu1.15', + 'arch': 'amd64', + 'type': 'deb' + } + ], + 'repos': [ + { + 'type': 'deb', + 'name': 'Ubuntu 22.04 amd64 main', + 'id': 'ubuntu-main', + 'priority': 500, + 'urls': ['http://archive.ubuntu.com/ubuntu/dists/jammy/main/binary-amd64'] + } + ], + 'modules': [], + 'sec_updates': [ + { + 'name': 'openssl', + 'version': '3.0.2-0ubuntu1.13', + 'arch': 'amd64', + 'repo': 'ubuntu-security' + } + ], + 'bug_updates': [] + } + response = self.client.post(self.url, data, format='json') + self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED) + + report = Report.objects.get(id=response.data['report_id']) + self.assertEqual(report.host, 'server1.example.com') + self.assertEqual(report.tags, 'web,production') + self.assertEqual(report.reboot, 'True') + + def test_upload_missing_required_field(self): + """Test that missing required fields return 400.""" + data = { + 'protocol': 2, + 'hostname': 'testhost.example.com', + # missing arch, kernel, os + } + response = self.client.post(self.url, data, format='json') + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertEqual(response.data['status'], 'error') + self.assertIn('errors', response.data) + + def test_upload_wrong_protocol(self): + """Test that wrong protocol version returns 400.""" + data = { + 'protocol': 1, # Should be 2 + 'hostname': 'testhost.example.com', + 'arch': 'x86_64', + 'kernel': '5.15.0', + 'os': 'Ubuntu 22.04', + } + response = self.client.post(self.url, data, format='json') + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + + def test_hostname_lowercased(self): + """Test that hostname is lowercased.""" + data = { + 'protocol': 2, + 'hostname': 'TESTHOST.EXAMPLE.COM', + 'arch': 'x86_64', + 'kernel': '5.15.0', + 'os': 'Ubuntu 22.04', + } + response = self.client.post(self.url, data, format='json') + self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED) + + report = Report.objects.get(id=response.data['report_id']) + self.assertEqual(report.host, 'testhost.example.com') + + def test_domain_extracted_from_hostname(self): + """Test that domain is extracted from FQDN.""" + data = { + 'protocol': 2, + 'hostname': 'server1.prod.example.com', + 'arch': 'x86_64', + 'kernel': '5.15.0', + 'os': 'Ubuntu 22.04', + } + response = self.client.post(self.url, data, format='json') + self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED) + + report = Report.objects.get(id=response.data['report_id']) + self.assertEqual(report.domain, 'prod.example.com') + + def test_invalid_package_type(self): + """Test that invalid package type returns 400.""" + data = { + 'protocol': 2, + 'hostname': 'testhost.example.com', + 'arch': 'x86_64', + 'kernel': '5.15.0', + 'os': 'Ubuntu 22.04', + 'packages': [ + { + 'name': 'nginx', + 'version': '1.18.0', + 'arch': 'amd64', + 'type': 'invalid' # Invalid type + } + ] + } + response = self.client.post(self.url, data, format='json') + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + + +class ReportSerializerTests(TestCase): + """Tests for report serializers.""" + + def test_package_serializer_valid(self): + from reports.serializers import PackageSerializer + data = { + 'name': 'nginx', + 'epoch': '', + 'version': '1.18.0', + 'release': '6ubuntu14', + 'arch': 'amd64', + 'type': 'deb' + } + serializer = PackageSerializer(data=data) + self.assertTrue(serializer.is_valid()) + + def test_package_serializer_minimal(self): + from reports.serializers import PackageSerializer + data = { + 'name': 'nginx', + 'version': '1.18.0', + 'arch': 'amd64', + 'type': 'deb' + } + serializer = PackageSerializer(data=data) + self.assertTrue(serializer.is_valid()) + + def test_repo_serializer_valid(self): + from reports.serializers import RepoSerializer + data = { + 'type': 'deb', + 'name': 'Ubuntu Main', + 'id': 'ubuntu-main', + 'priority': 500, + 'urls': ['http://archive.ubuntu.com/ubuntu'] + } + serializer = RepoSerializer(data=data) + self.assertTrue(serializer.is_valid()) + + def test_module_serializer_valid(self): + from reports.serializers import ModuleSerializer + data = { + 'name': 'nodejs', + 'stream': '18', + 'version': '8090020240101', + 'context': 'rhel9', + 'arch': 'x86_64', + 'repo': 'appstream', + 'packages': ['nodejs-18.19.0-1.module+el9'] + } + serializer = ModuleSerializer(data=data) + self.assertTrue(serializer.is_valid()) + + +@override_settings( + REQUIRE_API_KEY=True, + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class ApiKeyAuthTests(APITestCase): + """Tests for API key authentication using djangorestframework-api-key.""" + + def setUp(self): + self.url = '/api/report/' + self.api_key_obj, self.api_key = APIKey.objects.create_key(name='test-key') + self.valid_data = { + 'protocol': 2, + 'hostname': 'testhost.example.com', + 'arch': 'x86_64', + 'kernel': '5.15.0', + 'os': 'Ubuntu 22.04', + } + + def test_valid_api_key(self): + """Test that valid API key authenticates.""" + self.client.credentials(HTTP_AUTHORIZATION=f'Api-Key {self.api_key}') + response = self.client.post(self.url, self.valid_data, format='json') + self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED) + + def test_invalid_api_key(self): + """Test that invalid API key returns 403.""" + self.client.credentials(HTTP_AUTHORIZATION='Api-Key invalid_key') + response = self.client.post(self.url, self.valid_data, format='json') + # drf-api-key returns 403 for invalid keys when using HasAPIKey permission + self.assertIn(response.status_code, [status.HTTP_401_UNAUTHORIZED, status.HTTP_403_FORBIDDEN]) + + def test_revoked_api_key(self): + """Test that revoked API key returns 403.""" + self.api_key_obj.revoked = True + self.api_key_obj.save() + self.client.credentials(HTTP_AUTHORIZATION=f'Api-Key {self.api_key}') + response = self.client.post(self.url, self.valid_data, format='json') + self.assertIn(response.status_code, [status.HTTP_401_UNAUTHORIZED, status.HTTP_403_FORBIDDEN]) + + def test_missing_api_key_requires_auth(self): + """Test that missing API key returns 403 when REQUIRE_API_KEY is True.""" + # No credentials set + response = self.client.post(self.url, self.valid_data, format='json') + self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) + + @override_settings(REQUIRE_API_KEY=False) + def test_no_auth_allowed_when_disabled(self): + """Test that no auth is allowed when REQUIRE_API_KEY is False.""" + # No credentials set + response = self.client.post(self.url, self.valid_data, format='json') + self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED) diff --git a/reports/utils.py b/reports/utils.py index 1a08cf3c..85cf6e3b 100644 --- a/reports/utils.py +++ b/reports/utils.py @@ -48,7 +48,7 @@ def process_repos(report, host): pbar_start.send(sender=None, ptext=f'{host} Repos', plen=len(repos)) for i, repo_str in enumerate(repos): debug_message(f'Processing report {report.id} repo: {repo_str}') - repo, priority = process_repo(repo_str, report.arch) + repo, priority = process_repo_text(repo_str, report.arch) if repo: repo_ids.append(repo.id) try: @@ -74,7 +74,7 @@ def process_modules(report, host): pbar_start.send(sender=None, ptext=f'{host} Modules', plen=len(modules)) for i, module_str in enumerate(modules): - module = process_module(module_str) + module = process_module_text(module_str) if module: module_ids.append(module.id) host.modules.add(module) @@ -95,7 +95,7 @@ def process_packages(report, host): pbar_start.send(sender=None, ptext=f'{host} Packages', plen=len(packages)) for i, pkg_str in enumerate(packages): debug_message(f'Processing report {report.id} package: {pkg_str}') - package = process_package(pkg_str, report.protocol) + package = process_package_text(pkg_str) if package: package_ids.append(package.id) host.packages.add(package) @@ -142,7 +142,7 @@ def add_updates(updates, host): if ulen > 0: pbar_start.send(sender=None, ptext=f'{host} Updates', plen=ulen) for i, (u, sec) in enumerate(updates.items()): - update = process_update(host, u, sec) + update = process_update_text(host, u, sec) if update: host.updates.add(update) pbar_update.send(sender=None, index=i + 1) @@ -161,7 +161,7 @@ def parse_updates(updates_string, security): return updates -def process_update(host, update_string, security): +def process_update_text(host, update_string, security): """ Processes a single sanitized update string and converts to an update object. Only works if the original package exists. Returns None otherwise """ @@ -173,12 +173,19 @@ def process_update(host, update_string, security): p_arch = parts[2] p_epoch, p_version, p_release = find_evr(update_str[1]) + + return process_update(host, p_name, p_epoch, p_version, p_release, p_arch, repo_id, security) + + +def process_update(host, name, epoch, version, release, arch, repo_id, security): + """ Core update processing logic shared by text and JSON handlers + """ package = get_or_create_package( - name=p_name, - epoch=p_epoch, - version=p_version, - release=p_release, - arch=p_arch, + name=name, + epoch=epoch, + version=version, + release=release, + arch=arch, p_type=Package.RPM ) try: @@ -194,6 +201,7 @@ def process_update(host, update_string, security): installed_package = installed_packages[0] update = get_or_create_package_update(oldpackage=installed_package, newpackage=package, security=security) return update + return None def parse_repos(repos_string): @@ -208,35 +216,30 @@ def parse_repos(repos_string): return repos -def process_repo(repo, arch): - """ Processes a single sanitized repo string and converts to a repo object +def _get_repo_type(type_str): + """ Convert repo type string to Repository constant """ - repository = r_id = None - - if repo[0] == 'deb': - r_type = Repository.DEB - r_priority = int(repo[2]) - elif repo[0] == 'rpm': - r_type = Repository.RPM - r_id = repo.pop(2) - r_priority = int(repo[2]) * -1 - elif repo[0] == 'arch': - r_type = Repository.ARCH - r_id = repo[2] - r_priority = 0 - elif repo[0] == 'gentoo': - r_type = Repository.GENTOO - r_id = repo.pop(2) - r_priority = repo[2] - arch = 'any' - - if repo[1]: - r_name = repo[1] - - r_arch, c = MachineArchitecture.objects.get_or_create(name=arch) + type_str = type_str.lower() + if type_str == 'deb': + return Repository.DEB + elif type_str == 'rpm': + return Repository.RPM + elif type_str == 'arch': + return Repository.ARCH + elif type_str == 'gentoo': + return Repository.GENTOO + return None + + +def process_repo(r_type, r_name, r_id, r_priority, urls, arch): + """ Core repo processing logic shared by text and JSON handlers + """ + r_arch, _ = MachineArchitecture.objects.get_or_create(name=arch) + repository = None unknown = [] - for r_url in repo[3:]: + + for r_url in urls: if r_type == Repository.GENTOO and r_url.startswith('rsync'): r_url = 'https://api.gentoo.org/mirrors/distfiles.xml' try: @@ -248,6 +251,7 @@ def process_repo(repo, arch): unknown.append(r_url) else: repository = mirror.repo + if not repository: repository = get_or_create_repo(r_name, r_arch, r_type) @@ -272,6 +276,36 @@ def process_repo(repo, arch): return repository, r_priority +def process_repo_text(repo, arch): + """ Processes a single sanitized repo string and converts to a repo object + """ + r_id = None + + if repo[0] == 'deb': + r_type = Repository.DEB + r_priority = int(repo[2]) + elif repo[0] == 'rpm': + r_type = Repository.RPM + r_id = repo.pop(2) + r_priority = int(repo[2]) * -1 + elif repo[0] == 'arch': + r_type = Repository.ARCH + r_id = repo[2] + r_priority = 0 + elif repo[0] == 'gentoo': + r_type = Repository.GENTOO + r_id = repo.pop(2) + r_priority = repo[2] + arch = 'any' + else: + return None, 0 + + r_name = repo[1] if repo[1] else '' + urls = repo[3:] + + return process_repo(r_type, r_name, r_id, r_priority, urls, arch) + + def parse_modules(modules_string): """ Parses modules string in a report and returns a sanitized version """ @@ -283,17 +317,10 @@ def parse_modules(modules_string): return modules -def process_module(module_str): - """ Processes a single sanitized module string and converts to a module +def process_module(m_name, m_stream, m_version, m_context, m_arch, repo_id, package_strings): + """ Core module processing logic shared by text and JSON handlers """ - m_name = module_str[0] - m_stream = module_str[1] - m_version = module_str[2] - m_context = module_str[3] - m_arch = module_str[4] - repo_id = module_str[5] - - arch, c = PackageArchitecture.objects.get_or_create(name=m_arch) + arch, _ = PackageArchitecture.objects.get_or_create(name=m_arch) try: repo = Repository.objects.get(repo_id=repo_id) @@ -301,7 +328,7 @@ def process_module(module_str): repo = None packages = set() - for pkg_str in module_str[6:]: + for pkg_str in package_strings: p_type = Package.RPM p_name, p_epoch, p_ver, p_rel, p_dist, p_arch = parse_package_string(pkg_str) package = get_or_create_package(p_name, p_epoch, p_ver, p_rel, p_arch, p_type) @@ -313,6 +340,20 @@ def process_module(module_str): return module +def process_module_text(module_str): + """ Processes a single sanitized module string and converts to a module + """ + m_name = module_str[0] + m_stream = module_str[1] + m_version = module_str[2] + m_context = module_str[3] + m_arch = module_str[4] + repo_id = module_str[5] + package_strings = module_str[6:] + + return process_module(m_name, m_stream, m_version, m_context, m_arch, repo_id, package_strings) + + def parse_packages(packages_string): """ Parses packages string in a report and returns a sanitized version """ @@ -322,42 +363,60 @@ def parse_packages(packages_string): return packages -def process_package(pkg, protocol): +def _get_package_type(type_str): + """ Convert package type string to Package constant + """ + type_str = type_str.lower() if type_str else '' + if type_str == 'deb': + return Package.DEB + elif type_str == 'rpm': + return Package.RPM + elif type_str == 'arch': + return Package.ARCH + elif type_str == 'gentoo': + return Package.GENTOO + return Package.UNKNOWN + + +def process_package(name, epoch, version, release, arch, p_type, category=None, repo=None): + """ Core package processing logic shared by text and JSON handlers + """ + package = get_or_create_package(name, epoch, version, release, arch, p_type) + if p_type == Package.GENTOO and category: + process_gentoo_package(package, name, category, repo) + return package + + +def process_package_text(pkg): """ Processes a single sanitized package string and converts to a package object """ - if protocol == '1': - epoch = ver = rel = '' - arch = 'unknown' - - name = pkg[0] - p_category = p_repo = None - if pkg[1]: - epoch = pkg[1] - if pkg[2]: - ver = pkg[2] - if pkg[3]: - rel = pkg[3] - if pkg[4]: - arch = pkg[4] - - if pkg[5] == 'deb': - p_type = Package.DEB - elif pkg[5] == 'rpm': - p_type = Package.RPM - elif pkg[5] == 'arch': - p_type = Package.ARCH - elif pkg[5] == 'gentoo': - p_type = Package.GENTOO - p_category = pkg[6] - p_repo = pkg[7] - else: - p_type = Package.UNKNOWN + name = pkg[0] + epoch = pkg[1] if pkg[1] else '' + ver = pkg[2] if pkg[2] else '' + rel = pkg[3] if pkg[3] else '' + arch = pkg[4] if pkg[4] else 'unknown' + + p_type = _get_package_type(pkg[5]) + p_category = pkg[6] if p_type == Package.GENTOO and len(pkg) > 6 else None + p_repo = pkg[7] if p_type == Package.GENTOO and len(pkg) > 7 else None - package = get_or_create_package(name, epoch, ver, rel, arch, p_type) - if p_type == Package.GENTOO: - process_gentoo_package(package, name, p_category, p_repo) - return package + return process_package(name, epoch, ver, rel, arch, p_type, p_category, p_repo) + + +def process_package_json(pkg): + """ Processes a single JSON package dict and converts to a package object + """ + name = pkg['name'] + epoch = pkg.get('epoch', '') + ver = pkg.get('version', '') + rel = pkg.get('release', '') + arch = pkg.get('arch', 'unknown') + p_type = _get_package_type(pkg.get('type', '')) + p_category = pkg.get('category') if p_type == Package.GENTOO else None + p_repo = pkg.get('repo') if p_type == Package.GENTOO else None + + return process_package(name, epoch, ver, rel, arch, p_type, p_category, p_repo) def process_gentoo_package(package, name, category, repo): @@ -368,6 +427,143 @@ def process_gentoo_package(package, name, category, repo): package.save() +def process_packages_json(packages_json, host): + """ Processes packages from JSON data (protocol 2) + """ + package_ids = [] + pbar_start.send(sender=None, ptext=f'{host} Packages', plen=len(packages_json)) + + for i, pkg in enumerate(packages_json): + debug_message(f'Processing JSON package: {pkg}') + package = process_package_json(pkg) + if package: + package_ids.append(package.id) + host.packages.add(package) + else: + if pkg.get('name', '').lower() != 'gpg-pubkey': + info_message(text=f'No package returned for {pkg}') + pbar_update.send(sender=None, index=i + 1) + + for package in host.packages.all(): + if package.id not in package_ids: + host.packages.remove(package) + + +def process_repo_json(repo, arch): + """ Processes a single JSON repo dict and converts to a repo object + """ + r_type = _get_repo_type(repo.get('type', '')) + if r_type is None: + return None, 0 + + if r_type == Repository.GENTOO: + arch = 'any' + + r_name = repo.get('name', '') + r_id = repo.get('id', '') + r_priority = repo.get('priority', 0) + urls = repo.get('urls', []) + + # Adjust priority for RPM repos (negative) + if r_type == Repository.RPM: + r_priority = r_priority * -1 + + return process_repo(r_type, r_name, r_id, r_priority, urls, arch) + + +def process_repos_json(repos_json, host, arch): + """ Processes repos from JSON data (protocol 2) + """ + repo_ids = [] + host_repos = HostRepo.objects.filter(host=host) + + pbar_start.send(sender=None, ptext=f'{host} Repos', plen=len(repos_json)) + for i, repo in enumerate(repos_json): + debug_message(f'Processing JSON repo: {repo}') + repository, priority = process_repo_json(repo, arch) + if repository: + repo_ids.append(repository.id) + try: + hostrepo, _ = host_repos.get_or_create(host=host, repo=repository) + except IntegrityError: + hostrepo = host_repos.get(host=host, repo=repository) + if hostrepo.priority != priority: + hostrepo.priority = priority + hostrepo.save() + pbar_update.send(sender=None, index=i + 1) + + for hostrepo in host_repos: + if hostrepo.repo_id not in repo_ids: + hostrepo.delete() + + +def process_module_json(module): + """ Processes a single JSON module dict and converts to a module object + """ + m_name = module.get('name') + m_stream = module.get('stream') + m_version = module.get('version') + m_context = module.get('context') + m_arch = module.get('arch') + repo_id = module.get('repo', '') + package_strings = module.get('packages', []) + + return process_module(m_name, m_stream, m_version, m_context, m_arch, repo_id, package_strings) + + +def process_modules_json(modules_json, host): + """ Processes modules from JSON data (protocol 2) + """ + module_ids = [] + + pbar_start.send(sender=None, ptext=f'{host} Modules', plen=len(modules_json)) + for i, module in enumerate(modules_json): + mod = process_module_json(module) + if mod: + module_ids.append(mod.id) + host.modules.add(mod) + pbar_update.send(sender=None, index=i + 1) + + for mod in host.modules.all(): + if mod.id not in module_ids: + host.modules.remove(mod) + + +def process_update_json(host, update, security): + """ Processes a single JSON update dict and converts to an update object + """ + name = update.get('name') + version = update.get('version') + arch = update.get('arch') + repo_id = update.get('repo', '') + + p_epoch, p_version, p_release = find_evr(version) + + return process_update(host, name, p_epoch, p_version, p_release, arch, repo_id, security) + + +def process_updates_json(sec_updates_json, bug_updates_json, host): + """ Processes updates from JSON data (protocol 2) + """ + # Clear existing updates + for host_update in host.updates.all(): + host.updates.remove(host_update) + + # Merge updates, preferring security over bugfix + sec_keys = {(u['name'], u['arch']) for u in sec_updates_json} + bug_updates_filtered = [u for u in bug_updates_json if (u['name'], u['arch']) not in sec_keys] + + all_updates = [(u, True) for u in sec_updates_json] + [(u, False) for u in bug_updates_filtered] + + if all_updates: + pbar_start.send(sender=None, ptext=f'{host} Updates', plen=len(all_updates)) + for i, (update, security) in enumerate(all_updates): + update_obj = process_update_json(host, update, security) + if update_obj: + host.updates.add(update_obj) + pbar_update.send(sender=None, index=i + 1) + + def get_arch(arch): """ Get or create MachineArchitecture from arch Returns the MachineArchitecture diff --git a/reports/views.py b/reports/views.py index c46b7a41..ee1489f1 100644 --- a/reports/views.py +++ b/reports/views.py @@ -1,5 +1,5 @@ # Copyright 2012 VPAC, http://www.vpac.org -# Copyright 2013-2021 Marcus Furlong +# Copyright 2013-2025 Marcus Furlong # # This file is part of Patchman. # @@ -15,6 +15,8 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see +import json + from django.contrib import messages from django.contrib.auth.decorators import login_required from django.db.models import Q @@ -24,11 +26,14 @@ from django.urls import reverse from django.views.decorators.csrf import csrf_exempt from django_tables2 import RequestConfig +from rest_framework import status, viewsets +from rest_framework.response import Response from tenacity import ( retry, retry_if_exception_type, stop_after_attempt, wait_exponential, ) from reports.models import Report +from reports.serializers import ReportUploadSerializer from reports.tables import ReportTable from util import sanitize_filter_params from util.filterspecs import Filter, FilterBar @@ -149,9 +154,28 @@ def report_detail(request, report_id): report = get_object_or_404(Report, id=report_id) + context = {'report': report} + + # Add tables for Protocol 2 reports + if report.protocol == '2': + from reports.tables import ( + ReportModuleTable, ReportPackageTable, ReportRepoTable, + ReportUpdateTable, + ) + if report.has_packages: + context['packages_table'] = ReportPackageTable(report.packages_parsed) + if report.has_repos: + context['repos_table'] = ReportRepoTable(report.repos_parsed) + if report.has_modules: + context['modules_table'] = ReportModuleTable(report.modules_parsed) + if report.has_sec_updates: + context['sec_updates_table'] = ReportUpdateTable(report.sec_updates_parsed) + if report.has_bug_updates: + context['bug_updates_table'] = ReportUpdateTable(report.bug_updates_parsed) + return render(request, 'reports/report_detail.html', - {'report': report}) + context) @login_required @@ -233,3 +257,88 @@ def report_bulk_action(request): if filter_params: return redirect(f"{reverse('reports:report_list')}?{filter_params}") return redirect('reports:report_list') + + +class ReportViewSet(viewsets.ViewSet): + """ + ViewSet for protocol 2 JSON report uploads. + + POST /api/report/ - Upload a new report in JSON format + + Authentication is optional by default. Set REQUIRE_API_KEY=True in settings + to require API key authentication for report uploads. + """ + + def get_permissions(self): + from django.conf import settings + from rest_framework.permissions import AllowAny + from rest_framework_api_key.permissions import HasAPIKey + if getattr(settings, 'REQUIRE_API_KEY', False): + return [HasAPIKey()] + return [AllowAny()] + + def create(self, request): + """Handle protocol 2 JSON report upload.""" + serializer = ReportUploadSerializer(data=request.data) + if not serializer.is_valid(): + return Response( + {'status': 'error', 'errors': serializer.errors}, + status=status.HTTP_400_BAD_REQUEST + ) + + data = serializer.validated_data + + # Extract client IP + x_forwarded_for = request.META.get('HTTP_X_FORWARDED_FOR') + x_real_ip = request.META.get('HTTP_X_REAL_IP') + if x_forwarded_for: + report_ip = x_forwarded_for.split(',')[0] + elif x_real_ip: + report_ip = x_real_ip + else: + report_ip = request.META.get('REMOTE_ADDR') + + # Extract domain from hostname + hostname = data['hostname'].lower() + domain = None + fqdn = hostname.split('.', 1) + if len(fqdn) == 2: + domain = fqdn[1] + + # Convert tags list to comma-separated string + tags = ','.join(data.get('tags', [])) + + # Convert reboot_required to string for compatibility + reboot = 'True' if data.get('reboot_required') else 'False' + + # Store JSON data as strings in the report model + report = Report.objects.create( + host=hostname, + domain=domain, + tags=tags, + kernel=data['kernel'], + arch=data['arch'], + os=data['os'], + report_ip=report_ip, + protocol='2', + useragent=request.META.get('HTTP_USER_AGENT', ''), + packages=json.dumps(data.get('packages', [])), + repos=json.dumps(data.get('repos', [])), + modules=json.dumps(data.get('modules', [])), + sec_updates=json.dumps(data.get('sec_updates', [])), + bug_updates=json.dumps(data.get('bug_updates', [])), + reboot=reboot, + ) + + # Queue for async processing + from reports.tasks import process_report + process_report.delay(report.id) + + return Response( + { + 'status': 'accepted', + 'report_id': report.id, + 'message': 'Report queued for processing' + }, + status=status.HTTP_202_ACCEPTED + ) diff --git a/requirements.txt b/requirements.txt index 3101fac5..b85c0243 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,6 +8,7 @@ PyYAML==6.0.2 requests==2.32.4 colorama==0.4.6 djangorestframework==3.15.2 +djangorestframework-api-key==3.0.0 django-filter==25.1 humanize==4.12.1 version-utils==0.3.2 diff --git a/util/management/__init__.py b/util/management/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/util/management/commands/__init__.py b/util/management/commands/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/util/management/commands/create_api_key.py b/util/management/commands/create_api_key.py new file mode 100644 index 00000000..4cca1397 --- /dev/null +++ b/util/management/commands/create_api_key.py @@ -0,0 +1,43 @@ +# Copyright 2025 Marcus Furlong +# +# This file is part of Patchman. +# +# Patchman is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, version 3 only. +# +# Patchman is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Patchman. If not, see + +from django.core.management.base import BaseCommand +from rest_framework_api_key.models import APIKey + + +class Command(BaseCommand): + help = 'Create a new API key for protocol 2 report uploads' + + def add_arguments(self, parser): + parser.add_argument( + 'name', + type=str, + help='Descriptive name for this API key (e.g., "webserver-cluster")' + ) + + def handle(self, *args, **options): + name = options['name'] + + api_key, key = APIKey.objects.create_key(name=name) + + self.stdout.write(self.style.SUCCESS(f'Created API key: {name}')) + self.stdout.write('') + self.stdout.write(f' Key: {key}') + self.stdout.write('') + self.stdout.write('Add this to your patchman-client.conf:') + self.stdout.write(f' api_key={key}') + self.stdout.write('') + self.stdout.write(self.style.WARNING('Save this key - it cannot be retrieved later!')) diff --git a/util/management/commands/list_api_keys.py b/util/management/commands/list_api_keys.py new file mode 100644 index 00000000..fd77c364 --- /dev/null +++ b/util/management/commands/list_api_keys.py @@ -0,0 +1,52 @@ +# Copyright 2025 Marcus Furlong +# +# This file is part of Patchman. +# +# Patchman is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, version 3 only. +# +# Patchman is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Patchman. If not, see + +from django.core.management.base import BaseCommand +from rest_framework_api_key.models import APIKey + + +class Command(BaseCommand): + help = 'List all API keys' + + def add_arguments(self, parser): + parser.add_argument( + '--all', + action='store_true', + help='Show all keys including revoked ones' + ) + + def handle(self, *args, **options): + if options['all']: + api_keys = APIKey.objects.all() + else: + api_keys = APIKey.objects.filter(revoked=False) + + if not api_keys.exists(): + self.stdout.write('No API keys found.') + return + + self.stdout.write('') + self.stdout.write(f'{"Name":<30} {"Prefix":<12} {"Created":<20} {"Revoked":<8}') + self.stdout.write('-' * 72) + + for key in api_keys: + created = key.created.strftime('%Y-%m-%d %H:%M') + revoked = 'Yes' if key.revoked else 'No' + + self.stdout.write(f'{key.name:<30} {key.prefix:<12} {created:<20} {revoked:<8}') + + self.stdout.write('') + self.stdout.write(f'Total: {api_keys.count()} key(s)') diff --git a/util/management/commands/revoke_api_key.py b/util/management/commands/revoke_api_key.py new file mode 100644 index 00000000..d22955e7 --- /dev/null +++ b/util/management/commands/revoke_api_key.py @@ -0,0 +1,63 @@ +# Copyright 2025 Marcus Furlong +# +# This file is part of Patchman. +# +# Patchman is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, version 3 only. +# +# Patchman is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Patchman. If not, see + +from django.core.management.base import BaseCommand, CommandError +from rest_framework_api_key.models import APIKey + + +class Command(BaseCommand): + help = 'Revoke an API key' + + def add_arguments(self, parser): + parser.add_argument( + 'key_or_prefix', + type=str, + help='The API key prefix or name to revoke' + ) + parser.add_argument( + '--delete', + action='store_true', + help='Permanently delete the key instead of just revoking' + ) + + def handle(self, *args, **options): + key_input = options['key_or_prefix'] + delete = options['delete'] + + # Try to find by prefix first, then by name + api_keys = APIKey.objects.filter(prefix=key_input) + if not api_keys.exists(): + api_keys = APIKey.objects.filter(name=key_input) + + if api_keys.count() == 0: + raise CommandError(f'No API key found matching: {key_input}') + elif api_keys.count() > 1: + raise CommandError(f'Multiple keys match "{key_input}". Please be more specific.') + + api_key = api_keys.first() + + if delete: + name = api_key.name + api_key.delete() + self.stdout.write(self.style.SUCCESS(f'Permanently deleted API key: {name}')) + else: + if api_key.revoked: + self.stdout.write(self.style.WARNING(f'API key "{api_key.name}" is already revoked')) + return + + api_key.revoked = True + api_key.save() + self.stdout.write(self.style.SUCCESS(f'Revoked API key: {api_key.name}')) From f8478c6a4954f89a5ae17a93bb4ff15b79b73a19 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Sun, 1 Feb 2026 18:36:27 -0500 Subject: [PATCH 059/146] api tests --- .github/workflows/lint-and-test.yml | 2 +- arch/tests/__init__.py | 0 arch/tests/test_api.py | 121 ++++++++ domains/tests/__init__.py | 0 domains/tests/test_api.py | 83 +++++ errata/serializers.py | 2 +- errata/tests/__init__.py | 15 + errata/tests/test_api.py | 166 ++++++++++ hosts/serializers.py | 4 +- hosts/tests/__init__.py | 0 hosts/tests/test_api.py | 383 ++++++++++++++++++++++++ operatingsystems/tests/__init__.py | 15 + operatingsystems/tests/test_api.py | 232 ++++++++++++++ packages/tests/__init__.py | 0 packages/tests/test_api.py | 304 +++++++++++++++++++ patchman/urls.py | 1 + reports/serializers.py | 12 + reports/tests/__init__.py | 0 reports/{tests.py => tests/test_api.py} | 34 +++ reports/views.py | 35 ++- repos/tests/__init__.py | 0 repos/tests/test_api.py | 295 ++++++++++++++++++ security/serializers.py | 6 +- security/tests/__init__.py | 0 security/tests/test_api.py | 216 +++++++++++++ 25 files changed, 1915 insertions(+), 11 deletions(-) create mode 100644 arch/tests/__init__.py create mode 100644 arch/tests/test_api.py create mode 100644 domains/tests/__init__.py create mode 100644 domains/tests/test_api.py create mode 100644 errata/tests/__init__.py create mode 100644 errata/tests/test_api.py create mode 100644 hosts/tests/__init__.py create mode 100644 hosts/tests/test_api.py create mode 100644 operatingsystems/tests/__init__.py create mode 100644 operatingsystems/tests/test_api.py create mode 100644 packages/tests/__init__.py create mode 100644 packages/tests/test_api.py create mode 100644 reports/tests/__init__.py rename reports/{tests.py => tests/test_api.py} (89%) create mode 100644 repos/tests/__init__.py create mode 100644 repos/tests/test_api.py create mode 100644 security/tests/__init__.py create mode 100644 security/tests/test_api.py diff --git a/.github/workflows/lint-and-test.yml b/.github/workflows/lint-and-test.yml index 436698be..71268b75 100644 --- a/.github/workflows/lint-and-test.yml +++ b/.github/workflows/lint-and-test.yml @@ -10,7 +10,7 @@ jobs: strategy: max-parallel: 5 matrix: - python-version: ['3.x'] + python-version: ['3.13'] steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} diff --git a/arch/tests/__init__.py b/arch/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/arch/tests/test_api.py b/arch/tests/test_api.py new file mode 100644 index 00000000..f86ca727 --- /dev/null +++ b/arch/tests/test_api.py @@ -0,0 +1,121 @@ +# Copyright 2025 Marcus Furlong +# +# This file is part of Patchman. +# +# Patchman is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, version 3 only. +# +# Patchman is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Patchman. If not, see + +from django.contrib.auth.models import User +from django.test import TestCase, override_settings +from rest_framework import status +from rest_framework.test import APITestCase + +from arch.models import MachineArchitecture, PackageArchitecture + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class ArchitectureAPITests(APITestCase): + """Tests for the Architecture API endpoints.""" + + def setUp(self): + """Set up test data.""" + self.user = User.objects.create_user( + username='testuser', password='testpass' + ) + self.client.force_authenticate(user=self.user) + + self.machine_arch = MachineArchitecture.objects.create(name='x86_64') + self.pkg_arch = PackageArchitecture.objects.create(name='amd64') + + def test_list_machine_architectures(self): + """Test listing machine architectures.""" + response = self.client.get('/api/machine-architecture/') + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(len(response.data['results']), 1) + + def test_retrieve_machine_architecture(self): + """Test retrieving a machine architecture.""" + response = self.client.get(f'/api/machine-architecture/{self.machine_arch.id}/') + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.data['name'], 'x86_64') + + def test_list_package_architectures(self): + """Test listing package architectures.""" + response = self.client.get('/api/package-architecture/') + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(len(response.data['results']), 1) + + def test_retrieve_package_architecture(self): + """Test retrieving a package architecture.""" + response = self.client.get(f'/api/package-architecture/{self.pkg_arch.id}/') + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.data['name'], 'amd64') + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class MachineArchitectureModelTests(TestCase): + """Tests for the MachineArchitecture model.""" + + def test_machine_architecture_creation(self): + """Test creating a machine architecture.""" + arch = MachineArchitecture.objects.create(name='aarch64') + self.assertEqual(arch.name, 'aarch64') + + def test_machine_architecture_string_representation(self): + """Test MachineArchitecture __str__ method.""" + arch = MachineArchitecture.objects.create(name='i686') + self.assertEqual(str(arch), 'i686') + + def test_machine_architecture_unique_name(self): + """Test that machine architecture names must be unique.""" + MachineArchitecture.objects.create(name='unique-arch') + from django.db import IntegrityError + with self.assertRaises(IntegrityError): + MachineArchitecture.objects.create(name='unique-arch') + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class PackageArchitectureModelTests(TestCase): + """Tests for the PackageArchitecture model.""" + + def test_package_architecture_creation(self): + """Test creating a package architecture.""" + arch = PackageArchitecture.objects.create(name='arm64') + self.assertEqual(arch.name, 'arm64') + + def test_package_architecture_string_representation(self): + """Test PackageArchitecture __str__ method.""" + arch = PackageArchitecture.objects.create(name='noarch') + self.assertEqual(str(arch), 'noarch') + + def test_package_architecture_unique_name(self): + """Test that package architecture names must be unique.""" + PackageArchitecture.objects.create(name='all') + from django.db import IntegrityError + with self.assertRaises(IntegrityError): + PackageArchitecture.objects.create(name='all') + + def test_common_architectures(self): + """Test creating common architecture types.""" + archs = ['x86_64', 'amd64', 'i386', 'i686', 'noarch', 'all', 'arm64', 'aarch64'] + for arch_name in archs: + arch = PackageArchitecture.objects.create(name=arch_name) + self.assertEqual(str(arch), arch_name) diff --git a/domains/tests/__init__.py b/domains/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/domains/tests/test_api.py b/domains/tests/test_api.py new file mode 100644 index 00000000..42ff41b7 --- /dev/null +++ b/domains/tests/test_api.py @@ -0,0 +1,83 @@ +# Copyright 2025 Marcus Furlong +# +# This file is part of Patchman. +# +# Patchman is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, version 3 only. +# +# Patchman is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Patchman. If not, see + +from django.contrib.auth.models import User +from django.test import TestCase, override_settings +from rest_framework import status +from rest_framework.test import APITestCase + +from domains.models import Domain + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class DomainAPITests(APITestCase): + """Tests for the Domain API endpoints.""" + + def setUp(self): + """Set up test data.""" + self.user = User.objects.create_user( + username='testuser', password='testpass' + ) + self.client.force_authenticate(user=self.user) + self.domain = Domain.objects.create(name='example.com') + + def test_list_domains(self): + """Test listing all domains.""" + response = self.client.get('/api/domain/') + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(len(response.data['results']), 1) + + def test_retrieve_domain(self): + """Test retrieving a single domain.""" + response = self.client.get(f'/api/domain/{self.domain.id}/') + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.data['name'], 'example.com') + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class DomainModelTests(TestCase): + """Tests for the Domain model.""" + + def test_domain_creation(self): + """Test creating a domain.""" + domain = Domain.objects.create(name='test.example.com') + self.assertEqual(domain.name, 'test.example.com') + + def test_domain_string_representation(self): + """Test Domain __str__ method.""" + domain = Domain.objects.create(name='prod.example.com') + self.assertEqual(str(domain), 'prod.example.com') + + def test_domain_unique_name(self): + """Test that domain names must be unique.""" + Domain.objects.create(name='unique.example.com') + from django.db import IntegrityError + with self.assertRaises(IntegrityError): + Domain.objects.create(name='unique.example.com') + + def test_extract_domain_from_fqdn(self): + """Test extracting domain from fully qualified domain name.""" + # Domain extraction is done elsewhere, but test the model can store it + fqdn = 'server1.prod.example.com' + domain_name = '.'.join(fqdn.split('.')[1:]) # prod.example.com + domain = Domain.objects.create(name=domain_name) + self.assertEqual(domain.name, 'prod.example.com') diff --git a/errata/serializers.py b/errata/serializers.py index c559a422..e44ddffd 100644 --- a/errata/serializers.py +++ b/errata/serializers.py @@ -22,4 +22,4 @@ class ErratumSerializer(serializers.HyperlinkedModelSerializer): class Meta: model = Erratum - fields = ('id', 'name', 'e_type', 'issue_date', 'synopsis', 'cves', 'releases', 'references') + fields = ('id', 'name', 'e_type', 'issue_date', 'synopsis', 'cves', 'osreleases', 'references') diff --git a/errata/tests/__init__.py b/errata/tests/__init__.py new file mode 100644 index 00000000..c7e9429a --- /dev/null +++ b/errata/tests/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2025 Marcus Furlong +# +# This file is part of Patchman. +# +# Patchman is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, version 3 only. +# +# Patchman is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Patchman. If not, see diff --git a/errata/tests/test_api.py b/errata/tests/test_api.py new file mode 100644 index 00000000..8c1df1c5 --- /dev/null +++ b/errata/tests/test_api.py @@ -0,0 +1,166 @@ +# Copyright 2025 Marcus Furlong +# +# This file is part of Patchman. +# +# Patchman is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, version 3 only. +# +# Patchman is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Patchman. If not, see + +from django.contrib.auth.models import User +from django.test import TestCase, override_settings +from django.utils import timezone +from rest_framework import status +from rest_framework.test import APITestCase + +from errata.models import Erratum +from security.models import CVE + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class ErratumAPITests(APITestCase): + """Tests for the Erratum API endpoints.""" + + def setUp(self): + """Set up test data.""" + self.user = User.objects.create_user( + username='testuser', + password='testpass123' + ) + self.client.force_authenticate(user=self.user) + + self.erratum = Erratum.objects.create( + name='RHSA-2024:0001', + e_type='security', + issue_date=timezone.now(), + synopsis='Important: openssl security update', + ) + + def test_list_errata(self): + """Test listing all errata.""" + response = self.client.get('/api/erratum/') + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(len(response.data['results']), 1) + + def test_retrieve_erratum(self): + """Test retrieving a single erratum.""" + response = self.client.get(f'/api/erratum/{self.erratum.id}/') + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.data['name'], 'RHSA-2024:0001') + + def test_filter_errata_by_type(self): + """Test filtering errata by type.""" + Erratum.objects.create( + name='RHBA-2024:0002', + e_type='bugfix', + issue_date=timezone.now(), + synopsis='Bug fix update', + ) + response = self.client.get('/api/erratum/', {'e_type': 'security'}) + self.assertEqual(response.status_code, status.HTTP_200_OK) + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class ErratumModelTests(TestCase): + """Tests for the Erratum model.""" + + def test_erratum_creation(self): + """Test creating an erratum.""" + erratum = Erratum.objects.create( + name='RHSA-2024:1234', + e_type='security', + issue_date=timezone.now(), + synopsis='Critical: kernel security update', + ) + self.assertEqual(erratum.name, 'RHSA-2024:1234') + self.assertEqual(erratum.e_type, 'security') + + def test_erratum_string_representation(self): + """Test Erratum __str__ method.""" + erratum = Erratum.objects.create( + name='RHSA-2024:1234', + e_type='security', + issue_date=timezone.now(), + synopsis='Critical: kernel security update', + ) + str_repr = str(erratum) + self.assertIn('RHSA-2024:1234', str_repr) + self.assertIn('security', str_repr) + + def test_erratum_unique_name(self): + """Test that erratum names must be unique.""" + Erratum.objects.create( + name='RHSA-2024:1234', + e_type='security', + issue_date=timezone.now(), + synopsis='Test erratum', + ) + from django.db import IntegrityError + with self.assertRaises(IntegrityError): + Erratum.objects.create( + name='RHSA-2024:1234', + e_type='bugfix', + issue_date=timezone.now(), + synopsis='Duplicate erratum', + ) + + def test_erratum_get_absolute_url(self): + """Test Erratum.get_absolute_url().""" + erratum = Erratum.objects.create( + name='RHSA-2024:1234', + e_type='security', + issue_date=timezone.now(), + synopsis='Test erratum', + ) + url = erratum.get_absolute_url() + self.assertIn('RHSA-2024:1234', url) + + def test_erratum_with_cves(self): + """Test erratum with associated CVEs.""" + erratum = Erratum.objects.create( + name='RHSA-2024:1234', + e_type='security', + issue_date=timezone.now(), + synopsis='Test erratum', + ) + cve1 = CVE.objects.create(cve_id='CVE-2024-0001') + cve2 = CVE.objects.create(cve_id='CVE-2024-0002') + erratum.cves.add(cve1, cve2) + self.assertEqual(erratum.cves.count(), 2) + + def test_erratum_types(self): + """Test different erratum types.""" + security = Erratum.objects.create( + name='RHSA-2024:0001', + e_type='security', + issue_date=timezone.now(), + synopsis='Security update', + ) + bugfix = Erratum.objects.create( + name='RHBA-2024:0001', + e_type='bugfix', + issue_date=timezone.now(), + synopsis='Bug fix update', + ) + enhancement = Erratum.objects.create( + name='RHEA-2024:0001', + e_type='enhancement', + issue_date=timezone.now(), + synopsis='Enhancement update', + ) + self.assertEqual(security.e_type, 'security') + self.assertEqual(bugfix.e_type, 'bugfix') + self.assertEqual(enhancement.e_type, 'enhancement') diff --git a/hosts/serializers.py b/hosts/serializers.py index 713fc580..c0313e47 100644 --- a/hosts/serializers.py +++ b/hosts/serializers.py @@ -15,6 +15,7 @@ # along with Patchman. If not, see from rest_framework import serializers +from taggit.serializers import TagListSerializerField from hosts.models import Host, HostRepo @@ -22,11 +23,12 @@ class HostSerializer(serializers.HyperlinkedModelSerializer): bugfix_update_count = serializers.SerializerMethodField() security_update_count = serializers.SerializerMethodField() + tags = TagListSerializerField() class Meta: model = Host fields = ('id', 'hostname', 'ipaddress', 'reversedns', 'check_dns', - 'os', 'kernel', 'arch', 'domain', 'lastreport', 'repos', + 'osvariant', 'kernel', 'arch', 'domain', 'lastreport', 'repos', 'updates', 'reboot_required', 'host_repos_only', 'tags', 'updated_at', 'bugfix_update_count', 'security_update_count') diff --git a/hosts/tests/__init__.py b/hosts/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/hosts/tests/test_api.py b/hosts/tests/test_api.py new file mode 100644 index 00000000..b63ea8d5 --- /dev/null +++ b/hosts/tests/test_api.py @@ -0,0 +1,383 @@ +# Copyright 2025 Marcus Furlong +# +# This file is part of Patchman. +# +# Patchman is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, version 3 only. +# +# Patchman is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Patchman. If not, see + +from django.contrib.auth.models import User +from django.test import TestCase, override_settings +from django.utils import timezone +from rest_framework import status +from rest_framework.test import APITestCase + +from arch.models import MachineArchitecture, PackageArchitecture +from domains.models import Domain +from hosts.models import Host, HostRepo +from operatingsystems.models import OSRelease, OSVariant +from packages.models import Package, PackageName, PackageUpdate +from repos.models import Repository + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class HostAPITests(APITestCase): + """Tests for the Host API endpoints.""" + + def setUp(self): + """Set up test data.""" + # Create user for authentication + self.user = User.objects.create_user( + username='testuser', password='testpass' + ) + self.client.force_authenticate(user=self.user) + + # Create required related objects + self.arch = MachineArchitecture.objects.create(name='x86_64') + self.domain = Domain.objects.create(name='example.com') + self.os_release = OSRelease.objects.create( + name='Ubuntu 22.04', codename='jammy' + ) + self.os_variant = OSVariant.objects.create( + name='Ubuntu 22.04.3 LTS', osrelease=self.os_release + ) + + # Create test host + self.host = Host.objects.create( + hostname='testhost.example.com', + ipaddress='192.168.1.100', + osvariant=self.os_variant, + kernel='5.15.0-91-generic', + arch=self.arch, + domain=self.domain, + lastreport=timezone.now(), + ) + + def test_list_hosts(self): + """Test listing all hosts.""" + response = self.client.get('/api/host/') + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(len(response.data['results']), 1) + + def test_retrieve_host(self): + """Test retrieving a single host.""" + response = self.client.get(f'/api/host/{self.host.id}/') + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.data['hostname'], 'testhost.example.com') + + def test_filter_hosts_by_hostname(self): + """Test filtering hosts by hostname.""" + response = self.client.get('/api/host/', {'hostname': 'testhost.example.com'}) + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(len(response.data['results']), 1) + + response = self.client.get('/api/host/', {'hostname': 'nonexistent'}) + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(len(response.data['results']), 0) + + def test_host_update_counts(self): + """Test that bugfix and security update counts are returned.""" + response = self.client.get(f'/api/host/{self.host.id}/') + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertIn('bugfix_update_count', response.data) + self.assertIn('security_update_count', response.data) + self.assertEqual(response.data['bugfix_update_count'], 0) + self.assertEqual(response.data['security_update_count'], 0) + + def test_host_update_counts_with_updates(self): + """Test update counts with actual security and bugfix updates.""" + # Create package architecture and names + pkg_arch = PackageArchitecture.objects.create(name='amd64') + pkg_name = PackageName.objects.create(name='openssl') + + # Create old and new packages + old_pkg = Package.objects.create( + name=pkg_name, arch=pkg_arch, epoch='', + version='3.0.0', release='1', packagetype='D' + ) + new_security_pkg = Package.objects.create( + name=pkg_name, arch=pkg_arch, epoch='', + version='3.0.1', release='1', packagetype='D' + ) + new_bugfix_pkg = Package.objects.create( + name=pkg_name, arch=pkg_arch, epoch='', + version='3.0.2', release='1', packagetype='D' + ) + + # Create security and bugfix updates + security_update = PackageUpdate.objects.create( + oldpackage=old_pkg, newpackage=new_security_pkg, security=True + ) + bugfix_update = PackageUpdate.objects.create( + oldpackage=old_pkg, newpackage=new_bugfix_pkg, security=False + ) + + # Associate updates with host + self.host.updates.add(security_update, bugfix_update) + + response = self.client.get(f'/api/host/{self.host.id}/') + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.data['security_update_count'], 1) + self.assertEqual(response.data['bugfix_update_count'], 1) + + def test_create_host_via_api(self): + """Test creating a host via API.""" + data = { + 'hostname': 'newhost.example.com', + 'ipaddress': '192.168.1.101', + 'kernel': '5.15.0-92-generic', + 'arch': f'http://testserver/api/machine-architecture/{self.arch.id}/', + 'domain': f'http://testserver/api/domain/{self.domain.id}/', + 'osvariant': f'http://testserver/api/os-variant/{self.os_variant.id}/', + 'lastreport': timezone.now().isoformat(), + 'reboot_required': False, + 'host_repos_only': True, + } + response = self.client.post('/api/host/', data, format='json') + # ModelViewSet allows creation + self.assertIn(response.status_code, [status.HTTP_201_CREATED, status.HTTP_400_BAD_REQUEST]) + + def test_unauthenticated_access_denied(self): + """Test that unauthenticated requests are denied.""" + self.client.force_authenticate(user=None) + response = self.client.get('/api/host/') + # DRF default is IsAuthenticatedOrReadOnly, so GET may be allowed + self.assertIn( + response.status_code, + [status.HTTP_200_OK, status.HTTP_401_UNAUTHORIZED, status.HTTP_403_FORBIDDEN] + ) + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class HostModelTests(TestCase): + """Tests for the Host model.""" + + def setUp(self): + """Set up test data.""" + self.arch = MachineArchitecture.objects.create(name='x86_64') + self.domain = Domain.objects.create(name='example.com') + self.os_release = OSRelease.objects.create( + name='Ubuntu 22.04', codename='jammy' + ) + self.os_variant = OSVariant.objects.create( + name='Ubuntu 22.04.3 LTS', osrelease=self.os_release + ) + + def test_host_creation(self): + """Test creating a host with required fields.""" + host = Host.objects.create( + hostname='testhost.example.com', + ipaddress='192.168.1.100', + osvariant=self.os_variant, + kernel='5.15.0-91-generic', + arch=self.arch, + domain=self.domain, + lastreport=timezone.now(), + ) + self.assertEqual(host.hostname, 'testhost.example.com') + self.assertEqual(str(host.arch), 'x86_64') + + def test_host_string_representation(self): + """Test Host __str__ method.""" + host = Host.objects.create( + hostname='testhost.example.com', + ipaddress='192.168.1.100', + osvariant=self.os_variant, + kernel='5.15.0-91-generic', + arch=self.arch, + domain=self.domain, + lastreport=timezone.now(), + ) + self.assertEqual(str(host), 'testhost.example.com') + + def test_host_unique_hostname(self): + """Test that hostname must be unique.""" + Host.objects.create( + hostname='unique.example.com', + ipaddress='192.168.1.100', + osvariant=self.os_variant, + kernel='5.15.0-91-generic', + arch=self.arch, + domain=self.domain, + lastreport=timezone.now(), + ) + from django.db import IntegrityError + with self.assertRaises(IntegrityError): + Host.objects.create( + hostname='unique.example.com', + ipaddress='192.168.1.101', + osvariant=self.os_variant, + kernel='5.15.0-91-generic', + arch=self.arch, + domain=self.domain, + lastreport=timezone.now(), + ) + + def test_host_get_absolute_url(self): + """Test Host.get_absolute_url() returns correct URL.""" + host = Host.objects.create( + hostname='testhost.example.com', + ipaddress='192.168.1.100', + osvariant=self.os_variant, + kernel='5.15.0-91-generic', + arch=self.arch, + domain=self.domain, + lastreport=timezone.now(), + ) + url = host.get_absolute_url() + self.assertIn('testhost.example.com', url) + + def test_host_tags(self): + """Test Host tagging functionality.""" + host = Host.objects.create( + hostname='tagged.example.com', + ipaddress='192.168.1.100', + osvariant=self.os_variant, + kernel='5.15.0-91-generic', + arch=self.arch, + domain=self.domain, + lastreport=timezone.now(), + ) + host.tags.add('production', 'web') + self.assertEqual(host.tags.count(), 2) + self.assertTrue(host.tags.filter(name='production').exists()) + + def test_host_reboot_required_default(self): + """Test that reboot_required defaults to False.""" + host = Host.objects.create( + hostname='testhost.example.com', + ipaddress='192.168.1.100', + osvariant=self.os_variant, + kernel='5.15.0-91-generic', + arch=self.arch, + domain=self.domain, + lastreport=timezone.now(), + ) + self.assertFalse(host.reboot_required) + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class HostRepoTests(TestCase): + """Tests for the HostRepo model.""" + + def setUp(self): + """Set up test data.""" + self.arch = MachineArchitecture.objects.create(name='x86_64') + self.domain = Domain.objects.create(name='example.com') + self.os_release = OSRelease.objects.create( + name='Ubuntu 22.04', codename='jammy' + ) + self.os_variant = OSVariant.objects.create( + name='Ubuntu 22.04.3 LTS', osrelease=self.os_release + ) + self.host = Host.objects.create( + hostname='testhost.example.com', + ipaddress='192.168.1.100', + osvariant=self.os_variant, + kernel='5.15.0-91-generic', + arch=self.arch, + domain=self.domain, + lastreport=timezone.now(), + ) + self.repo = Repository.objects.create( + name='ubuntu-main', + repotype='D', + arch=self.arch, + ) + + def test_host_repo_creation(self): + """Test creating a HostRepo relationship.""" + host_repo = HostRepo.objects.create( + host=self.host, + repo=self.repo, + enabled=True, + priority=500, + ) + self.assertEqual(host_repo.host, self.host) + self.assertEqual(host_repo.repo, self.repo) + self.assertTrue(host_repo.enabled) + self.assertEqual(host_repo.priority, 500) + + def test_host_repos_relationship(self): + """Test Host.repos ManyToMany relationship via HostRepo.""" + HostRepo.objects.create( + host=self.host, + repo=self.repo, + enabled=True, + ) + self.assertIn(self.repo, self.host.repos.all()) + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class HostRepoAPITests(APITestCase): + """Tests for the HostRepo API endpoints.""" + + def setUp(self): + """Set up test data.""" + self.user = User.objects.create_user( + username='testuser', + password='testpass123' + ) + self.client.force_authenticate(user=self.user) + + self.arch = MachineArchitecture.objects.create(name='x86_64') + self.domain = Domain.objects.create(name='example.com') + self.os_release = OSRelease.objects.create( + name='Ubuntu 22.04', codename='jammy' + ) + self.os_variant = OSVariant.objects.create( + name='Ubuntu 22.04.3 LTS', osrelease=self.os_release + ) + self.host = Host.objects.create( + hostname='testhost.example.com', + ipaddress='192.168.1.100', + osvariant=self.os_variant, + kernel='5.15.0-91-generic', + arch=self.arch, + domain=self.domain, + lastreport=timezone.now(), + ) + self.repo = Repository.objects.create( + name='ubuntu-main', + repotype='D', + arch=self.arch, + ) + self.host_repo = HostRepo.objects.create( + host=self.host, + repo=self.repo, + enabled=True, + priority=500, + ) + + def test_list_host_repos(self): + """Test listing all host-repo relationships.""" + response = self.client.get('/api/host-repo/') + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(len(response.data['results']), 1) + + def test_retrieve_host_repo(self): + """Test retrieving a single host-repo relationship.""" + response = self.client.get(f'/api/host-repo/{self.host_repo.id}/') + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.data['enabled'], True) + self.assertEqual(response.data['priority'], 500) diff --git a/operatingsystems/tests/__init__.py b/operatingsystems/tests/__init__.py new file mode 100644 index 00000000..c7e9429a --- /dev/null +++ b/operatingsystems/tests/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2025 Marcus Furlong +# +# This file is part of Patchman. +# +# Patchman is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, version 3 only. +# +# Patchman is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Patchman. If not, see diff --git a/operatingsystems/tests/test_api.py b/operatingsystems/tests/test_api.py new file mode 100644 index 00000000..85a4a474 --- /dev/null +++ b/operatingsystems/tests/test_api.py @@ -0,0 +1,232 @@ +# Copyright 2025 Marcus Furlong +# +# This file is part of Patchman. +# +# Patchman is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, version 3 only. +# +# Patchman is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Patchman. If not, see + +from django.contrib.auth.models import User +from django.test import TestCase, override_settings +from rest_framework import status +from rest_framework.test import APITestCase + +from arch.models import MachineArchitecture +from operatingsystems.models import OSRelease, OSVariant + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class OSReleaseAPITests(APITestCase): + """Tests for the OS Release API endpoints.""" + + def setUp(self): + """Set up test data.""" + self.user = User.objects.create_user( + username='testuser', + password='testpass123' + ) + self.client.force_authenticate(user=self.user) + + self.os_release = OSRelease.objects.create( + name='Ubuntu 22.04', + codename='jammy', + ) + + def test_list_os_releases(self): + """Test listing all OS releases.""" + response = self.client.get('/api/os-release/') + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(len(response.data['results']), 1) + + def test_retrieve_os_release(self): + """Test retrieving a single OS release.""" + response = self.client.get(f'/api/os-release/{self.os_release.id}/') + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.data['name'], 'Ubuntu 22.04') + + def test_filter_os_releases_by_name(self): + """Test filtering OS releases by name.""" + OSRelease.objects.create(name='Rocky Linux 9', codename='') + response = self.client.get('/api/os-release/', {'name': 'Ubuntu 22.04'}) + self.assertEqual(response.status_code, status.HTTP_200_OK) + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class OSVariantAPITests(APITestCase): + """Tests for the OS Variant API endpoints.""" + + def setUp(self): + """Set up test data.""" + self.user = User.objects.create_user( + username='testuser', + password='testpass123' + ) + self.client.force_authenticate(user=self.user) + + self.arch = MachineArchitecture.objects.create(name='x86_64') + self.os_release = OSRelease.objects.create( + name='Ubuntu 22.04', + codename='jammy', + ) + self.os_variant = OSVariant.objects.create( + name='Ubuntu 22.04.3 LTS', + arch=self.arch, + osrelease=self.os_release, + ) + + def test_list_os_variants(self): + """Test listing all OS variants.""" + response = self.client.get('/api/os-variant/') + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(len(response.data['results']), 1) + + def test_retrieve_os_variant(self): + """Test retrieving a single OS variant.""" + response = self.client.get(f'/api/os-variant/{self.os_variant.id}/') + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.data['name'], 'Ubuntu 22.04.3 LTS') + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class OSReleaseModelTests(TestCase): + """Tests for the OSRelease model.""" + + def test_os_release_creation(self): + """Test creating an OS release.""" + os_release = OSRelease.objects.create( + name='Debian 12', + codename='bookworm', + ) + self.assertEqual(os_release.name, 'Debian 12') + self.assertEqual(os_release.codename, 'bookworm') + + def test_os_release_string_representation(self): + """Test OSRelease __str__ method with codename.""" + os_release = OSRelease.objects.create( + name='Ubuntu 22.04', + codename='jammy', + ) + self.assertEqual(str(os_release), 'Ubuntu 22.04 (jammy)') + + def test_os_release_string_without_codename(self): + """Test OSRelease __str__ method without codename.""" + os_release = OSRelease.objects.create( + name='Rocky Linux 9', + codename='', + ) + self.assertEqual(str(os_release), 'Rocky Linux 9') + + def test_os_release_unique_name(self): + """Test that OS release names must be unique.""" + OSRelease.objects.create(name='Ubuntu 22.04', codename='jammy') + from django.db import IntegrityError + with self.assertRaises(IntegrityError): + OSRelease.objects.create(name='Ubuntu 22.04', codename='') + + def test_os_release_get_absolute_url(self): + """Test OSRelease.get_absolute_url().""" + os_release = OSRelease.objects.create(name='Ubuntu 22.04', codename='jammy') + url = os_release.get_absolute_url() + self.assertIn(str(os_release.id), url) + + def test_os_release_with_cpe_name(self): + """Test OS release with CPE name.""" + os_release = OSRelease.objects.create( + name='Rocky Linux 9', + codename='', + cpe_name='cpe:/o:rocky:rocky:9', + ) + self.assertEqual(os_release.cpe_name, 'cpe:/o:rocky:rocky:9') + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class OSVariantModelTests(TestCase): + """Tests for the OSVariant model.""" + + def setUp(self): + """Set up test data.""" + self.arch = MachineArchitecture.objects.create(name='x86_64') + self.os_release = OSRelease.objects.create( + name='Ubuntu 22.04', + codename='jammy', + ) + + def test_os_variant_creation(self): + """Test creating an OS variant.""" + os_variant = OSVariant.objects.create( + name='Ubuntu 22.04.3 LTS', + arch=self.arch, + osrelease=self.os_release, + ) + self.assertEqual(os_variant.name, 'Ubuntu 22.04.3 LTS') + + def test_os_variant_string_representation(self): + """Test OSVariant __str__ method.""" + os_variant = OSVariant.objects.create( + name='Ubuntu 22.04.3 LTS', + arch=self.arch, + osrelease=self.os_release, + ) + self.assertIn('Ubuntu 22.04.3 LTS', str(os_variant)) + self.assertIn('x86_64', str(os_variant)) + + def test_os_variant_unique_name(self): + """Test that OS variant names must be unique.""" + OSVariant.objects.create( + name='Ubuntu 22.04.3 LTS', + arch=self.arch, + osrelease=self.os_release, + ) + from django.db import IntegrityError + with self.assertRaises(IntegrityError): + OSVariant.objects.create( + name='Ubuntu 22.04.3 LTS', + arch=self.arch, + osrelease=self.os_release, + ) + + def test_os_variant_get_absolute_url(self): + """Test OSVariant.get_absolute_url().""" + os_variant = OSVariant.objects.create( + name='Ubuntu 22.04.3 LTS', + arch=self.arch, + osrelease=self.os_release, + ) + url = os_variant.get_absolute_url() + self.assertIn(str(os_variant.id), url) + + def test_os_variant_without_arch(self): + """Test OS variant can be created without arch.""" + os_variant = OSVariant.objects.create( + name='Generic Linux', + osrelease=self.os_release, + ) + self.assertIsNone(os_variant.arch) + + def test_os_variant_without_osrelease(self): + """Test OS variant can be created without osrelease.""" + os_variant = OSVariant.objects.create( + name='Unknown OS', + arch=self.arch, + ) + self.assertIsNone(os_variant.osrelease) diff --git a/packages/tests/__init__.py b/packages/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/packages/tests/test_api.py b/packages/tests/test_api.py new file mode 100644 index 00000000..5bd9b70c --- /dev/null +++ b/packages/tests/test_api.py @@ -0,0 +1,304 @@ +# Copyright 2025 Marcus Furlong +# +# This file is part of Patchman. +# +# Patchman is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, version 3 only. +# +# Patchman is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Patchman. If not, see + +from django.contrib.auth.models import User +from django.test import TestCase, override_settings +from rest_framework import status +from rest_framework.test import APITestCase + +from arch.models import PackageArchitecture +from packages.models import Package, PackageName, PackageUpdate + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class PackageAPITests(APITestCase): + """Tests for the Package API endpoints.""" + + def setUp(self): + """Set up test data.""" + self.user = User.objects.create_user( + username='testuser', password='testpass' + ) + self.client.force_authenticate(user=self.user) + + self.arch = PackageArchitecture.objects.create(name='amd64') + self.pkg_name = PackageName.objects.create(name='nginx') + self.package = Package.objects.create( + name=self.pkg_name, + epoch='', + version='1.18.0', + release='6ubuntu14', + arch=self.arch, + packagetype='D', + ) + + def test_list_packages(self): + """Test listing all packages.""" + response = self.client.get('/api/package/') + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(len(response.data['results']), 1) + + def test_retrieve_package(self): + """Test retrieving a single package.""" + response = self.client.get(f'/api/package/{self.package.id}/') + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.data['version'], '1.18.0') + + def test_filter_packages_by_name(self): + """Test filtering packages by name.""" + response = self.client.get('/api/package/', {'name': self.pkg_name.id}) + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(len(response.data['results']), 1) + + def test_filter_packages_by_version(self): + """Test filtering packages by version.""" + response = self.client.get('/api/package/', {'version': '1.18.0'}) + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(len(response.data['results']), 1) + + def test_list_package_names(self): + """Test listing package names.""" + response = self.client.get('/api/package-name/') + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(len(response.data['results']), 1) + + def test_filter_package_names(self): + """Test filtering package names.""" + response = self.client.get('/api/package-name/', {'name': 'nginx'}) + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(len(response.data['results']), 1) + + def test_retrieve_package_name(self): + """Test retrieving a single package name.""" + response = self.client.get(f'/api/package-name/{self.pkg_name.id}/') + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.data['name'], 'nginx') + + def test_list_package_updates(self): + """Test listing package updates.""" + response = self.client.get('/api/package-update/') + self.assertEqual(response.status_code, status.HTTP_200_OK) + + def test_retrieve_package_update(self): + """Test retrieving a single package update.""" + from packages.models import PackageUpdate + new_pkg = Package.objects.create( + name=self.pkg_name, + arch=self.arch, + epoch='', + version='1.20.0', + release='1', + packagetype='D', + ) + update = PackageUpdate.objects.create( + oldpackage=self.package, + newpackage=new_pkg, + security=True, + ) + response = self.client.get(f'/api/package-update/{update.id}/') + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertTrue(response.data['security']) + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class PackageModelTests(TestCase): + """Tests for the Package model.""" + + def setUp(self): + """Set up test data.""" + self.arch = PackageArchitecture.objects.create(name='amd64') + self.pkg_name = PackageName.objects.create(name='nginx') + + def test_package_creation(self): + """Test creating a package with all fields.""" + package = Package.objects.create( + name=self.pkg_name, + epoch='1', + version='1.18.0', + release='6ubuntu14', + arch=self.arch, + packagetype='D', + ) + self.assertEqual(package.version, '1.18.0') + self.assertEqual(package.packagetype, 'D') + + def test_package_string_representation(self): + """Test Package __str__ method.""" + package = Package.objects.create( + name=self.pkg_name, + version='1.18.0', + release='6ubuntu14', + arch=self.arch, + packagetype='D', + ) + str_rep = str(package) + self.assertIn('nginx', str_rep) + self.assertIn('1.18.0', str_rep) + + def test_package_type_choices(self): + """Test package type constants.""" + self.assertEqual(Package.RPM, 'R') + self.assertEqual(Package.DEB, 'D') + self.assertEqual(Package.ARCH, 'A') + self.assertEqual(Package.GENTOO, 'G') + self.assertEqual(Package.UNKNOWN, 'U') + + def test_package_with_epoch(self): + """Test package with epoch field.""" + package = Package.objects.create( + name=self.pkg_name, + epoch='2', + version='1.18.0', + release='1', + arch=self.arch, + packagetype='R', + ) + self.assertEqual(package.epoch, '2') + + def test_package_without_release(self): + """Test package without release field (Debian style).""" + package = Package.objects.create( + name=self.pkg_name, + version='1.18.0-6ubuntu14', + arch=self.arch, + packagetype='D', + ) + self.assertIsNone(package.release) + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class PackageNameModelTests(TestCase): + """Tests for the PackageName model.""" + + def test_package_name_creation(self): + """Test creating a package name.""" + pkg_name = PackageName.objects.create(name='curl') + self.assertEqual(pkg_name.name, 'curl') + + def test_package_name_string_representation(self): + """Test PackageName __str__ method.""" + pkg_name = PackageName.objects.create(name='openssl') + self.assertEqual(str(pkg_name), 'openssl') + + def test_package_name_unique(self): + """Test that package names must be unique.""" + PackageName.objects.create(name='unique-package') + from django.db import IntegrityError + with self.assertRaises(IntegrityError): + PackageName.objects.create(name='unique-package') + + def test_package_name_get_absolute_url(self): + """Test PackageName.get_absolute_url().""" + pkg_name = PackageName.objects.create(name='testpkg') + url = pkg_name.get_absolute_url() + self.assertIn('testpkg', url) + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class PackageUpdateModelTests(TestCase): + """Tests for the PackageUpdate model.""" + + def setUp(self): + """Set up test data.""" + self.arch = PackageArchitecture.objects.create(name='amd64') + self.pkg_name = PackageName.objects.create(name='openssl') + self.old_package = Package.objects.create( + name=self.pkg_name, + version='3.0.2', + release='0ubuntu1.12', + arch=self.arch, + packagetype='D', + ) + self.new_package = Package.objects.create( + name=self.pkg_name, + version='3.0.2', + release='0ubuntu1.13', + arch=self.arch, + packagetype='D', + ) + + def test_package_update_creation(self): + """Test creating a package update.""" + update = PackageUpdate.objects.create( + oldpackage=self.old_package, + newpackage=self.new_package, + security=True, + ) + self.assertEqual(update.oldpackage, self.old_package) + self.assertEqual(update.newpackage, self.new_package) + self.assertTrue(update.security) + + def test_package_update_security_flag(self): + """Test security vs bugfix classification.""" + security_update = PackageUpdate.objects.create( + oldpackage=self.old_package, + newpackage=self.new_package, + security=True, + ) + self.assertTrue(security_update.security) + + bugfix_update = PackageUpdate.objects.create( + oldpackage=self.old_package, + newpackage=self.new_package, + security=False, + ) + self.assertFalse(bugfix_update.security) + + def test_package_update_api(self): + """Test PackageUpdate API endpoint.""" + PackageUpdate.objects.create( + oldpackage=self.old_package, + newpackage=self.new_package, + security=True, + ) + user = User.objects.create_user(username='testuser', password='testpass') + self.client.force_login(user) + + from rest_framework.test import APIClient + api_client = APIClient() + api_client.force_authenticate(user=user) + + response = api_client.get('/api/package-update/') + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(len(response.data['results']), 1) + + def test_filter_package_updates_by_security(self): + """Test filtering package updates by security flag.""" + PackageUpdate.objects.create( + oldpackage=self.old_package, + newpackage=self.new_package, + security=True, + ) + user = User.objects.create_user(username='testuser', password='testpass') + from rest_framework.test import APIClient + api_client = APIClient() + api_client.force_authenticate(user=user) + + response = api_client.get('/api/package-update/', {'security': 'true'}) + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(len(response.data['results']), 1) diff --git a/patchman/urls.py b/patchman/urls.py index 4cc22900..9f5065fd 100644 --- a/patchman/urls.py +++ b/patchman/urls.py @@ -44,6 +44,7 @@ router.register(r'package', package_views.PackageViewSet) router.register(r'package-update', package_views.PackageUpdateViewSet) router.register(r'cve', security_views.CVEViewSet) +router.register(r'cwe', security_views.CWEViewSet) router.register(r'reference', security_views.ReferenceViewSet) router.register(r'erratum', errata_views.ErratumViewSet) router.register(r'repo', repo_views.RepositoryViewSet) diff --git a/reports/serializers.py b/reports/serializers.py index 6822afa7..6d560254 100644 --- a/reports/serializers.py +++ b/reports/serializers.py @@ -89,3 +89,15 @@ def validate_protocol(self, value): if value != 2: raise serializers.ValidationError('This endpoint only accepts protocol 2') return value + + +class ReportSerializer(serializers.HyperlinkedModelSerializer): + """Serializer for reading Report model instances.""" + + class Meta: + from reports.models import Report + model = Report + fields = ( + 'id', 'host', 'domain', 'tags', 'kernel', 'arch', 'os', + 'report_ip', 'protocol', 'useragent', 'processed', 'created' + ) diff --git a/reports/tests/__init__.py b/reports/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/reports/tests.py b/reports/tests/test_api.py similarity index 89% rename from reports/tests.py rename to reports/tests/test_api.py index c75fe305..3c2578e1 100644 --- a/reports/tests.py +++ b/reports/tests/test_api.py @@ -14,6 +14,7 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see +from django.contrib.auth.models import User from django.test import TestCase, override_settings from rest_framework import status from rest_framework.test import APITestCase @@ -32,6 +33,39 @@ class ReportAPITests(APITestCase): def setUp(self): self.url = '/api/report/' + self.user = User.objects.create_user( + username='testuser', password='testpass' + ) + self.client.force_authenticate(user=self.user) + + def test_list_reports(self): + """Test listing all reports.""" + # Create a report first + Report.objects.create( + host='testhost.example.com', + domain='example.com', + kernel='5.15.0', + arch='x86_64', + os='Ubuntu 22.04', + protocol='2', + ) + response = self.client.get(self.url) + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(len(response.data), 1) + + def test_retrieve_report(self): + """Test retrieving a single report.""" + report = Report.objects.create( + host='testhost.example.com', + domain='example.com', + kernel='5.15.0', + arch='x86_64', + os='Ubuntu 22.04', + protocol='2', + ) + response = self.client.get(f'{self.url}{report.id}/') + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.data['host'], 'testhost.example.com') def test_upload_minimal_report(self): """Test uploading a minimal valid report.""" diff --git a/reports/views.py b/reports/views.py index ee1489f1..1a7651ae 100644 --- a/reports/views.py +++ b/reports/views.py @@ -261,8 +261,10 @@ def report_bulk_action(request): class ReportViewSet(viewsets.ViewSet): """ - ViewSet for protocol 2 JSON report uploads. + ViewSet for protocol 2 JSON report uploads and report listing. + GET /api/report/ - List all reports + GET /api/report/{id}/ - Retrieve a single report POST /api/report/ - Upload a new report in JSON format Authentication is optional by default. Set REQUIRE_API_KEY=True in settings @@ -271,11 +273,34 @@ class ReportViewSet(viewsets.ViewSet): def get_permissions(self): from django.conf import settings - from rest_framework.permissions import AllowAny + from rest_framework.permissions import ( + AllowAny, IsAuthenticatedOrReadOnly, + ) from rest_framework_api_key.permissions import HasAPIKey - if getattr(settings, 'REQUIRE_API_KEY', False): - return [HasAPIKey()] - return [AllowAny()] + + # POST requires API key if configured, otherwise allow any + if self.action == 'create': + if getattr(settings, 'REQUIRE_API_KEY', False): + return [HasAPIKey()] + return [AllowAny()] + # GET is read-only (authenticated or read-only) + return [IsAuthenticatedOrReadOnly()] + + def list(self, request): + """List all reports.""" + from reports.serializers import ReportSerializer + queryset = Report.objects.all().order_by('-created') + serializer = ReportSerializer(queryset, many=True, context={'request': request}) + return Response(serializer.data) + + def retrieve(self, request, pk=None): + """Retrieve a single report.""" + from django.shortcuts import get_object_or_404 + + from reports.serializers import ReportSerializer + report = get_object_or_404(Report, pk=pk) + serializer = ReportSerializer(report, context={'request': request}) + return Response(serializer.data) def create(self, request): """Handle protocol 2 JSON report upload.""" diff --git a/repos/tests/__init__.py b/repos/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/repos/tests/test_api.py b/repos/tests/test_api.py new file mode 100644 index 00000000..a8040005 --- /dev/null +++ b/repos/tests/test_api.py @@ -0,0 +1,295 @@ +# Copyright 2025 Marcus Furlong +# +# This file is part of Patchman. +# +# Patchman is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, version 3 only. +# +# Patchman is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Patchman. If not, see + +from django.contrib.auth.models import User +from django.test import TestCase, override_settings +from rest_framework import status +from rest_framework.test import APITestCase + +from arch.models import MachineArchitecture, PackageArchitecture +from packages.models import Package, PackageName +from repos.models import Mirror, MirrorPackage, Repository + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class RepositoryAPITests(APITestCase): + """Tests for the Repository API endpoints.""" + + def setUp(self): + """Set up test data.""" + self.user = User.objects.create_user( + username='testuser', password='testpass' + ) + self.client.force_authenticate(user=self.user) + + self.arch = MachineArchitecture.objects.create(name='x86_64') + self.repo = Repository.objects.create( + name='ubuntu-main', + arch=self.arch, + repotype='D', + enabled=True, + ) + + def test_list_repos(self): + """Test listing all repositories.""" + response = self.client.get('/api/repo/') + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(len(response.data['results']), 1) + + def test_retrieve_repo(self): + """Test retrieving a single repository.""" + response = self.client.get(f'/api/repo/{self.repo.id}/') + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.data['name'], 'ubuntu-main') + + def test_list_mirrors(self): + """Test listing mirrors.""" + Mirror.objects.create( + repo=self.repo, + url='http://archive.ubuntu.com/ubuntu/dists/jammy/main/binary-amd64', + ) + response = self.client.get('/api/mirror/') + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(len(response.data['results']), 1) + + def test_retrieve_mirror(self): + """Test retrieving a single mirror.""" + mirror = Mirror.objects.create( + repo=self.repo, + url='http://archive.ubuntu.com/ubuntu/dists/jammy/main/binary-amd64', + ) + response = self.client.get(f'/api/mirror/{mirror.id}/') + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertIn('archive.ubuntu.com', response.data['url']) + + def test_list_mirror_packages(self): + """Test listing mirror packages.""" + response = self.client.get('/api/mirror-package/') + self.assertEqual(response.status_code, status.HTTP_200_OK) + + def test_retrieve_mirror_package(self): + """Test retrieving a single mirror package.""" + from arch.models import PackageArchitecture + from packages.models import Package, PackageName + pkg_arch = PackageArchitecture.objects.create(name='amd64') + pkg_name = PackageName.objects.create(name='nginx') + package = Package.objects.create( + name=pkg_name, + arch=pkg_arch, + epoch='', + version='1.18.0', + release='1', + packagetype='D', + ) + mirror = Mirror.objects.create( + repo=self.repo, + url='http://archive.ubuntu.com/ubuntu/dists/jammy/main/binary-amd64', + ) + mirror_package = MirrorPackage.objects.create( + mirror=mirror, + package=package, + ) + response = self.client.get(f'/api/mirror-package/{mirror_package.id}/') + self.assertEqual(response.status_code, status.HTTP_200_OK) + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class RepositoryModelTests(TestCase): + """Tests for the Repository model.""" + + def setUp(self): + """Set up test data.""" + self.arch = MachineArchitecture.objects.create(name='x86_64') + + def test_repository_creation(self): + """Test creating a repository.""" + repo = Repository.objects.create( + name='test-repo', + arch=self.arch, + repotype='R', + enabled=True, + ) + self.assertEqual(repo.name, 'test-repo') + self.assertEqual(repo.repotype, 'R') + + def test_repository_string_representation(self): + """Test Repository __str__ method.""" + repo = Repository.objects.create( + name='ubuntu-security', + arch=self.arch, + repotype='D', + ) + self.assertEqual(str(repo), 'ubuntu-security') + + def test_repository_type_choices(self): + """Test repository type constants.""" + self.assertEqual(Repository.RPM, 'R') + self.assertEqual(Repository.DEB, 'D') + self.assertEqual(Repository.ARCH, 'A') + self.assertEqual(Repository.GENTOO, 'G') + + def test_repository_unique_name(self): + """Test that repository names must be unique.""" + Repository.objects.create( + name='unique-repo', + arch=self.arch, + repotype='D', + ) + from django.db import IntegrityError + with self.assertRaises(IntegrityError): + Repository.objects.create( + name='unique-repo', + arch=self.arch, + repotype='D', + ) + + def test_repository_security_flag(self): + """Test repository security flag.""" + security_repo = Repository.objects.create( + name='ubuntu-security', + arch=self.arch, + repotype='D', + security=True, + ) + self.assertTrue(security_repo.security) + + normal_repo = Repository.objects.create( + name='ubuntu-main', + arch=self.arch, + repotype='D', + security=False, + ) + self.assertFalse(normal_repo.security) + + def test_repository_enabled_default(self): + """Test that repository enabled defaults to True.""" + repo = Repository.objects.create( + name='test-repo', + arch=self.arch, + repotype='D', + ) + self.assertTrue(repo.enabled) + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class MirrorModelTests(TestCase): + """Tests for the Mirror model.""" + + def setUp(self): + """Set up test data.""" + self.arch = MachineArchitecture.objects.create(name='x86_64') + self.repo = Repository.objects.create( + name='test-repo', + arch=self.arch, + repotype='D', + ) + + def test_mirror_creation(self): + """Test creating a mirror.""" + mirror = Mirror.objects.create( + repo=self.repo, + url='http://archive.ubuntu.com/ubuntu/dists/jammy/main/binary-amd64', + ) + self.assertEqual(mirror.repo, self.repo) + self.assertIn('archive.ubuntu.com', mirror.url) + + def test_mirror_string_representation(self): + """Test Mirror __str__ method.""" + mirror = Mirror.objects.create( + repo=self.repo, + url='http://archive.ubuntu.com/ubuntu/dists/jammy/main/binary-amd64', + ) + str_rep = str(mirror) + self.assertIn('archive.ubuntu.com', str_rep) + + def test_mirror_enabled_default(self): + """Test that mirror enabled defaults to True.""" + mirror = Mirror.objects.create( + repo=self.repo, + url='http://example.com/repo', + ) + self.assertTrue(mirror.enabled) + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class MirrorPackageModelTests(TestCase): + """Tests for the MirrorPackage model.""" + + def setUp(self): + """Set up test data.""" + self.machine_arch = MachineArchitecture.objects.create(name='x86_64') + self.pkg_arch = PackageArchitecture.objects.create(name='amd64') + self.repo = Repository.objects.create( + name='test-repo', + arch=self.machine_arch, + repotype='D', + ) + self.mirror = Mirror.objects.create( + repo=self.repo, + url='http://example.com/repo', + ) + self.pkg_name = PackageName.objects.create(name='nginx') + self.package = Package.objects.create( + name=self.pkg_name, + version='1.18.0', + release='6ubuntu14', + arch=self.pkg_arch, + packagetype='D', + ) + + def test_mirror_package_creation(self): + """Test creating a mirror package link.""" + mp = MirrorPackage.objects.create( + mirror=self.mirror, + package=self.package, + ) + self.assertEqual(mp.mirror, self.mirror) + self.assertEqual(mp.package, self.package) + + def test_mirror_packages_relationship(self): + """Test Mirror.packages ManyToMany relationship.""" + MirrorPackage.objects.create( + mirror=self.mirror, + package=self.package, + ) + self.assertIn(self.package, self.mirror.packages.all()) + + def test_mirror_package_api(self): + """Test MirrorPackage API endpoint.""" + MirrorPackage.objects.create( + mirror=self.mirror, + package=self.package, + ) + user = User.objects.create_user(username='testuser', password='testpass') + from rest_framework.test import APIClient + api_client = APIClient() + api_client.force_authenticate(user=user) + + response = api_client.get('/api/mirror-package/') + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(len(response.data['results']), 1) diff --git a/security/serializers.py b/security/serializers.py index 979fc2c9..10ccaca1 100644 --- a/security/serializers.py +++ b/security/serializers.py @@ -22,14 +22,14 @@ class CWESerializer(serializers.HyperlinkedModelSerializer): class Meta: model = CWE - fields = ('cwe_id', 'title', 'description') + fields = ('cwe_id', 'name', 'description') class CVESerializer(serializers.HyperlinkedModelSerializer): class Meta: model = CVE - fields = ('cve_id', 'description', 'cvss_score', 'cwe', - 'registered_date', 'published_date', 'updated_date') + fields = ('cve_id', 'description', 'cvss_scores', 'cwes', + 'reserved_date', 'published_date', 'updated_date') class ReferenceSerializer(serializers.HyperlinkedModelSerializer): diff --git a/security/tests/__init__.py b/security/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/security/tests/test_api.py b/security/tests/test_api.py new file mode 100644 index 00000000..cd3dd936 --- /dev/null +++ b/security/tests/test_api.py @@ -0,0 +1,216 @@ +# Copyright 2025 Marcus Furlong +# +# This file is part of Patchman. +# +# Patchman is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, version 3 only. +# +# Patchman is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Patchman. If not, see + +from django.contrib.auth.models import User +from django.test import TestCase, override_settings +from rest_framework import status +from rest_framework.test import APITestCase + +from security.models import CVE, CWE, Reference + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class SecurityAPITests(APITestCase): + """Tests for the Security API endpoints.""" + + def setUp(self): + """Set up test data.""" + self.user = User.objects.create_user( + username='testuser', password='testpass' + ) + self.client.force_authenticate(user=self.user) + + self.cve = CVE.objects.create( + cve_id='CVE-2024-1234', + description='Test vulnerability', + ) + self.reference = Reference.objects.create( + ref_type='ADVISORY', + url='https://example.com/advisory/CVE-2024-1234', + ) + self.cwe = CWE.objects.create( + cwe_id='CWE-79', + name='Cross-site Scripting', + ) + + def test_list_cves(self): + """Test listing all CVEs.""" + response = self.client.get('/api/cve/') + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(len(response.data['results']), 1) + + def test_retrieve_cve(self): + """Test retrieving a single CVE.""" + response = self.client.get(f'/api/cve/{self.cve.id}/') + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.data['cve_id'], 'CVE-2024-1234') + + def test_list_references(self): + """Test listing references.""" + response = self.client.get('/api/reference/') + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(len(response.data['results']), 1) + + def test_retrieve_reference(self): + """Test retrieving a single reference.""" + response = self.client.get(f'/api/reference/{self.reference.id}/') + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.data['ref_type'], 'ADVISORY') + + def test_list_cwes(self): + """Test listing CWEs.""" + response = self.client.get('/api/cwe/') + self.assertEqual(response.status_code, status.HTTP_200_OK) + + def test_retrieve_cwe(self): + """Test retrieving a single CWE.""" + response = self.client.get(f'/api/cwe/{self.cwe.id}/') + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.data['cwe_id'], 'CWE-79') + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class CVEModelTests(TestCase): + """Tests for the CVE model.""" + + def test_cve_creation(self): + """Test creating a CVE.""" + cve = CVE.objects.create( + cve_id='CVE-2024-5678', + description='Remote code execution vulnerability', + ) + self.assertEqual(cve.cve_id, 'CVE-2024-5678') + + def test_cve_string_representation(self): + """Test CVE __str__ method.""" + cve = CVE.objects.create( + cve_id='CVE-2024-9999', + description='Buffer overflow', + ) + self.assertEqual(str(cve), 'CVE-2024-9999') + + def test_cve_unique_id(self): + """Test that CVE IDs must be unique.""" + CVE.objects.create(cve_id='CVE-2024-0001') + from django.db import IntegrityError + with self.assertRaises(IntegrityError): + CVE.objects.create(cve_id='CVE-2024-0001') + + def test_cve_get_absolute_url(self): + """Test CVE.get_absolute_url().""" + cve = CVE.objects.create(cve_id='CVE-2024-1111') + url = cve.get_absolute_url() + self.assertIn('CVE-2024-1111', url) + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class CWEModelTests(TestCase): + """Tests for the CWE model.""" + + def test_cwe_creation(self): + """Test creating a CWE.""" + cwe = CWE.objects.create( + cwe_id='CWE-79', + name='Improper Neutralization of Input During Web Page Generation', + ) + self.assertEqual(cwe.cwe_id, 'CWE-79') + + def test_cwe_string_representation(self): + """Test CWE __str__ method.""" + cwe = CWE.objects.create( + cwe_id='CWE-89', + name='SQL Injection', + ) + str_rep = str(cwe) + self.assertIn('CWE-89', str_rep) + self.assertIn('SQL Injection', str_rep) + + def test_cwe_unique_id(self): + """Test that CWE IDs must be unique.""" + CWE.objects.create(cwe_id='CWE-100') + from django.db import IntegrityError + with self.assertRaises(IntegrityError): + CWE.objects.create(cwe_id='CWE-100') + + def test_cwe_int_id_property(self): + """Test CWE.int_id property extracts numeric part.""" + cwe = CWE.objects.create(cwe_id='CWE-787') + self.assertEqual(cwe.int_id, 787) + + def test_cwe_get_absolute_url(self): + """Test CWE.get_absolute_url().""" + cwe = CWE.objects.create(cwe_id='CWE-20') + url = cwe.get_absolute_url() + self.assertIn('CWE-20', url) + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class ReferenceModelTests(TestCase): + """Tests for the Reference model.""" + + def test_reference_creation(self): + """Test creating a reference.""" + ref = Reference.objects.create( + ref_type='VENDOR', + url='https://vendor.com/security/advisory', + ) + self.assertEqual(ref.ref_type, 'VENDOR') + + def test_reference_string_representation(self): + """Test Reference __str__ method.""" + ref = Reference.objects.create( + ref_type='ADVISORY', + url='https://example.com/advisory', + ) + self.assertEqual(str(ref), 'https://example.com/advisory') + + def test_reference_unique_together(self): + """Test that ref_type + url must be unique together.""" + Reference.objects.create( + ref_type='PATCH', + url='https://example.com/patch', + ) + from django.db import IntegrityError + with self.assertRaises(IntegrityError): + Reference.objects.create( + ref_type='PATCH', + url='https://example.com/patch', + ) + + def test_reference_same_url_different_type(self): + """Test that same URL with different type is allowed.""" + Reference.objects.create( + ref_type='ADVISORY', + url='https://example.com/security', + ) + # Same URL, different type should work + ref2 = Reference.objects.create( + ref_type='VENDOR', + url='https://example.com/security', + ) + self.assertEqual(ref2.ref_type, 'VENDOR') From 92fffaa7fee84471d707ab1e59235445063b6068 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Sun, 1 Feb 2026 21:43:22 -0500 Subject: [PATCH 060/146] update tests --- arch/tests/test_models.py | 83 ++++ domains/tests/test_models.py | 52 +++ errata/tests/test_integration.py | 74 ++++ errata/tests/test_models.py | 122 ++++++ hosts/tests/test_find_updates.py | 364 +++++++++++++++++ hosts/tests/test_managers.py | 205 ++++++++++ hosts/tests/test_models.py | 130 ++++++ operatingsystems/tests/test_models.py | 102 +++++ packages/tests/test_models.py | 202 +++++++++ packages/tests/test_version_compare.py | 316 +++++++++++++++ reports/tests/test_edge_cases.py | 302 ++++++++++++++ reports/tests/test_integration.py | 274 +++++++++++++ reports/tests/test_models.py | 476 ++++++++++++++++++++++ reports/tests/test_parsing.py | 180 +++++++++ reports/tests/test_serializers.py | 290 +++++++++++++ reports/tests/test_tasks.py | 271 +++++++++++++ reports/tests/test_utils.py | 539 +++++++++++++++++++++++++ repos/tests/test_managers.py | 170 ++++++++ repos/tests/test_mirror_sync.py | 141 +++++++ repos/tests/test_models.py | 193 +++++++++ security/tests/test_models.py | 180 +++++++++ util/tests/__init__.py | 0 util/tests/test_commands.py | 176 ++++++++ util/tests/test_utils.py | 261 ++++++++++++ 24 files changed, 5103 insertions(+) create mode 100644 arch/tests/test_models.py create mode 100644 domains/tests/test_models.py create mode 100644 errata/tests/test_integration.py create mode 100644 errata/tests/test_models.py create mode 100644 hosts/tests/test_find_updates.py create mode 100644 hosts/tests/test_managers.py create mode 100644 hosts/tests/test_models.py create mode 100644 operatingsystems/tests/test_models.py create mode 100644 packages/tests/test_models.py create mode 100644 packages/tests/test_version_compare.py create mode 100644 reports/tests/test_edge_cases.py create mode 100644 reports/tests/test_integration.py create mode 100644 reports/tests/test_models.py create mode 100644 reports/tests/test_parsing.py create mode 100644 reports/tests/test_serializers.py create mode 100644 reports/tests/test_tasks.py create mode 100644 reports/tests/test_utils.py create mode 100644 repos/tests/test_managers.py create mode 100644 repos/tests/test_mirror_sync.py create mode 100644 repos/tests/test_models.py create mode 100644 security/tests/test_models.py create mode 100644 util/tests/__init__.py create mode 100644 util/tests/test_commands.py create mode 100644 util/tests/test_utils.py diff --git a/arch/tests/test_models.py b/arch/tests/test_models.py new file mode 100644 index 00000000..d33cecb2 --- /dev/null +++ b/arch/tests/test_models.py @@ -0,0 +1,83 @@ +# Copyright 2025 Marcus Furlong +# +# This file is part of Patchman. +# +# Patchman is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, version 3 only. +# +# Patchman is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Patchman. If not, see + +from django.test import TestCase, override_settings + +from arch.models import MachineArchitecture, PackageArchitecture + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class MachineArchitectureMethodTests(TestCase): + """Tests for MachineArchitecture model methods.""" + + def test_machine_arch_creation(self): + """Test creating a MachineArchitecture.""" + arch = MachineArchitecture.objects.create(name='x86_64') + self.assertEqual(arch.name, 'x86_64') + + def test_machine_arch_str(self): + """Test MachineArchitecture __str__ method.""" + arch = MachineArchitecture.objects.create(name='x86_64') + self.assertEqual(str(arch), 'x86_64') + + def test_machine_arch_unique_name(self): + """Test MachineArchitecture name is unique.""" + MachineArchitecture.objects.create(name='x86_64') + from django.db import IntegrityError + with self.assertRaises(IntegrityError): + MachineArchitecture.objects.create(name='x86_64') + + def test_common_machine_architectures(self): + """Test common machine architecture values.""" + archs = ['x86_64', 'aarch64', 'i686', 'armv7l', 'ppc64le'] + for name in archs: + arch = MachineArchitecture.objects.create(name=name) + self.assertEqual(str(arch), name) + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class PackageArchitectureMethodTests(TestCase): + """Tests for PackageArchitecture model methods.""" + + def test_package_arch_creation(self): + """Test creating a PackageArchitecture.""" + arch = PackageArchitecture.objects.create(name='amd64') + self.assertEqual(arch.name, 'amd64') + + def test_package_arch_str(self): + """Test PackageArchitecture __str__ method.""" + arch = PackageArchitecture.objects.create(name='amd64') + self.assertEqual(str(arch), 'amd64') + + def test_package_arch_unique_name(self): + """Test PackageArchitecture name is unique.""" + PackageArchitecture.objects.create(name='amd64') + from django.db import IntegrityError + with self.assertRaises(IntegrityError): + PackageArchitecture.objects.create(name='amd64') + + def test_common_package_architectures(self): + """Test common package architecture values.""" + archs = ['amd64', 'i386', 'all', 'noarch', 'x86_64', 'arm64'] + for name in archs: + arch = PackageArchitecture.objects.create(name=name) + self.assertEqual(str(arch), name) diff --git a/domains/tests/test_models.py b/domains/tests/test_models.py new file mode 100644 index 00000000..1b553346 --- /dev/null +++ b/domains/tests/test_models.py @@ -0,0 +1,52 @@ +# Copyright 2025 Marcus Furlong +# +# This file is part of Patchman. +# +# Patchman is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, version 3 only. +# +# Patchman is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Patchman. If not, see + +from django.test import TestCase, override_settings + +from domains.models import Domain + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class DomainMethodTests(TestCase): + """Tests for Domain model methods.""" + + def test_domain_creation(self): + """Test creating a Domain.""" + domain = Domain.objects.create(name='example.com') + self.assertEqual(domain.name, 'example.com') + + def test_domain_str(self): + """Test Domain __str__ method.""" + domain = Domain.objects.create(name='example.com') + self.assertEqual(str(domain), 'example.com') + + def test_domain_unique_name(self): + """Test Domain name is unique.""" + Domain.objects.create(name='example.com') + from django.db import IntegrityError + with self.assertRaises(IntegrityError): + Domain.objects.create(name='example.com') + + def test_domain_extract_from_fqdn(self): + """Test extracting domain from FQDN via Host creation.""" + # Domains are typically extracted when hosts are created + # Test the domain itself can be created with subdomain parts + Domain.objects.create(name='subdomain.example.com') + domain = Domain.objects.get(name='subdomain.example.com') + self.assertEqual(domain.name, 'subdomain.example.com') diff --git a/errata/tests/test_integration.py b/errata/tests/test_integration.py new file mode 100644 index 00000000..254e7aae --- /dev/null +++ b/errata/tests/test_integration.py @@ -0,0 +1,74 @@ +# Copyright 2025 Marcus Furlong +# +# This file is part of Patchman. +# +# Patchman is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, version 3 only. +# +# Patchman is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Patchman. If not, see + +from django.test import TestCase, override_settings + +from errata.models import Erratum +from operatingsystems.models import OSRelease +from security.models import CVE + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class ErrataIntegrationTests(TestCase): + """Integration tests for errata functionality.""" + + def test_erratum_with_cves(self): + """Test erratum can be associated with CVEs.""" + cve1 = CVE.objects.create(cve_id='CVE-2024-1001') + cve2 = CVE.objects.create(cve_id='CVE-2024-1002') + + erratum = Erratum.objects.create( + name='RHSA-2024:1234', + e_type='Security Advisory', + synopsis='Important: curl security update', + issue_date='2024-03-15', + ) + erratum.cves.add(cve1, cve2) + + self.assertEqual(erratum.cves.count(), 2) + self.assertIn(cve1, erratum.cves.all()) + self.assertIn(cve2, erratum.cves.all()) + + def test_erratum_with_osreleases(self): + """Test erratum can be associated with OS releases.""" + osrelease1 = OSRelease.objects.create(name='Rocky Linux 9') + osrelease2 = OSRelease.objects.create(name='Rocky Linux 8') + + erratum = Erratum.objects.create( + name='RHSA-2024:1235', + e_type='Security Advisory', + synopsis='Important: openssl security update', + issue_date='2024-03-16', + ) + erratum.osreleases.add(osrelease1, osrelease2) + + self.assertEqual(erratum.osreleases.count(), 2) + + def test_erratum_with_packages(self): + """Test erratum can reference package names.""" + erratum = Erratum.objects.create( + name='RHSA-2024:1236', + e_type='Bug Fix', + synopsis='Bug fix: httpd update', + issue_date='2024-03-17', + ) + + # Verify erratum can store package references + self.assertIsNotNone(erratum) + self.assertEqual(erratum.e_type, 'Bug Fix') diff --git a/errata/tests/test_models.py b/errata/tests/test_models.py new file mode 100644 index 00000000..fc316297 --- /dev/null +++ b/errata/tests/test_models.py @@ -0,0 +1,122 @@ +# Copyright 2025 Marcus Furlong +# +# This file is part of Patchman. +# +# Patchman is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, version 3 only. +# +# Patchman is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Patchman. If not, see + +from django.test import TestCase, override_settings +from django.utils import timezone + +from errata.models import Erratum +from operatingsystems.models import OSRelease +from security.models import CVE + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class ErratumMethodTests(TestCase): + """Tests for Erratum model methods.""" + + def test_erratum_creation(self): + """Test creating an Erratum.""" + erratum = Erratum.objects.create( + name='USN-1234-1', + e_type='security', + synopsis='Security update', + issue_date=timezone.now(), + ) + self.assertEqual(erratum.name, 'USN-1234-1') + + def test_erratum_str(self): + """Test Erratum __str__ method.""" + erratum = Erratum.objects.create( + name='USN-1234-1', + e_type='security', + synopsis='Security update', + issue_date=timezone.now(), + ) + self.assertIn('USN-1234-1', str(erratum)) + + def test_erratum_get_absolute_url(self): + """Test Erratum.get_absolute_url().""" + erratum = Erratum.objects.create( + name='USN-1234-1', + e_type='security', + synopsis='Security update', + issue_date=timezone.now(), + ) + url = erratum.get_absolute_url() + self.assertIn(erratum.name, url) + + def test_erratum_unique_name(self): + """Test Erratum name is unique.""" + Erratum.objects.create( + name='USN-1234-1', + e_type='security', + synopsis='Security update', + issue_date=timezone.now(), + ) + from django.db import IntegrityError + with self.assertRaises(IntegrityError): + Erratum.objects.create( + name='USN-1234-1', + e_type='bugfix', + synopsis='Bugfix update', + issue_date=timezone.now(), + ) + + def test_erratum_with_cves(self): + """Test Erratum with associated CVEs.""" + erratum = Erratum.objects.create( + name='USN-1234-1', + e_type='security', + synopsis='Security update', + issue_date=timezone.now(), + ) + cve = CVE.objects.create(cve_id='CVE-2024-12345') + erratum.cves.add(cve) + self.assertIn(cve, erratum.cves.all()) + + def test_erratum_with_osreleases(self): + """Test Erratum with associated OS releases.""" + erratum = Erratum.objects.create( + name='USN-1234-1', + e_type='security', + synopsis='Security update', + issue_date=timezone.now(), + ) + release = OSRelease.objects.create(name='Ubuntu 22.04') + erratum.osreleases.add(release) + self.assertIn(release, erratum.osreleases.all()) + + def test_security_erratum(self): + """Test creating a security erratum.""" + erratum = Erratum.objects.create( + name='RHSA-2024:1234', + e_type='security', + synopsis='Important security update', + issue_date=timezone.now(), + ) + self.assertEqual(erratum.e_type, 'security') + + def test_bugfix_erratum(self): + """Test creating a bugfix erratum.""" + erratum = Erratum.objects.create( + name='RHBA-2024:1234', + e_type='bugfix', + synopsis='Bug fix update', + issue_date=timezone.now(), + ) + self.assertEqual(erratum.e_type, 'bugfix') diff --git a/hosts/tests/test_find_updates.py b/hosts/tests/test_find_updates.py new file mode 100644 index 00000000..fd253079 --- /dev/null +++ b/hosts/tests/test_find_updates.py @@ -0,0 +1,364 @@ +# Copyright 2025 Marcus Furlong +# +# This file is part of Patchman. +# +# Patchman is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, version 3 only. +# +# Patchman is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Patchman. If not, see + +from django.test import TestCase, override_settings +from django.utils import timezone + +from arch.models import MachineArchitecture, PackageArchitecture +from domains.models import Domain +from hosts.models import Host, HostRepo +from operatingsystems.models import OSRelease, OSVariant +from packages.models import Package, PackageName, PackageUpdate +from repos.models import Mirror, MirrorPackage, Repository + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class HostFindUpdatesTests(TestCase): + """Tests for Host.find_updates() method.""" + + def setUp(self): + """Set up test data.""" + self.machine_arch = MachineArchitecture.objects.create(name='x86_64') + self.pkg_arch = PackageArchitecture.objects.create(name='x86_64') + self.osrelease = OSRelease.objects.create(name='Rocky Linux 9') + self.osvariant = OSVariant.objects.create( + name='Rocky Linux 9 x86_64', + osrelease=self.osrelease, + arch=self.machine_arch, + ) + self.domain = Domain.objects.create(name='example.com') + self.host = Host.objects.create( + hostname='updatetest.example.com', + ipaddress='192.168.1.100', + arch=self.machine_arch, + osvariant=self.osvariant, + domain=self.domain, + kernel='5.14.0-362.el9.x86_64', + lastreport=timezone.now(), + ) + + # Create a repository + self.repo = Repository.objects.create( + name='baseos', + arch=self.machine_arch, + repotype=Repository.RPM, + enabled=True, + ) + self.mirror = Mirror.objects.create( + repo=self.repo, + url='http://mirror.rockylinux.org/rocky/9/BaseOS/x86_64/os', + enabled=True, + refresh=True, + ) + + # Associate repo with host + HostRepo.objects.create( + host=self.host, + repo=self.repo, + enabled=True, + ) + + def test_find_updates_no_packages(self): + """Test find_updates with no packages installed.""" + # Host has no packages, should not crash + self.host.find_updates() + self.assertEqual(self.host.updates.count(), 0) + + def test_find_updates_no_updates_available(self): + """Test find_updates when no updates are available.""" + # Install a package on host + pkg_name = PackageName.objects.create(name='httpd') + installed_pkg = Package.objects.create( + name=pkg_name, + arch=self.pkg_arch, + epoch='0', + version='2.4.57', + release='5.el9', + packagetype=Package.RPM, + ) + self.host.packages.add(installed_pkg) + + # No newer package in mirror + MirrorPackage.objects.create( + mirror=self.mirror, + package=installed_pkg, + ) + + self.host.find_updates() + # No updates should be found (same version in repo) + self.assertEqual(self.host.updates.count(), 0) + + def test_find_updates_with_available_update(self): + """Test find_updates when an update is available.""" + # Install an old package on host + pkg_name = PackageName.objects.create(name='openssl') + old_pkg = Package.objects.create( + name=pkg_name, + arch=self.pkg_arch, + epoch='1', + version='3.0.0', + release='1.el9', + packagetype=Package.RPM, + ) + self.host.packages.add(old_pkg) + + # Add old package to mirror + MirrorPackage.objects.create( + mirror=self.mirror, + package=old_pkg, + ) + + # Add newer package to mirror + new_pkg = Package.objects.create( + name=pkg_name, + arch=self.pkg_arch, + epoch='1', + version='3.0.7', + release='1.el9', + packagetype=Package.RPM, + ) + MirrorPackage.objects.create( + mirror=self.mirror, + package=new_pkg, + ) + + self.host.find_updates() + # Should find the update + self.assertEqual(self.host.updates.count(), 1) + update = self.host.updates.first() + self.assertEqual(update.oldpackage, old_pkg) + self.assertEqual(update.newpackage, new_pkg) + + def test_find_updates_security_repo(self): + """Test find_updates marks security repo updates correctly.""" + # Create a security repository + security_repo = Repository.objects.create( + name='baseos-security', + arch=self.machine_arch, + repotype=Repository.RPM, + enabled=True, + security=True, # Mark as security repo + ) + security_mirror = Mirror.objects.create( + repo=security_repo, + url='http://mirror.rockylinux.org/rocky/9/BaseOS/x86_64/security', + enabled=True, + refresh=True, + ) + HostRepo.objects.create( + host=self.host, + repo=security_repo, + enabled=True, + ) + + # Install an old package + pkg_name = PackageName.objects.create(name='curl') + old_pkg = Package.objects.create( + name=pkg_name, + arch=self.pkg_arch, + epoch='0', + version='7.76.0', + release='1.el9', + packagetype=Package.RPM, + ) + self.host.packages.add(old_pkg) + MirrorPackage.objects.create(mirror=self.mirror, package=old_pkg) + + # Add security update + new_pkg = Package.objects.create( + name=pkg_name, + arch=self.pkg_arch, + epoch='0', + version='7.76.1', + release='1.el9_security', + packagetype=Package.RPM, + ) + MirrorPackage.objects.create(mirror=security_mirror, package=new_pkg) + + self.host.find_updates() + # Update should be marked as security + if self.host.updates.count() > 0: + update = self.host.updates.first() + self.assertTrue(update.security) + + def test_find_updates_excludes_kernel_packages(self): + """Test find_updates handles kernel packages separately.""" + # Kernel packages have special handling + pkg_name = PackageName.objects.create(name='kernel') + old_kernel = Package.objects.create( + name=pkg_name, + arch=self.pkg_arch, + epoch='0', + version='5.14.0', + release='362.el9', + packagetype=Package.RPM, + ) + self.host.packages.add(old_kernel) + MirrorPackage.objects.create(mirror=self.mirror, package=old_kernel) + + new_kernel = Package.objects.create( + name=pkg_name, + arch=self.pkg_arch, + epoch='0', + version='5.14.0', + release='427.el9', + packagetype=Package.RPM, + ) + MirrorPackage.objects.create(mirror=self.mirror, package=new_kernel) + + # Should not crash when processing kernels + self.host.find_updates() + + def test_find_updates_removes_stale_updates(self): + """Test find_updates removes updates that are no longer valid.""" + # Create an update that's no longer valid + pkg_name = PackageName.objects.create(name='stale-pkg') + old_pkg = Package.objects.create( + name=pkg_name, + arch=self.pkg_arch, + epoch='0', + version='1.0.0', + release='1.el9', + packagetype=Package.RPM, + ) + new_pkg = Package.objects.create( + name=pkg_name, + arch=self.pkg_arch, + epoch='0', + version='1.0.1', + release='1.el9', + packagetype=Package.RPM, + ) + + # Create a stale update manually + stale_update = PackageUpdate.objects.create( + oldpackage=old_pkg, + newpackage=new_pkg, + security=False, + ) + self.host.updates.add(stale_update) + self.assertEqual(self.host.updates.count(), 1) + + # Host doesn't have the old package installed + # find_updates should remove this stale update + self.host.find_updates() + self.assertEqual(self.host.updates.count(), 0) + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class HostProcessUpdateTests(TestCase): + """Tests for Host.process_update() method.""" + + def setUp(self): + """Set up test data.""" + self.machine_arch = MachineArchitecture.objects.create(name='x86_64') + self.pkg_arch = PackageArchitecture.objects.create(name='x86_64') + self.osrelease = OSRelease.objects.create(name='Rocky Linux 9') + self.osvariant = OSVariant.objects.create( + name='Rocky Linux 9 x86_64', + osrelease=self.osrelease, + arch=self.machine_arch, + ) + self.domain = Domain.objects.create(name='example.com') + self.host = Host.objects.create( + hostname='processupdate.example.com', + ipaddress='192.168.1.101', + arch=self.machine_arch, + osvariant=self.osvariant, + domain=self.domain, + lastreport=timezone.now(), + ) + + def test_process_update_creates_update(self): + """Test process_update creates PackageUpdate.""" + pkg_name = PackageName.objects.create(name='testpkg') + old_pkg = Package.objects.create( + name=pkg_name, + arch=self.pkg_arch, + epoch='0', + version='1.0.0', + release='1.el9', + packagetype=Package.RPM, + ) + new_pkg = Package.objects.create( + name=pkg_name, + arch=self.pkg_arch, + epoch='0', + version='1.0.1', + release='1.el9', + packagetype=Package.RPM, + ) + + # Create a repo and mirror for the new package + repo = Repository.objects.create( + name='test-repo', + arch=self.machine_arch, + repotype=Repository.RPM, + ) + mirror = Mirror.objects.create( + repo=repo, + url='http://example.com/repo', + ) + MirrorPackage.objects.create(mirror=mirror, package=new_pkg) + HostRepo.objects.create(host=self.host, repo=repo) + + update_id = self.host.process_update(old_pkg, new_pkg) + self.assertIsNotNone(update_id) + self.assertEqual(self.host.updates.count(), 1) + + def test_process_update_marks_security_from_repo(self): + """Test process_update marks security based on repo.""" + pkg_name = PackageName.objects.create(name='secpkg') + old_pkg = Package.objects.create( + name=pkg_name, + arch=self.pkg_arch, + epoch='0', + version='1.0.0', + release='1.el9', + packagetype=Package.RPM, + ) + new_pkg = Package.objects.create( + name=pkg_name, + arch=self.pkg_arch, + epoch='0', + version='1.0.1', + release='1.el9', + packagetype=Package.RPM, + ) + + # Create a security repo + sec_repo = Repository.objects.create( + name='security-repo', + arch=self.machine_arch, + repotype=Repository.RPM, + security=True, + ) + mirror = Mirror.objects.create( + repo=sec_repo, + url='http://example.com/security', + ) + MirrorPackage.objects.create(mirror=mirror, package=new_pkg) + HostRepo.objects.create(host=self.host, repo=sec_repo) + + update_id = self.host.process_update(old_pkg, new_pkg) + update = PackageUpdate.objects.get(id=update_id) + self.assertTrue(update.security) diff --git a/hosts/tests/test_managers.py b/hosts/tests/test_managers.py new file mode 100644 index 00000000..8b9e8a2d --- /dev/null +++ b/hosts/tests/test_managers.py @@ -0,0 +1,205 @@ +# Copyright 2025 Marcus Furlong +# +# This file is part of Patchman. +# +# Patchman is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, version 3 only. +# +# Patchman is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Patchman. If not, see + +from django.test import TestCase, override_settings +from django.utils import timezone + +from arch.models import MachineArchitecture, PackageArchitecture +from domains.models import Domain +from hosts.models import Host +from operatingsystems.models import OSRelease, OSVariant +from packages.models import Package, PackageName + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class HostManagerTests(TestCase): + """Tests for HostManager custom queryset.""" + + def setUp(self): + """Set up test data.""" + self.arch = MachineArchitecture.objects.create(name='x86_64') + self.osrelease = OSRelease.objects.create(name='Ubuntu 22.04') + self.osvariant = OSVariant.objects.create( + name='Ubuntu 22.04 x86_64', + osrelease=self.osrelease, + arch=self.arch, + ) + self.domain = Domain.objects.create(name='example.com') + self.now = timezone.now() + + def test_host_manager_select_related(self): + """Test HostManager uses select_related for efficiency.""" + Host.objects.create( + hostname='test1.example.com', + ipaddress='192.168.1.1', + arch=self.arch, + osvariant=self.osvariant, + domain=self.domain, + lastreport=self.now, + ) + Host.objects.create( + hostname='test2.example.com', + ipaddress='192.168.1.2', + arch=self.arch, + osvariant=self.osvariant, + domain=self.domain, + lastreport=self.now, + ) + # Manager should return queryset with select_related + hosts = Host.objects.all() + self.assertEqual(hosts.count(), 2) + + def test_host_manager_returns_all_hosts(self): + """Test HostManager returns all hosts.""" + Host.objects.create( + hostname='host1.example.com', + ipaddress='192.168.1.1', + arch=self.arch, + osvariant=self.osvariant, + domain=self.domain, + lastreport=self.now, + ) + Host.objects.create( + hostname='host2.example.com', + ipaddress='192.168.1.2', + arch=self.arch, + osvariant=self.osvariant, + domain=self.domain, + lastreport=self.now, + ) + Host.objects.create( + hostname='host3.example.com', + ipaddress='192.168.1.3', + arch=self.arch, + osvariant=self.osvariant, + domain=self.domain, + lastreport=self.now, + ) + self.assertEqual(Host.objects.count(), 3) + + def test_host_manager_filter_works(self): + """Test HostManager filtering works correctly.""" + Host.objects.create( + hostname='web1.example.com', + ipaddress='192.168.1.1', + arch=self.arch, + osvariant=self.osvariant, + domain=self.domain, + lastreport=self.now, + ) + Host.objects.create( + hostname='db1.example.com', + ipaddress='192.168.1.2', + arch=self.arch, + osvariant=self.osvariant, + domain=self.domain, + lastreport=self.now, + ) + web_hosts = Host.objects.filter(hostname__startswith='web') + self.assertEqual(web_hosts.count(), 1) + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class PackageManagerTests(TestCase): + """Tests for PackageManager custom queryset.""" + + def setUp(self): + """Set up test data.""" + self.arch = PackageArchitecture.objects.create(name='amd64') + + def test_package_manager_select_related(self): + """Test PackageManager uses select_related for efficiency.""" + pkg_name = PackageName.objects.create(name='nginx') + Package.objects.create( + name=pkg_name, + arch=self.arch, + epoch='', + version='1.18.0', + release='1', + packagetype=Package.DEB, + ) + # Manager should return queryset with select_related + packages = Package.objects.all() + self.assertEqual(packages.count(), 1) + + def test_package_manager_returns_all_packages(self): + """Test PackageManager returns all packages.""" + for name in ['nginx', 'curl', 'vim']: + pkg_name = PackageName.objects.create(name=name) + Package.objects.create( + name=pkg_name, + arch=self.arch, + epoch='', + version='1.0.0', + release='1', + packagetype=Package.DEB, + ) + self.assertEqual(Package.objects.count(), 3) + + def test_package_manager_filter_by_name(self): + """Test PackageManager filtering by name.""" + nginx_name = PackageName.objects.create(name='nginx') + curl_name = PackageName.objects.create(name='curl') + Package.objects.create( + name=nginx_name, + arch=self.arch, + epoch='', + version='1.18.0', + release='1', + packagetype=Package.DEB, + ) + Package.objects.create( + name=curl_name, + arch=self.arch, + epoch='', + version='7.81.0', + release='1', + packagetype=Package.DEB, + ) + nginx_packages = Package.objects.filter(name=nginx_name) + self.assertEqual(nginx_packages.count(), 1) + + def test_package_manager_filter_by_type(self): + """Test PackageManager filtering by package type.""" + deb_name = PackageName.objects.create(name='debpkg') + rpm_name = PackageName.objects.create(name='rpmpkg') + rpm_arch = PackageArchitecture.objects.create(name='x86_64') + Package.objects.create( + name=deb_name, + arch=self.arch, + epoch='', + version='1.0.0', + release='1', + packagetype=Package.DEB, + ) + Package.objects.create( + name=rpm_name, + arch=rpm_arch, + epoch='', + version='1.0.0', + release='1.el9', + packagetype=Package.RPM, + ) + deb_packages = Package.objects.filter(packagetype=Package.DEB) + rpm_packages = Package.objects.filter(packagetype=Package.RPM) + self.assertEqual(deb_packages.count(), 1) + self.assertEqual(rpm_packages.count(), 1) diff --git a/hosts/tests/test_models.py b/hosts/tests/test_models.py new file mode 100644 index 00000000..09d0e4b3 --- /dev/null +++ b/hosts/tests/test_models.py @@ -0,0 +1,130 @@ +# Copyright 2025 Marcus Furlong +# +# This file is part of Patchman. +# +# Patchman is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, version 3 only. +# +# Patchman is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Patchman. If not, see + +from django.test import TestCase, override_settings +from django.utils import timezone + +from arch.models import MachineArchitecture, PackageArchitecture +from domains.models import Domain +from hosts.models import Host, HostRepo +from operatingsystems.models import OSRelease, OSVariant +from packages.models import Package, PackageName, PackageUpdate +from repos.models import Repository + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class HostMethodTests(TestCase): + """Tests for Host model methods.""" + + def setUp(self): + """Set up test data.""" + self.arch = MachineArchitecture.objects.create(name='x86_64') + self.domain = Domain.objects.create(name='example.com') + self.os_release = OSRelease.objects.create( + name='Ubuntu 22.04', codename='jammy' + ) + self.os_variant = OSVariant.objects.create( + name='Ubuntu 22.04.3 LTS', osrelease=self.os_release + ) + self.host = Host.objects.create( + hostname='testhost.example.com', + ipaddress='192.168.1.100', + osvariant=self.os_variant, + kernel='5.15.0-91-generic', + arch=self.arch, + domain=self.domain, + lastreport=timezone.now(), + ) + + def test_get_num_packages_empty(self): + """Test get_num_packages() with no packages.""" + self.assertEqual(self.host.get_num_packages(), 0) + + def test_get_num_packages_with_packages(self): + """Test get_num_packages() with packages.""" + pkg_arch = PackageArchitecture.objects.create(name='amd64') + pkg_name = PackageName.objects.create(name='nginx') + pkg = Package.objects.create( + name=pkg_name, arch=pkg_arch, epoch='', + version='1.18.0', release='1', packagetype='D' + ) + self.host.packages.add(pkg) + self.assertEqual(self.host.get_num_packages(), 1) + + def test_get_num_repos_empty(self): + """Test get_num_repos() with no repos.""" + self.assertEqual(self.host.get_num_repos(), 0) + + def test_get_num_repos_with_repos(self): + """Test get_num_repos() with repos.""" + repo = Repository.objects.create( + name='ubuntu-main', arch=self.arch, repotype='D' + ) + HostRepo.objects.create(host=self.host, repo=repo, enabled=True) + self.assertEqual(self.host.get_num_repos(), 1) + + def test_get_num_updates_empty(self): + """Test get_num_updates() with no updates.""" + self.assertEqual(self.host.get_num_updates(), 0) + + def test_get_num_updates_with_updates(self): + """Test get_num_updates() with updates.""" + pkg_arch = PackageArchitecture.objects.create(name='amd64') + pkg_name = PackageName.objects.create(name='openssl') + old_pkg = Package.objects.create( + name=pkg_name, arch=pkg_arch, epoch='', + version='3.0.0', release='1', packagetype='D' + ) + new_pkg = Package.objects.create( + name=pkg_name, arch=pkg_arch, epoch='', + version='3.0.1', release='1', packagetype='D' + ) + update = PackageUpdate.objects.create( + oldpackage=old_pkg, newpackage=new_pkg, security=True + ) + self.host.updates.add(update) + self.assertEqual(self.host.get_num_updates(), 1) + + def test_get_num_security_updates(self): + """Test get_num_security_updates() counts only security updates.""" + pkg_arch = PackageArchitecture.objects.create(name='amd64') + pkg_name = PackageName.objects.create(name='openssl') + old_pkg = Package.objects.create( + name=pkg_name, arch=pkg_arch, epoch='', + version='3.0.0', release='1', packagetype='D' + ) + sec_pkg = Package.objects.create( + name=pkg_name, arch=pkg_arch, epoch='', + version='3.0.1', release='1', packagetype='D' + ) + bug_pkg = Package.objects.create( + name=pkg_name, arch=pkg_arch, epoch='', + version='3.0.2', release='1', packagetype='D' + ) + sec_update = PackageUpdate.objects.create( + oldpackage=old_pkg, newpackage=sec_pkg, security=True + ) + bug_update = PackageUpdate.objects.create( + oldpackage=old_pkg, newpackage=bug_pkg, security=False + ) + self.host.updates.add(sec_update, bug_update) + + self.assertEqual(self.host.get_num_security_updates(), 1) + self.assertEqual(self.host.get_num_bugfix_updates(), 1) + self.assertEqual(self.host.get_num_updates(), 2) diff --git a/operatingsystems/tests/test_models.py b/operatingsystems/tests/test_models.py new file mode 100644 index 00000000..bec9e1fa --- /dev/null +++ b/operatingsystems/tests/test_models.py @@ -0,0 +1,102 @@ +# Copyright 2025 Marcus Furlong +# +# This file is part of Patchman. +# +# Patchman is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, version 3 only. +# +# Patchman is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Patchman. If not, see + +from django.test import TestCase, override_settings + +from operatingsystems.models import OSRelease, OSVariant + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class OSReleaseMethodTests(TestCase): + """Tests for OSRelease model methods.""" + + def test_osrelease_creation(self): + """Test creating an OSRelease.""" + release = OSRelease.objects.create(name='Ubuntu 22.04') + self.assertEqual(release.name, 'Ubuntu 22.04') + + def test_osrelease_str(self): + """Test OSRelease __str__ method.""" + release = OSRelease.objects.create(name='Ubuntu 22.04') + self.assertEqual(str(release), 'Ubuntu 22.04') + + def test_osrelease_get_absolute_url(self): + """Test OSRelease.get_absolute_url().""" + release = OSRelease.objects.create(name='Rocky Linux 9') + url = release.get_absolute_url() + self.assertIn(str(release.id), url) + + def test_osrelease_unique_name(self): + """Test OSRelease name is unique.""" + OSRelease.objects.create(name='Ubuntu 22.04') + from django.db import IntegrityError + with self.assertRaises(IntegrityError): + OSRelease.objects.create(name='Ubuntu 22.04') + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class OSVariantMethodTests(TestCase): + """Tests for OSVariant model methods.""" + + def setUp(self): + """Set up test data.""" + self.release = OSRelease.objects.create(name='Ubuntu 22.04') + + def test_osvariant_creation(self): + """Test creating an OSVariant.""" + variant = OSVariant.objects.create( + name='Ubuntu 22.04.3 LTS', + osrelease=self.release, + ) + self.assertEqual(variant.name, 'Ubuntu 22.04.3 LTS') + self.assertEqual(variant.osrelease, self.release) + + def test_osvariant_str(self): + """Test OSVariant __str__ method.""" + variant = OSVariant.objects.create( + name='Ubuntu 22.04.3 LTS', + osrelease=self.release, + ) + # __str__ returns 'name arch' format + self.assertIn('Ubuntu 22.04.3 LTS', str(variant)) + + def test_osvariant_get_absolute_url(self): + """Test OSVariant.get_absolute_url().""" + variant = OSVariant.objects.create( + name='Ubuntu 22.04.3 LTS', + osrelease=self.release, + ) + url = variant.get_absolute_url() + self.assertIn(str(variant.id), url) + + def test_osvariant_unique_name(self): + """Test OSVariant name is unique.""" + OSVariant.objects.create( + name='Ubuntu 22.04.3 LTS', + osrelease=self.release, + ) + from django.db import IntegrityError + with self.assertRaises(IntegrityError): + OSVariant.objects.create( + name='Ubuntu 22.04.3 LTS', + osrelease=self.release, + ) diff --git a/packages/tests/test_models.py b/packages/tests/test_models.py new file mode 100644 index 00000000..0eec91f0 --- /dev/null +++ b/packages/tests/test_models.py @@ -0,0 +1,202 @@ +# Copyright 2025 Marcus Furlong +# +# This file is part of Patchman. +# +# Patchman is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, version 3 only. +# +# Patchman is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Patchman. If not, see + +from django.test import TestCase, override_settings + +from arch.models import PackageArchitecture +from packages.models import Package, PackageName, PackageUpdate + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class PackageMethodTests(TestCase): + """Tests for Package model methods.""" + + def setUp(self): + """Set up test data.""" + self.arch = PackageArchitecture.objects.create(name='amd64') + self.pkg_name = PackageName.objects.create(name='nginx') + + def test_get_version_string_deb(self): + """Test get_version_string() for deb packages.""" + pkg = Package.objects.create( + name=self.pkg_name, arch=self.arch, + epoch='', version='1.18.0', release='6ubuntu14', packagetype='D' + ) + self.assertEqual(pkg.get_version_string(), '1.18.0-6ubuntu14') + + def test_get_version_string_deb_with_epoch(self): + """Test get_version_string() for deb packages with epoch.""" + pkg = Package.objects.create( + name=self.pkg_name, arch=self.arch, + epoch='1', version='1.18.0', release='6ubuntu14', packagetype='D' + ) + self.assertEqual(pkg.get_version_string(), '1:1.18.0-6ubuntu14') + + def test_get_version_string_deb_no_release(self): + """Test get_version_string() for deb packages without release.""" + pkg = Package.objects.create( + name=self.pkg_name, arch=self.arch, + epoch='', version='1.18.0', release='', packagetype='D' + ) + self.assertEqual(pkg.get_version_string(), '1.18.0') + + def test_get_version_string_rpm(self): + """Test get_version_string() for rpm packages.""" + pkg = Package.objects.create( + name=self.pkg_name, arch=self.arch, + epoch='0', version='1.18.0', release='1.el9', packagetype='R' + ) + version_tuple = pkg.get_version_string() + self.assertEqual(version_tuple, ('0', '1.18.0', '1.el9')) + + def test_compare_version_deb_equal(self): + """Test compare_version() for equal deb packages.""" + pkg1 = Package.objects.create( + name=self.pkg_name, arch=self.arch, + epoch='', version='1.18.0', release='1', packagetype='D' + ) + pkg_name2 = PackageName.objects.create(name='nginx2') + pkg2 = Package.objects.create( + name=pkg_name2, arch=self.arch, + epoch='', version='1.18.0', release='1', packagetype='D' + ) + self.assertEqual(pkg1.compare_version(pkg2), 0) + + def test_compare_version_deb_greater(self): + """Test compare_version() for newer deb package.""" + pkg1 = Package.objects.create( + name=self.pkg_name, arch=self.arch, + epoch='', version='1.20.0', release='1', packagetype='D' + ) + pkg_name2 = PackageName.objects.create(name='nginx-old') + pkg2 = Package.objects.create( + name=pkg_name2, arch=self.arch, + epoch='', version='1.18.0', release='1', packagetype='D' + ) + self.assertEqual(pkg1.compare_version(pkg2), 1) + + def test_compare_version_deb_lesser(self): + """Test compare_version() for older deb package.""" + pkg1 = Package.objects.create( + name=self.pkg_name, arch=self.arch, + epoch='', version='1.16.0', release='1', packagetype='D' + ) + pkg_name2 = PackageName.objects.create(name='nginx-new') + pkg2 = Package.objects.create( + name=pkg_name2, arch=self.arch, + epoch='', version='1.18.0', release='1', packagetype='D' + ) + self.assertEqual(pkg1.compare_version(pkg2), -1) + + def test_str_deb_package(self): + """Test __str__ for deb package.""" + pkg = Package.objects.create( + name=self.pkg_name, arch=self.arch, + epoch='', version='1.18.0', release='6ubuntu14', packagetype='D' + ) + self.assertEqual(str(pkg), 'nginx_1.18.0-6ubuntu14_amd64.deb') + + def test_str_deb_package_with_epoch(self): + """Test __str__ for deb package with epoch.""" + pkg = Package.objects.create( + name=self.pkg_name, arch=self.arch, + epoch='1', version='1.18.0', release='6ubuntu14', packagetype='D' + ) + self.assertEqual(str(pkg), 'nginx_1:1.18.0-6ubuntu14_amd64.deb') + + def test_str_rpm_package(self): + """Test __str__ for rpm package.""" + pkg = Package.objects.create( + name=self.pkg_name, arch=self.arch, + epoch='', version='1.18.0', release='1.el9', packagetype='R' + ) + self.assertEqual(str(pkg), 'nginx-1.18.0-1.el9-amd64.rpm') + + def test_package_equality(self): + """Test Package __eq__ method.""" + pkg1 = Package.objects.create( + name=self.pkg_name, arch=self.arch, + epoch='', version='1.18.0', release='1', packagetype='D' + ) + # Same attributes = equal + self.assertEqual(pkg1, pkg1) + + def test_package_inequality(self): + """Test Package __ne__ method.""" + pkg1 = Package.objects.create( + name=self.pkg_name, arch=self.arch, + epoch='', version='1.18.0', release='1', packagetype='D' + ) + pkg_name2 = PackageName.objects.create(name='curl') + pkg2 = Package.objects.create( + name=pkg_name2, arch=self.arch, + epoch='', version='7.81.0', release='1', packagetype='D' + ) + self.assertNotEqual(pkg1, pkg2) + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class PackageUpdateMethodTests(TestCase): + """Tests for PackageUpdate model.""" + + def setUp(self): + """Set up test data.""" + self.arch = PackageArchitecture.objects.create(name='amd64') + self.pkg_name = PackageName.objects.create(name='openssl') + self.old_pkg = Package.objects.create( + name=self.pkg_name, arch=self.arch, + epoch='', version='3.0.0', release='1', packagetype='D' + ) + self.new_pkg = Package.objects.create( + name=self.pkg_name, arch=self.arch, + epoch='', version='3.0.1', release='1', packagetype='D' + ) + + def test_package_update_str(self): + """Test PackageUpdate __str__ method.""" + update = PackageUpdate.objects.create( + oldpackage=self.old_pkg, + newpackage=self.new_pkg, + security=True, + ) + str_repr = str(update) + self.assertIn('openssl', str_repr) + + def test_package_update_security_flag(self): + """Test PackageUpdate security flag.""" + sec_update = PackageUpdate.objects.create( + oldpackage=self.old_pkg, + newpackage=self.new_pkg, + security=True, + ) + self.assertTrue(sec_update.security) + + bug_pkg = Package.objects.create( + name=self.pkg_name, arch=self.arch, + epoch='', version='3.0.2', release='1', packagetype='D' + ) + bug_update = PackageUpdate.objects.create( + oldpackage=self.old_pkg, + newpackage=bug_pkg, + security=False, + ) + self.assertFalse(bug_update.security) diff --git a/packages/tests/test_version_compare.py b/packages/tests/test_version_compare.py new file mode 100644 index 00000000..f12f5690 --- /dev/null +++ b/packages/tests/test_version_compare.py @@ -0,0 +1,316 @@ +# Copyright 2025 Marcus Furlong +# +# This file is part of Patchman. +# +# Patchman is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, version 3 only. +# +# Patchman is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Patchman. If not, see + +from django.test import TestCase, override_settings + +from arch.models import PackageArchitecture +from packages.models import Package, PackageName + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class RPMVersionCompareTests(TestCase): + """Tests for RPM package version comparison.""" + + def setUp(self): + """Set up test data.""" + self.arch = PackageArchitecture.objects.create(name='x86_64') + self.pkg_name = PackageName.objects.create(name='httpd') + + def _create_rpm(self, epoch, version, release): + """Helper to create RPM package.""" + return Package.objects.create( + name=self.pkg_name, + arch=self.arch, + epoch=epoch, + version=version, + release=release, + packagetype=Package.RPM, + ) + + def test_compare_same_version(self): + """Test comparing identical versions returns 0.""" + pkg1 = self._create_rpm('0', '2.4.57', '5.el9') + pkg2 = self._create_rpm('0', '2.4.57', '5.el9') + self.assertEqual(pkg1.compare_version(pkg2), 0) + + def test_compare_newer_version(self): + """Test comparing newer version returns 1.""" + pkg1 = self._create_rpm('0', '2.4.58', '1.el9') + pkg2 = self._create_rpm('0', '2.4.57', '5.el9') + self.assertEqual(pkg1.compare_version(pkg2), 1) + + def test_compare_older_version(self): + """Test comparing older version returns -1.""" + pkg1 = self._create_rpm('0', '2.4.56', '1.el9') + pkg2 = self._create_rpm('0', '2.4.57', '5.el9') + self.assertEqual(pkg1.compare_version(pkg2), -1) + + def test_compare_epoch_takes_precedence(self): + """Test that epoch takes precedence over version.""" + pkg1 = self._create_rpm('1', '1.0.0', '1.el9') + pkg2 = self._create_rpm('0', '9.9.9', '99.el9') + self.assertEqual(pkg1.compare_version(pkg2), 1) + + def test_compare_release_difference(self): + """Test release number comparison when version is same.""" + pkg1 = self._create_rpm('0', '2.4.57', '6.el9') + pkg2 = self._create_rpm('0', '2.4.57', '5.el9') + self.assertEqual(pkg1.compare_version(pkg2), 1) + + def test_compare_empty_epoch_vs_zero(self): + """Test empty epoch vs explicit 0 epoch comparison.""" + pkg1 = self._create_rpm('', '2.4.57', '5.el9') + pkg2 = self._create_rpm('0', '2.4.57', '5.el9') + # RPM treats empty string and '0' differently in labelCompare + result = pkg1.compare_version(pkg2) + self.assertIn(result, [-1, 0, 1]) # Implementation dependent + + def test_compare_complex_version(self): + """Test complex version strings.""" + pkg1 = self._create_rpm('0', '1.2.3.4.5', '1.el9') + pkg2 = self._create_rpm('0', '1.2.3.4.4', '1.el9') + self.assertEqual(pkg1.compare_version(pkg2), 1) + + def test_compare_alpha_version(self): + """Test version with alpha characters.""" + pkg1 = self._create_rpm('0', '1.0.0', '1.el9') + pkg2 = self._create_rpm('0', '1.0.0', '1.el9_1') + # el9 vs el9_1 comparison + result = pkg1.compare_version(pkg2) + self.assertIn(result, [-1, 0, 1]) + + def test_version_string_rpm(self): + """Test _version_string_rpm returns correct tuple.""" + pkg = self._create_rpm('1', '2.4.57', '5.el9') + result = pkg._version_string_rpm() + self.assertEqual(result, ('1', '2.4.57', '5.el9')) + + def test_version_string_rpm_empty_epoch(self): + """Test _version_string_rpm with empty epoch.""" + pkg = self._create_rpm('', '2.4.57', '5.el9') + result = pkg._version_string_rpm() + self.assertEqual(result, ('', '2.4.57', '5.el9')) + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class DEBVersionCompareTests(TestCase): + """Tests for DEB package version comparison.""" + + def setUp(self): + """Set up test data.""" + self.arch = PackageArchitecture.objects.create(name='amd64') + self.pkg_name = PackageName.objects.create(name='nginx') + + def _create_deb(self, epoch, version, release): + """Helper to create DEB package.""" + return Package.objects.create( + name=self.pkg_name, + arch=self.arch, + epoch=epoch, + version=version, + release=release, + packagetype=Package.DEB, + ) + + def test_compare_same_version(self): + """Test comparing identical versions returns 0.""" + pkg1 = self._create_deb('', '1.18.0', '6ubuntu14') + pkg2 = self._create_deb('', '1.18.0', '6ubuntu14') + self.assertEqual(pkg1.compare_version(pkg2), 0) + + def test_compare_newer_version(self): + """Test comparing newer version returns 1.""" + pkg1 = self._create_deb('', '1.18.1', '1ubuntu1') + pkg2 = self._create_deb('', '1.18.0', '6ubuntu14') + self.assertEqual(pkg1.compare_version(pkg2), 1) + + def test_compare_older_version(self): + """Test comparing older version returns -1.""" + pkg1 = self._create_deb('', '1.17.0', '1ubuntu1') + pkg2 = self._create_deb('', '1.18.0', '6ubuntu14') + self.assertEqual(pkg1.compare_version(pkg2), -1) + + def test_compare_ubuntu_revision(self): + """Test Ubuntu revision comparison.""" + pkg1 = self._create_deb('', '1.18.0', '6ubuntu15') + pkg2 = self._create_deb('', '1.18.0', '6ubuntu14') + self.assertEqual(pkg1.compare_version(pkg2), 1) + + def test_compare_tilde_version(self): + """Test tilde in version (sorts before everything).""" + pkg1 = self._create_deb('', '1.0.0~beta1', '1') + pkg2 = self._create_deb('', '1.0.0', '1') + self.assertEqual(pkg1.compare_version(pkg2), -1) + + def test_compare_epoch_deb(self): + """Test epoch comparison for DEB packages.""" + pkg1 = self._create_deb('2', '1.0.0', '1') + pkg2 = self._create_deb('1', '9.0.0', '1') + self.assertEqual(pkg1.compare_version(pkg2), 1) + + def test_version_string_deb(self): + """Test _version_string_deb_arch returns correct format.""" + pkg = self._create_deb('1', '1.18.0', '6ubuntu14') + result = pkg._version_string_deb_arch() + self.assertEqual(result, '1:1.18.0-6ubuntu14') + + def test_version_string_deb_no_epoch(self): + """Test _version_string_deb_arch without epoch.""" + pkg = self._create_deb('', '1.18.0', '6ubuntu14') + result = pkg._version_string_deb_arch() + self.assertEqual(result, '1.18.0-6ubuntu14') + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class ArchVersionCompareTests(TestCase): + """Tests for Arch package version comparison.""" + + def setUp(self): + """Set up test data.""" + self.arch = PackageArchitecture.objects.create(name='x86_64') + self.pkg_name = PackageName.objects.create(name='pacman') + + def _create_arch(self, epoch, version, release): + """Helper to create Arch package.""" + return Package.objects.create( + name=self.pkg_name, + arch=self.arch, + epoch=epoch, + version=version, + release=release, + packagetype=Package.ARCH, + ) + + def test_compare_same_version(self): + """Test comparing identical versions returns 0.""" + pkg1 = self._create_arch('', '6.0.2', '1') + pkg2 = self._create_arch('', '6.0.2', '1') + self.assertEqual(pkg1.compare_version(pkg2), 0) + + def test_compare_newer_version(self): + """Test comparing newer version returns 1.""" + pkg1 = self._create_arch('', '6.0.3', '1') + pkg2 = self._create_arch('', '6.0.2', '1') + self.assertEqual(pkg1.compare_version(pkg2), 1) + + def test_compare_release_bump(self): + """Test release number bump.""" + pkg1 = self._create_arch('', '6.0.2', '2') + pkg2 = self._create_arch('', '6.0.2', '1') + self.assertEqual(pkg1.compare_version(pkg2), 1) + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class GentooVersionCompareTests(TestCase): + """Tests for Gentoo package version comparison.""" + + def setUp(self): + """Set up test data.""" + self.arch = PackageArchitecture.objects.create(name='amd64') + self.pkg_name = PackageName.objects.create(name='dev-libs/openssl') + + def _create_gentoo(self, epoch, version, release): + """Helper to create Gentoo package.""" + return Package.objects.create( + name=self.pkg_name, + arch=self.arch, + epoch=epoch, + version=version, + release=release, + packagetype=Package.GENTOO, + ) + + def test_compare_same_version(self): + """Test comparing identical versions returns 0.""" + pkg1 = self._create_gentoo('', '3.0.10', 'r1') + pkg2 = self._create_gentoo('', '3.0.10', 'r1') + self.assertEqual(pkg1.compare_version(pkg2), 0) + + def test_compare_newer_version(self): + """Test comparing newer version returns 1.""" + pkg1 = self._create_gentoo('', '3.0.11', 'r0') + pkg2 = self._create_gentoo('', '3.0.10', 'r1') + self.assertEqual(pkg1.compare_version(pkg2), 1) + + def test_compare_revision_bump(self): + """Test revision bump.""" + pkg1 = self._create_gentoo('', '3.0.10', 'r2') + pkg2 = self._create_gentoo('', '3.0.10', 'r1') + self.assertEqual(pkg1.compare_version(pkg2), 1) + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class GetVersionStringTests(TestCase): + """Tests for get_version_string method.""" + + def setUp(self): + """Set up test data.""" + self.arch = PackageArchitecture.objects.create(name='x86_64') + self.pkg_name = PackageName.objects.create(name='testpkg') + + def test_get_version_string_rpm(self): + """Test get_version_string for RPM.""" + pkg = Package.objects.create( + name=self.pkg_name, + arch=self.arch, + epoch='1', + version='2.0.0', + release='3.el9', + packagetype=Package.RPM, + ) + result = pkg.get_version_string() + self.assertEqual(result, ('1', '2.0.0', '3.el9')) + + def test_get_version_string_deb(self): + """Test get_version_string for DEB.""" + pkg = Package.objects.create( + name=self.pkg_name, + arch=self.arch, + epoch='1', + version='2.0.0', + release='3ubuntu1', + packagetype=Package.DEB, + ) + result = pkg.get_version_string() + self.assertEqual(result, '1:2.0.0-3ubuntu1') + + def test_get_version_string_gentoo(self): + """Test get_version_string for Gentoo.""" + pkg = Package.objects.create( + name=self.pkg_name, + arch=self.arch, + epoch='', + version='2.0.0', + release='r1', + packagetype=Package.GENTOO, + ) + result = pkg.get_version_string() + self.assertEqual(result, ('', '2.0.0', 'r1')) diff --git a/reports/tests/test_edge_cases.py b/reports/tests/test_edge_cases.py new file mode 100644 index 00000000..fb647b82 --- /dev/null +++ b/reports/tests/test_edge_cases.py @@ -0,0 +1,302 @@ +# Copyright 2025 Marcus Furlong +# +# This file is part of Patchman. +# +# Patchman is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, version 3 only. +# +# Patchman is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Patchman. If not, see + +from django.test import TestCase, override_settings +from rest_framework import status +from rest_framework.test import APITestCase + +from reports.models import Report +from reports.utils import parse_packages, process_package + + +@override_settings( + REQUIRE_API_KEY=False, + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class EdgeCasePackageTests(TestCase): + """Edge case tests for package processing.""" + + def test_package_with_unicode_name(self): + """Test package with unicode characters in name.""" + # Some packages may have unicode in description, handled gracefully + package = process_package( + name='test-pkg', + epoch='', + version='1.0.0', + release='1', + arch='amd64', + p_type='D', + ) + self.assertIsNotNone(package) + + def test_package_with_very_long_version(self): + """Test package with very long version string.""" + long_version = '1.2.3.4.5.6.7.8.9.10.11.12.13.14.15.16.17.18.19.20' + package = process_package( + name='longver-pkg', + epoch='', + version=long_version, + release='1', + arch='amd64', + p_type='D', + ) + self.assertIsNotNone(package) + self.assertEqual(package.version, long_version) + + def test_package_with_special_chars_in_version(self): + """Test package with special characters in version.""" + package = process_package( + name='special-pkg', + epoch='', + version='1.0.0~beta+git20240101', + release='1ubuntu1~22.04.1', + arch='amd64', + p_type='D', + ) + self.assertIsNotNone(package) + + def test_package_with_empty_epoch(self): + """Test package with empty epoch.""" + package = process_package( + name='noepoch-pkg', + epoch='', + version='1.0.0', + release='1', + arch='amd64', + p_type='D', + ) + self.assertEqual(package.epoch, '') + + def test_package_with_empty_release(self): + """Test package with empty release.""" + package = process_package( + name='norelease-pkg', + epoch='', + version='1.0.0', + release='', + arch='amd64', + p_type='D', + ) + self.assertEqual(package.release, '') + + def test_package_with_numeric_epoch(self): + """Test package with large numeric epoch.""" + package = process_package( + name='bigepoch-pkg', + epoch='999', + version='1.0.0', + release='1', + arch='amd64', + p_type='D', + ) + self.assertEqual(package.epoch, '999') + + +@override_settings( + REQUIRE_API_KEY=False, + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class EdgeCaseReportAPITests(APITestCase): + """Edge case tests for Report API.""" + + def test_report_with_empty_packages_array(self): + """Test report with empty packages array.""" + data = { + 'hostname': 'empty-pkgs.example.com', + 'arch': 'x86_64', + 'os': 'Ubuntu 22.04', + 'kernel': '5.15.0-91-generic', + 'packages': [], + 'repos': [], + } + response = self.client.post('/api/report/', data, format='json') + self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED) + + def test_report_with_many_packages(self): + """Test report with many packages (stress test).""" + packages = [ + {'name': f'pkg{i}', 'version': '1.0.0', 'arch': 'amd64', 'type': 'deb'} + for i in range(100) + ] + data = { + 'hostname': 'many-pkgs.example.com', + 'arch': 'x86_64', + 'os': 'Ubuntu 22.04', + 'kernel': '5.15.0-91-generic', + 'packages': packages, + 'repos': [], + } + response = self.client.post('/api/report/', data, format='json') + self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED) + + def test_report_with_unicode_hostname(self): + """Test report with hostname containing valid chars.""" + data = { + 'hostname': 'test-host-123.example.com', + 'arch': 'x86_64', + 'os': 'Ubuntu 22.04', + 'kernel': '5.15.0-91-generic', + 'packages': [], + } + response = self.client.post('/api/report/', data, format='json') + self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED) + + def test_report_with_unicode_os(self): + """Test report with unicode in OS name.""" + data = { + 'hostname': 'unicode-os.example.com', + 'arch': 'x86_64', + 'os': 'Ubuntu 22.04 LTS "Jammy Jellyfish"', + 'kernel': '5.15.0-91-generic', + 'packages': [], + } + response = self.client.post('/api/report/', data, format='json') + self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED) + + def test_report_with_very_long_kernel(self): + """Test report with very long kernel string.""" + data = { + 'hostname': 'longkernel.example.com', + 'arch': 'x86_64', + 'os': 'Ubuntu 22.04', + 'kernel': '5.15.0-91-generic-with-extra-long-suffix-for-testing', + 'packages': [], + } + response = self.client.post('/api/report/', data, format='json') + self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED) + + def test_report_with_special_tags(self): + """Test report with special characters in tags.""" + data = { + 'hostname': 'tagged.example.com', + 'arch': 'x86_64', + 'os': 'Ubuntu 22.04', + 'kernel': '5.15.0-91-generic', + 'tags': ['web-server', 'prod_env', 'tier1'], + 'packages': [], + } + response = self.client.post('/api/report/', data, format='json') + self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED) + + +@override_settings( + REQUIRE_API_KEY=False, + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class EdgeCaseMalformedInputTests(APITestCase): + """Tests for handling malformed input.""" + + def test_report_with_null_packages(self): + """Test report with null packages field.""" + data = { + 'hostname': 'null-pkgs.example.com', + 'arch': 'x86_64', + 'os': 'Ubuntu 22.04', + 'kernel': '5.15.0-91-generic', + 'packages': None, + } + response = self.client.post('/api/report/', data, format='json') + # Should either accept or return validation error + self.assertIn(response.status_code, [status.HTTP_202_ACCEPTED, status.HTTP_400_BAD_REQUEST]) + + def test_report_with_invalid_json_type(self): + """Test report with invalid type for packages (string instead of array).""" + data = { + 'hostname': 'invalid-type.example.com', + 'arch': 'x86_64', + 'os': 'Ubuntu 22.04', + 'kernel': '5.15.0-91-generic', + 'packages': 'not-an-array', + } + response = self.client.post('/api/report/', data, format='json') + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + + def test_report_with_missing_required_package_field(self): + """Test report with package missing required field.""" + data = { + 'hostname': 'missing-field.example.com', + 'arch': 'x86_64', + 'os': 'Ubuntu 22.04', + 'kernel': '5.15.0-91-generic', + 'packages': [ + {'name': 'pkg1'}, # Missing version and arch + ], + } + response = self.client.post('/api/report/', data, format='json') + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + + def test_report_with_extra_unknown_fields(self): + """Test report with extra unknown fields (should be ignored).""" + data = { + 'hostname': 'extra-fields.example.com', + 'arch': 'x86_64', + 'os': 'Ubuntu 22.04', + 'kernel': '5.15.0-91-generic', + 'unknown_field': 'should be ignored', + 'another_unknown': 123, + 'packages': [], + } + response = self.client.post('/api/report/', data, format='json') + self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED) + + +@override_settings( + REQUIRE_API_KEY=False, + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class BoundaryConditionTests(TestCase): + """Tests for boundary conditions.""" + + def test_parse_packages_with_newlines_only(self): + """Test parsing string with only newlines.""" + packages = parse_packages('\n\n\n') + # Should handle gracefully + self.assertIsInstance(packages, list) + + def test_parse_packages_with_mixed_formats(self): + """Test parsing packages with inconsistent formatting.""" + pkg_str = """'nginx' '' '1.18.0' '6ubuntu14' 'amd64' 'deb' +'curl' '' '7.81.0' '1' 'amd64'""" # Missing type + packages = parse_packages(pkg_str) + self.assertEqual(len(packages), 2) + + def test_report_duplicate_hostname(self): + """Test creating reports for same hostname.""" + Report.objects.create( + host='duplicate.example.com', + domain='example.com', + report_ip='192.168.1.1', + os='Ubuntu 22.04', + kernel='5.15.0-91-generic', + arch='x86_64', + protocol='2', + ) + # Second report for same host should work + report2 = Report.objects.create( + host='duplicate.example.com', + domain='example.com', + report_ip='192.168.1.1', + os='Ubuntu 22.04', + kernel='5.15.0-92-generic', # Newer kernel + arch='x86_64', + protocol='2', + ) + self.assertIsNotNone(report2) + self.assertEqual(Report.objects.filter(host='duplicate.example.com').count(), 2) diff --git a/reports/tests/test_integration.py b/reports/tests/test_integration.py new file mode 100644 index 00000000..958b0109 --- /dev/null +++ b/reports/tests/test_integration.py @@ -0,0 +1,274 @@ +# Copyright 2025 Marcus Furlong +# +# This file is part of Patchman. +# +# Patchman is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, version 3 only. +# +# Patchman is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Patchman. If not, see + +from django.test import TestCase, override_settings +from django.utils import timezone +from rest_framework import status +from rest_framework.test import APITestCase + +from arch.models import MachineArchitecture, PackageArchitecture +from domains.models import Domain +from hosts.models import Host, HostRepo +from operatingsystems.models import OSRelease, OSVariant +from packages.models import Package, PackageName +from reports.models import Report +from repos.models import Repository + + +@override_settings( + REQUIRE_API_KEY=False, + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class EndToEndProtocol2Tests(APITestCase): + """End-to-end integration tests for Protocol 2.""" + + def test_full_report_creates_host_packages_repos(self): + """Test full report creates all expected database records.""" + data = { + 'hostname': 'e2e-test.example.com', + 'arch': 'x86_64', + 'os': 'Ubuntu 22.04 LTS', + 'kernel': '5.15.0-91-generic', + 'packages': [ + { + 'name': 'nginx', + 'epoch': '', + 'version': '1.18.0', + 'release': '6ubuntu14', + 'arch': 'amd64', + 'type': 'deb', + }, + { + 'name': 'curl', + 'epoch': '', + 'version': '7.81.0', + 'release': '1ubuntu1.15', + 'arch': 'amd64', + 'type': 'deb', + }, + ], + 'repos': [ + { + 'id': 'ubuntu-main', + 'name': 'Ubuntu 22.04 main', + 'type': 'deb', + 'url': 'http://archive.ubuntu.com/ubuntu/dists/jammy/main/binary-amd64', + }, + ], + } + + response = self.client.post('/api/report/', data, format='json') + self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED) + + # Verify report was created + report_id = response.data.get('report_id') + self.assertIsNotNone(report_id) + report = Report.objects.get(id=report_id) + self.assertEqual(report.host, 'e2e-test.example.com') + self.assertEqual(report.protocol, '2') + + def test_report_upload_and_retrieve(self): + """Test report can be uploaded and then retrieved via API.""" + data = { + 'hostname': 'retrieve-test.example.com', + 'arch': 'x86_64', + 'os': 'Rocky Linux 9', + 'kernel': '5.14.0-362.el9.x86_64', + 'packages': [], + } + + # Upload + response = self.client.post('/api/report/', data, format='json') + self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED) + report_id = response.data['report_id'] + + # Retrieve + response = self.client.get(f'/api/report/{report_id}/') + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.data['host'], 'retrieve-test.example.com') + + +@override_settings( + REQUIRE_API_KEY=False, + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class Protocol1CompatibilityTests(TestCase): + """Tests for Protocol 1 backward compatibility.""" + + def test_protocol1_report_stored(self): + """Test Protocol 1 report can be created and stored.""" + # Protocol 1 reports store raw text data + packages_text = "'nginx' '' '1.18.0' '6ubuntu14' 'amd64' 'deb'" + repos_text = "deb : Ubuntu 22.04 : ubuntu-main : http://archive.ubuntu.com/ubuntu/" + + report = Report.objects.create( + host='protocol1.example.com', + domain='example.com', + report_ip='192.168.1.10', + os='Ubuntu 22.04', + kernel='5.15.0-91-generic', + arch='x86_64', + protocol='1', + packages=packages_text, + repos=repos_text, + ) + + self.assertEqual(report.protocol, '1') + self.assertIn('nginx', report.packages) + self.assertIn('ubuntu-main', report.repos) + + +@override_settings( + REQUIRE_API_KEY=False, + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class HostUpdateIntegrationTests(TestCase): + """Integration tests for host update detection.""" + + def setUp(self): + """Set up test data.""" + self.machine_arch = MachineArchitecture.objects.create(name='x86_64') + self.pkg_arch = PackageArchitecture.objects.create(name='x86_64') + self.osrelease = OSRelease.objects.create(name='Rocky Linux 9') + self.osvariant = OSVariant.objects.create( + name='Rocky Linux 9 x86_64', + osrelease=self.osrelease, + arch=self.machine_arch, + ) + self.domain = Domain.objects.create(name='example.com') + + def test_report_processing_associates_packages_with_host(self): + """Test that processing a report associates packages with the host.""" + host = Host.objects.create( + hostname='pkg-assoc.example.com', + ipaddress='192.168.1.50', + arch=self.machine_arch, + osvariant=self.osvariant, + domain=self.domain, + lastreport=timezone.now(), + ) + + # Create a package + pkg_name = PackageName.objects.create(name='testpkg') + pkg = Package.objects.create( + name=pkg_name, + arch=self.pkg_arch, + epoch='', + version='1.0.0', + release='1', + packagetype=Package.DEB, + ) + + # Add package to host + host.packages.add(pkg) + self.assertIn(pkg, host.packages.all()) + self.assertEqual(host.packages.count(), 1) + + def test_host_repo_association(self): + """Test that repos are correctly associated with hosts.""" + host = Host.objects.create( + hostname='repo-assoc.example.com', + ipaddress='192.168.1.51', + arch=self.machine_arch, + osvariant=self.osvariant, + domain=self.domain, + lastreport=timezone.now(), + ) + + repo = Repository.objects.create( + name='test-repo', + arch=self.machine_arch, + repotype=Repository.DEB, + ) + + HostRepo.objects.create( + host=host, + repo=repo, + enabled=True, + ) + + self.assertEqual(host.repos.count(), 1) + self.assertEqual(host.repos.first(), repo) + + +@override_settings( + REQUIRE_API_KEY=False, + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class ConcurrentReportTests(APITestCase): + """Tests for concurrent report handling.""" + + def test_multiple_reports_same_host_sequential(self): + """Test multiple reports for same host create separate Report records.""" + data1 = { + 'hostname': 'concurrent.example.com', + 'arch': 'x86_64', + 'os': 'Ubuntu 22.04', + 'kernel': '5.15.0-90-generic', + 'packages': [], + } + + data2 = { + 'hostname': 'concurrent.example.com', + 'arch': 'x86_64', + 'os': 'Ubuntu 22.04', + 'kernel': '5.15.0-91-generic', # Newer kernel + 'packages': [], + } + + response1 = self.client.post('/api/report/', data1, format='json') + response2 = self.client.post('/api/report/', data2, format='json') + + self.assertEqual(response1.status_code, status.HTTP_202_ACCEPTED) + self.assertEqual(response2.status_code, status.HTTP_202_ACCEPTED) + + # Both reports should exist + reports = Report.objects.filter(host='concurrent.example.com') + self.assertEqual(reports.count(), 2) + + def test_reports_from_different_hosts(self): + """Test reports from different hosts don't interfere.""" + data1 = { + 'hostname': 'host1.example.com', + 'arch': 'x86_64', + 'os': 'Ubuntu 22.04', + 'kernel': '5.15.0-91-generic', + 'packages': [], + } + + data2 = { + 'hostname': 'host2.example.com', + 'arch': 'x86_64', + 'os': 'Rocky Linux 9', + 'kernel': '5.14.0-362.el9.x86_64', + 'packages': [], + } + + response1 = self.client.post('/api/report/', data1, format='json') + response2 = self.client.post('/api/report/', data2, format='json') + + self.assertEqual(response1.status_code, status.HTTP_202_ACCEPTED) + self.assertEqual(response2.status_code, status.HTTP_202_ACCEPTED) + + report1 = Report.objects.get(id=response1.data['report_id']) + report2 = Report.objects.get(id=response2.data['report_id']) + + self.assertEqual(report1.host, 'host1.example.com') + self.assertEqual(report2.host, 'host2.example.com') diff --git a/reports/tests/test_models.py b/reports/tests/test_models.py new file mode 100644 index 00000000..b42e780d --- /dev/null +++ b/reports/tests/test_models.py @@ -0,0 +1,476 @@ +# Copyright 2025 Marcus Furlong +# +# This file is part of Patchman. +# +# Patchman is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, version 3 only. +# +# Patchman is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Patchman. If not, see + +import json + +from django.test import TestCase, override_settings + +from reports.models import Report + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class ReportModelTests(TestCase): + """Tests for the Report model.""" + + def test_report_creation(self): + """Test creating a report.""" + report = Report.objects.create( + host='testhost.example.com', + domain='example.com', + kernel='5.15.0', + arch='x86_64', + os='Ubuntu 22.04', + protocol='2', + ) + self.assertEqual(report.host, 'testhost.example.com') + self.assertEqual(report.protocol, '2') + self.assertFalse(report.processed) + + def test_report_string_representation(self): + """Test Report __str__ method.""" + report = Report.objects.create( + host='testhost.example.com', + kernel='5.15.0', + arch='x86_64', + os='Ubuntu 22.04', + protocol='2', + ) + str_repr = str(report) + self.assertIn('testhost.example.com', str_repr) + + def test_report_get_absolute_url(self): + """Test Report.get_absolute_url().""" + report = Report.objects.create( + host='testhost.example.com', + kernel='5.15.0', + arch='x86_64', + os='Ubuntu 22.04', + protocol='2', + ) + url = report.get_absolute_url() + self.assertIn(str(report.id), url) + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class ReportParsedPropertiesTests(TestCase): + """Tests for Report _parsed properties.""" + + def test_packages_parsed_protocol2(self): + """Test packages_parsed returns parsed JSON for protocol 2.""" + packages = [ + {'name': 'nginx', 'version': '1.18.0', 'arch': 'amd64', 'type': 'deb'} + ] + report = Report.objects.create( + host='testhost.example.com', + kernel='5.15.0', + arch='x86_64', + os='Ubuntu 22.04', + protocol='2', + packages=json.dumps(packages), + ) + self.assertEqual(report.packages_parsed, packages) + self.assertEqual(len(report.packages_parsed), 1) + self.assertEqual(report.packages_parsed[0]['name'], 'nginx') + + def test_packages_parsed_empty(self): + """Test packages_parsed returns empty list when no packages.""" + report = Report.objects.create( + host='testhost.example.com', + kernel='5.15.0', + arch='x86_64', + os='Ubuntu 22.04', + protocol='2', + packages='', + ) + self.assertEqual(report.packages_parsed, []) + + def test_packages_parsed_invalid_json(self): + """Test packages_parsed returns empty list for invalid JSON.""" + report = Report.objects.create( + host='testhost.example.com', + kernel='5.15.0', + arch='x86_64', + os='Ubuntu 22.04', + protocol='2', + packages='invalid json {', + ) + self.assertEqual(report.packages_parsed, []) + + def test_packages_parsed_protocol1(self): + """Test packages_parsed returns empty list for protocol 1.""" + report = Report.objects.create( + host='testhost.example.com', + kernel='5.15.0', + arch='x86_64', + os='Ubuntu 22.04', + protocol='1', + packages='nginx 1.18.0 amd64', + ) + self.assertEqual(report.packages_parsed, []) + + def test_repos_parsed_protocol2(self): + """Test repos_parsed returns parsed JSON for protocol 2.""" + repos = [ + {'type': 'deb', 'name': 'ubuntu-main', 'id': 'main', 'urls': ['http://example.com']} + ] + report = Report.objects.create( + host='testhost.example.com', + kernel='5.15.0', + arch='x86_64', + os='Ubuntu 22.04', + protocol='2', + repos=json.dumps(repos), + ) + self.assertEqual(report.repos_parsed, repos) + + def test_modules_parsed_protocol2(self): + """Test modules_parsed returns parsed JSON for protocol 2.""" + modules = [ + {'name': 'nodejs', 'stream': '18', 'version': '123', 'context': 'rhel9'} + ] + report = Report.objects.create( + host='testhost.example.com', + kernel='5.15.0', + arch='x86_64', + os='Ubuntu 22.04', + protocol='2', + modules=json.dumps(modules), + ) + self.assertEqual(report.modules_parsed, modules) + + def test_sec_updates_parsed_protocol2(self): + """Test sec_updates_parsed returns parsed JSON for protocol 2.""" + updates = [ + {'name': 'openssl', 'version': '3.0.1', 'arch': 'amd64', 'repo': 'security'} + ] + report = Report.objects.create( + host='testhost.example.com', + kernel='5.15.0', + arch='x86_64', + os='Ubuntu 22.04', + protocol='2', + sec_updates=json.dumps(updates), + ) + self.assertEqual(report.sec_updates_parsed, updates) + + def test_bug_updates_parsed_protocol2(self): + """Test bug_updates_parsed returns parsed JSON for protocol 2.""" + updates = [ + {'name': 'curl', 'version': '7.81.0', 'arch': 'amd64', 'repo': 'main'} + ] + report = Report.objects.create( + host='testhost.example.com', + kernel='5.15.0', + arch='x86_64', + os='Ubuntu 22.04', + protocol='2', + bug_updates=json.dumps(updates), + ) + self.assertEqual(report.bug_updates_parsed, updates) + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class ReportHasPropertiesTests(TestCase): + """Tests for Report has_* properties.""" + + def test_has_packages_protocol2_with_data(self): + """Test has_packages returns True when packages exist (protocol 2).""" + packages = [{'name': 'nginx', 'version': '1.18.0', 'arch': 'amd64', 'type': 'deb'}] + report = Report.objects.create( + host='testhost.example.com', + kernel='5.15.0', + arch='x86_64', + os='Ubuntu 22.04', + protocol='2', + packages=json.dumps(packages), + ) + self.assertTrue(report.has_packages) + + def test_has_packages_protocol2_empty(self): + """Test has_packages returns False when no packages (protocol 2).""" + report = Report.objects.create( + host='testhost.example.com', + kernel='5.15.0', + arch='x86_64', + os='Ubuntu 22.04', + protocol='2', + packages=json.dumps([]), + ) + self.assertFalse(report.has_packages) + + def test_has_packages_protocol1_with_data(self): + """Test has_packages returns True when packages exist (protocol 1).""" + report = Report.objects.create( + host='testhost.example.com', + kernel='5.15.0', + arch='x86_64', + os='Ubuntu 22.04', + protocol='1', + packages='nginx 1.18.0 amd64\ncurl 7.81.0 amd64', + ) + self.assertTrue(report.has_packages) + + def test_has_packages_protocol1_empty(self): + """Test has_packages returns False when no packages (protocol 1).""" + report = Report.objects.create( + host='testhost.example.com', + kernel='5.15.0', + arch='x86_64', + os='Ubuntu 22.04', + protocol='1', + packages='', + ) + self.assertFalse(report.has_packages) + + def test_has_packages_protocol1_whitespace(self): + """Test has_packages returns False for whitespace-only (protocol 1).""" + report = Report.objects.create( + host='testhost.example.com', + kernel='5.15.0', + arch='x86_64', + os='Ubuntu 22.04', + protocol='1', + packages=' \n\n ', + ) + self.assertFalse(report.has_packages) + + def test_has_repos_protocol2(self): + """Test has_repos property for protocol 2.""" + repos = [{'type': 'deb', 'name': 'main', 'urls': ['http://example.com']}] + report = Report.objects.create( + host='testhost.example.com', + kernel='5.15.0', + arch='x86_64', + os='Ubuntu 22.04', + protocol='2', + repos=json.dumps(repos), + ) + self.assertTrue(report.has_repos) + + def test_has_modules_protocol2(self): + """Test has_modules property for protocol 2.""" + modules = [{'name': 'nodejs', 'stream': '18'}] + report = Report.objects.create( + host='testhost.example.com', + kernel='5.15.0', + arch='x86_64', + os='Ubuntu 22.04', + protocol='2', + modules=json.dumps(modules), + ) + self.assertTrue(report.has_modules) + + def test_has_sec_updates_protocol2(self): + """Test has_sec_updates property for protocol 2.""" + updates = [{'name': 'openssl', 'version': '3.0.1'}] + report = Report.objects.create( + host='testhost.example.com', + kernel='5.15.0', + arch='x86_64', + os='Ubuntu 22.04', + protocol='2', + sec_updates=json.dumps(updates), + ) + self.assertTrue(report.has_sec_updates) + + def test_has_bug_updates_protocol2(self): + """Test has_bug_updates property for protocol 2.""" + updates = [{'name': 'curl', 'version': '7.81.0'}] + report = Report.objects.create( + host='testhost.example.com', + kernel='5.15.0', + arch='x86_64', + os='Ubuntu 22.04', + protocol='2', + bug_updates=json.dumps(updates), + ) + self.assertTrue(report.has_bug_updates) + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class ReportParseMethodTests(TestCase): + """Tests for Report.parse() method.""" + + def test_parse_basic_data(self): + """Test parse() sets basic report attributes.""" + report = Report() + data = { + 'host': 'TESTHOST.EXAMPLE.COM', + 'arch': 'x86_64', + 'kernel': '5.15.0', + 'os': 'Ubuntu 22.04', + 'protocol': '2', + } + meta = { + 'REMOTE_ADDR': '192.168.1.100', + 'HTTP_USER_AGENT': 'patchman-client/1.0', + } + report.parse(data, meta) + + self.assertEqual(report.host, 'testhost.example.com') # Lowercased + self.assertEqual(report.domain, 'example.com') # Extracted + self.assertEqual(report.arch, 'x86_64') + self.assertEqual(report.kernel, '5.15.0') + self.assertEqual(report.os, 'Ubuntu 22.04') + self.assertEqual(report.protocol, '2') + self.assertEqual(report.report_ip, '192.168.1.100') + self.assertEqual(report.useragent, 'patchman-client/1.0') + + def test_parse_extracts_domain_from_fqdn(self): + """Test parse() extracts domain from FQDN.""" + report = Report() + data = { + 'host': 'server1.prod.example.com', + 'arch': 'x86_64', + 'kernel': '5.15.0', + 'os': 'Ubuntu 22.04', + 'protocol': '2', + } + meta = { + 'REMOTE_ADDR': '192.168.1.100', + 'HTTP_USER_AGENT': 'patchman-client/1.0', + } + report.parse(data, meta) + + self.assertEqual(report.host, 'server1.prod.example.com') + self.assertEqual(report.domain, 'prod.example.com') + + def test_parse_hostname_without_domain(self): + """Test parse() handles hostname without domain.""" + report = Report() + data = { + 'host': 'localhost', + 'arch': 'x86_64', + 'kernel': '5.15.0', + 'os': 'Ubuntu 22.04', + 'protocol': '2', + } + meta = { + 'REMOTE_ADDR': '127.0.0.1', + 'HTTP_USER_AGENT': 'patchman-client/1.0', + } + report.parse(data, meta) + + self.assertEqual(report.host, 'localhost') + self.assertIsNone(report.domain) + + def test_parse_x_forwarded_for(self): + """Test parse() uses X-Forwarded-For header.""" + report = Report() + data = { + 'host': 'testhost.example.com', + 'arch': 'x86_64', + 'kernel': '5.15.0', + 'os': 'Ubuntu 22.04', + 'protocol': '2', + } + meta = { + 'REMOTE_ADDR': '10.0.0.1', + 'HTTP_X_FORWARDED_FOR': '203.0.113.50, 10.0.0.1', + 'HTTP_USER_AGENT': 'patchman-client/1.0', + } + report.parse(data, meta) + + self.assertEqual(report.report_ip, '203.0.113.50') + + def test_parse_x_real_ip(self): + """Test parse() uses X-Real-IP header.""" + report = Report() + data = { + 'host': 'testhost.example.com', + 'arch': 'x86_64', + 'kernel': '5.15.0', + 'os': 'Ubuntu 22.04', + 'protocol': '2', + } + meta = { + 'REMOTE_ADDR': '10.0.0.1', + 'HTTP_X_REAL_IP': '203.0.113.100', + 'HTTP_USER_AGENT': 'patchman-client/1.0', + } + report.parse(data, meta) + + self.assertEqual(report.report_ip, '203.0.113.100') + + def test_parse_with_packages(self): + """Test parse() stores packages data.""" + report = Report() + packages_data = 'nginx 1.18.0 amd64\ncurl 7.81.0 amd64' + data = { + 'host': 'testhost.example.com', + 'arch': 'x86_64', + 'kernel': '5.15.0', + 'os': 'Ubuntu 22.04', + 'protocol': '1', + 'packages': packages_data, + } + meta = { + 'REMOTE_ADDR': '192.168.1.100', + 'HTTP_USER_AGENT': 'patchman-client/1.0', + } + report.parse(data, meta) + + self.assertEqual(report.packages, packages_data) + + def test_parse_with_tags(self): + """Test parse() stores tags.""" + report = Report() + data = { + 'host': 'testhost.example.com', + 'arch': 'x86_64', + 'kernel': '5.15.0', + 'os': 'Ubuntu 22.04', + 'protocol': '2', + 'tags': 'web,production', + } + meta = { + 'REMOTE_ADDR': '192.168.1.100', + 'HTTP_USER_AGENT': 'patchman-client/1.0', + } + report.parse(data, meta) + + self.assertEqual(report.tags, 'web,production') + + def test_parse_with_reboot(self): + """Test parse() stores reboot status.""" + report = Report() + data = { + 'host': 'testhost.example.com', + 'arch': 'x86_64', + 'kernel': '5.15.0', + 'os': 'Ubuntu 22.04', + 'protocol': '2', + 'reboot': 'True', + } + meta = { + 'REMOTE_ADDR': '192.168.1.100', + 'HTTP_USER_AGENT': 'patchman-client/1.0', + } + report.parse(data, meta) + + self.assertEqual(report.reboot, 'True') diff --git a/reports/tests/test_parsing.py b/reports/tests/test_parsing.py new file mode 100644 index 00000000..9d50d2f3 --- /dev/null +++ b/reports/tests/test_parsing.py @@ -0,0 +1,180 @@ +# Copyright 2025 Marcus Furlong +# +# This file is part of Patchman. +# +# Patchman is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, version 3 only. +# +# Patchman is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Patchman. If not, see + +from django.test import TestCase, override_settings + +from packages.models import Package +from reports.utils import ( + _get_package_type, _get_repo_type, parse_packages, parse_repos, +) +from repos.models import Repository + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class ParsePackagesTests(TestCase): + """Tests for parse_packages() function - Protocol 1 text parsing.""" + + def test_parse_packages_single_package(self): + """Test parsing a single package string.""" + pkg_str = "'nginx' '' '1.18.0' '6ubuntu14' 'amd64' 'deb'" + packages = parse_packages(pkg_str) + self.assertEqual(len(packages), 1) + self.assertEqual(packages[0][0], 'nginx') + + def test_parse_packages_multiple_packages(self): + """Test parsing multiple package strings.""" + pkg_str = """'nginx' '' '1.18.0' '6ubuntu14' 'amd64' 'deb' +'curl' '' '7.81.0' '1' 'amd64' 'deb' +'vim' '' '8.2.0' '1' 'amd64' 'deb'""" + packages = parse_packages(pkg_str) + self.assertEqual(len(packages), 3) + self.assertEqual(packages[0][0], 'nginx') + self.assertEqual(packages[1][0], 'curl') + self.assertEqual(packages[2][0], 'vim') + + def test_parse_packages_with_epoch(self): + """Test parsing package with epoch.""" + pkg_str = "'vim' '2' '8.2.0' '1' 'amd64' 'deb'" + packages = parse_packages(pkg_str) + self.assertEqual(packages[0][1], '2') + + def test_parse_packages_rpm_format(self): + """Test parsing RPM package format.""" + pkg_str = "'httpd' '0' '2.4.57' '5.el9' 'x86_64' 'rpm'" + packages = parse_packages(pkg_str) + self.assertEqual(packages[0][0], 'httpd') + self.assertEqual(packages[0][5], 'rpm') + + def test_parse_packages_empty_string(self): + """Test parsing empty string returns empty list.""" + packages = parse_packages('') + # Empty string results in empty list + self.assertEqual(len(packages), 0) + + def test_parse_packages_strips_quotes(self): + """Test that quotes are removed from parsed values.""" + pkg_str = "'nginx' '' '1.18.0' '6ubuntu14' 'amd64' 'deb'" + packages = parse_packages(pkg_str) + # Quotes should be stripped + self.assertNotIn("'", packages[0][0]) + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class ParseReposTests(TestCase): + """Tests for parse_repos() function - Protocol 1 text parsing.""" + + def test_parse_repos_single_repo(self): + """Test parsing a single repo string.""" + repo_str = "'deb' 'Ubuntu Main' 'ubuntu-main' '500' 'http://archive.ubuntu.com/ubuntu'" + repos = parse_repos(repo_str) + self.assertEqual(len(repos), 1) + self.assertEqual(repos[0][0], 'deb') + + def test_parse_repos_multiple_repos(self): + """Test parsing multiple repo strings.""" + repo_str = """'deb' 'Ubuntu Main' 'ubuntu-main' '500' 'http://archive.ubuntu.com/ubuntu' +'deb' 'Ubuntu Security' 'ubuntu-security' '500' 'http://security.ubuntu.com/ubuntu'""" + repos = parse_repos(repo_str) + self.assertEqual(len(repos), 2) + + def test_parse_repos_rpm_format(self): + """Test parsing RPM repo format.""" + repo_str = "'rpm' 'Rocky BaseOS' 'baseos' '99' 'http://mirror.rockylinux.org/rocky/9/BaseOS/x86_64/os'" + repos = parse_repos(repo_str) + self.assertEqual(repos[0][0], 'rpm') + self.assertEqual(repos[0][1], 'Rocky BaseOS') + + def test_parse_repos_empty_string(self): + """Test parsing empty string returns empty list.""" + repos = parse_repos('') + self.assertEqual(len(repos), 0) + + def test_parse_repos_strips_quotes(self): + """Test that quotes are removed from parsed values.""" + repo_str = "'deb' 'Ubuntu Main' 'ubuntu-main' '500' 'http://archive.ubuntu.com/ubuntu'" + repos = parse_repos(repo_str) + self.assertNotIn("'", repos[0][0]) + self.assertNotIn("'", repos[0][1]) + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class GetPackageTypeTests(TestCase): + """Tests for _get_package_type() function.""" + + def test_get_package_type_deb(self): + """Test DEB package type detection.""" + self.assertEqual(_get_package_type('deb'), Package.DEB) + self.assertEqual(_get_package_type('DEB'), Package.DEB) + self.assertEqual(_get_package_type('Deb'), Package.DEB) + + def test_get_package_type_rpm(self): + """Test RPM package type detection.""" + self.assertEqual(_get_package_type('rpm'), Package.RPM) + self.assertEqual(_get_package_type('RPM'), Package.RPM) + + def test_get_package_type_arch(self): + """Test Arch package type detection.""" + self.assertEqual(_get_package_type('arch'), Package.ARCH) + + def test_get_package_type_gentoo(self): + """Test Gentoo package type detection.""" + self.assertEqual(_get_package_type('gentoo'), Package.GENTOO) + + def test_get_package_type_unknown(self): + """Test unknown package type returns UNKNOWN.""" + self.assertEqual(_get_package_type(''), Package.UNKNOWN) + self.assertEqual(_get_package_type('invalid'), Package.UNKNOWN) + self.assertEqual(_get_package_type(None), Package.UNKNOWN) + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class GetRepoTypeTests(TestCase): + """Tests for _get_repo_type() function.""" + + def test_get_repo_type_deb(self): + """Test DEB repo type detection.""" + self.assertEqual(_get_repo_type('deb'), Repository.DEB) + self.assertEqual(_get_repo_type('DEB'), Repository.DEB) + + def test_get_repo_type_rpm(self): + """Test RPM repo type detection.""" + self.assertEqual(_get_repo_type('rpm'), Repository.RPM) + self.assertEqual(_get_repo_type('RPM'), Repository.RPM) + + def test_get_repo_type_arch(self): + """Test Arch repo type detection.""" + self.assertEqual(_get_repo_type('arch'), Repository.ARCH) + + def test_get_repo_type_gentoo(self): + """Test Gentoo repo type detection.""" + self.assertEqual(_get_repo_type('gentoo'), Repository.GENTOO) + + def test_get_repo_type_unknown(self): + """Test unknown repo type returns None.""" + self.assertIsNone(_get_repo_type('')) + self.assertIsNone(_get_repo_type('invalid')) diff --git a/reports/tests/test_serializers.py b/reports/tests/test_serializers.py new file mode 100644 index 00000000..693c89a8 --- /dev/null +++ b/reports/tests/test_serializers.py @@ -0,0 +1,290 @@ +# Copyright 2025 Marcus Furlong +# +# This file is part of Patchman. +# +# Patchman is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, version 3 only. +# +# Patchman is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Patchman. If not, see + +from django.test import TestCase, override_settings + +from reports.serializers import ( + PackageSerializer, ReportUploadSerializer, RepoSerializer, + UpdateSerializer, +) + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class ReportUploadSerializerValidationTests(TestCase): + """Tests for ReportUploadSerializer validation.""" + + def test_valid_minimal_report(self): + """Test valid report with minimal required fields.""" + data = { + 'hostname': 'test.example.com', + 'arch': 'x86_64', + 'os': 'Ubuntu 22.04', + 'kernel': '5.15.0-91-generic', + } + serializer = ReportUploadSerializer(data=data) + self.assertTrue(serializer.is_valid(), serializer.errors) + + def test_valid_full_report(self): + """Test valid report with all fields.""" + data = { + 'hostname': 'test.example.com', + 'arch': 'x86_64', + 'os': 'Ubuntu 22.04', + 'kernel': '5.15.0-91-generic', + 'protocol': 2, + 'tags': ['web', 'production'], + 'reboot_required': False, + 'packages': [ + {'name': 'nginx', 'version': '1.18.0', 'arch': 'amd64', 'type': 'deb'}, + ], + 'repos': [], + 'modules': [], + 'sec_updates': [], + 'bug_updates': [], + } + serializer = ReportUploadSerializer(data=data) + self.assertTrue(serializer.is_valid(), serializer.errors) + + def test_missing_hostname(self): + """Test validation fails without hostname.""" + data = { + 'arch': 'x86_64', + 'os': 'Ubuntu 22.04', + 'kernel': '5.15.0-91-generic', + } + serializer = ReportUploadSerializer(data=data) + self.assertFalse(serializer.is_valid()) + self.assertIn('hostname', serializer.errors) + + def test_missing_arch(self): + """Test validation fails without arch.""" + data = { + 'hostname': 'test.example.com', + 'os': 'Ubuntu 22.04', + 'kernel': '5.15.0-91-generic', + } + serializer = ReportUploadSerializer(data=data) + self.assertFalse(serializer.is_valid()) + self.assertIn('arch', serializer.errors) + + def test_missing_os(self): + """Test validation fails without os.""" + data = { + 'hostname': 'test.example.com', + 'arch': 'x86_64', + 'kernel': '5.15.0-91-generic', + } + serializer = ReportUploadSerializer(data=data) + self.assertFalse(serializer.is_valid()) + self.assertIn('os', serializer.errors) + + def test_missing_kernel(self): + """Test validation fails without kernel.""" + data = { + 'hostname': 'test.example.com', + 'arch': 'x86_64', + 'os': 'Ubuntu 22.04', + } + serializer = ReportUploadSerializer(data=data) + self.assertFalse(serializer.is_valid()) + self.assertIn('kernel', serializer.errors) + + def test_empty_hostname(self): + """Test validation fails with empty hostname.""" + data = { + 'hostname': '', + 'arch': 'x86_64', + 'os': 'Ubuntu 22.04', + 'kernel': '5.15.0-91-generic', + } + serializer = ReportUploadSerializer(data=data) + self.assertFalse(serializer.is_valid()) + self.assertIn('hostname', serializer.errors) + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class PackageSerializerValidationTests(TestCase): + """Tests for PackageSerializer validation.""" + + def test_valid_package(self): + """Test valid package data.""" + data = { + 'name': 'nginx', + 'version': '1.18.0', + 'arch': 'amd64', + 'type': 'deb', + } + serializer = PackageSerializer(data=data) + self.assertTrue(serializer.is_valid(), serializer.errors) + + def test_valid_package_with_optional_fields(self): + """Test valid package with all optional fields.""" + data = { + 'name': 'nginx', + 'epoch': '1', + 'version': '1.18.0', + 'release': '6ubuntu14', + 'arch': 'amd64', + 'type': 'deb', + } + serializer = PackageSerializer(data=data) + self.assertTrue(serializer.is_valid(), serializer.errors) + + def test_missing_name(self): + """Test validation fails without name.""" + data = { + 'version': '1.18.0', + 'arch': 'amd64', + } + serializer = PackageSerializer(data=data) + self.assertFalse(serializer.is_valid()) + self.assertIn('name', serializer.errors) + + def test_missing_version(self): + """Test validation fails without version.""" + data = { + 'name': 'nginx', + 'arch': 'amd64', + } + serializer = PackageSerializer(data=data) + self.assertFalse(serializer.is_valid()) + self.assertIn('version', serializer.errors) + + def test_missing_arch(self): + """Test validation fails without arch.""" + data = { + 'name': 'nginx', + 'version': '1.18.0', + } + serializer = PackageSerializer(data=data) + self.assertFalse(serializer.is_valid()) + self.assertIn('arch', serializer.errors) + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class RepoSerializerValidationTests(TestCase): + """Tests for RepoSerializer validation.""" + + def test_valid_repo(self): + """Test valid repo data.""" + data = { + 'type': 'deb', + 'name': 'Ubuntu Main', + 'urls': ['http://archive.ubuntu.com/ubuntu'], + } + serializer = RepoSerializer(data=data) + self.assertTrue(serializer.is_valid(), serializer.errors) + + def test_valid_repo_with_optional_fields(self): + """Test valid repo with all optional fields.""" + data = { + 'type': 'deb', + 'name': 'Ubuntu Main', + 'id': 'ubuntu-main', + 'priority': 500, + 'urls': ['http://archive.ubuntu.com/ubuntu'], + } + serializer = RepoSerializer(data=data) + self.assertTrue(serializer.is_valid(), serializer.errors) + + def test_missing_type(self): + """Test validation fails without type.""" + data = { + 'name': 'Ubuntu Main', + 'urls': ['http://archive.ubuntu.com/ubuntu'], + } + serializer = RepoSerializer(data=data) + self.assertFalse(serializer.is_valid()) + self.assertIn('type', serializer.errors) + + def test_missing_urls(self): + """Test validation passes without urls (urls is optional).""" + data = { + 'type': 'deb', + 'name': 'Ubuntu Main', + } + serializer = RepoSerializer(data=data) + self.assertTrue(serializer.is_valid()) + # urls defaults to empty list + self.assertEqual(serializer.validated_data['urls'], []) + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class UpdateSerializerValidationTests(TestCase): + """Tests for UpdateSerializer validation.""" + + def test_valid_update(self): + """Test valid update data.""" + data = { + 'name': 'openssl', + 'version': '3.0.1-1', + 'arch': 'amd64', + } + serializer = UpdateSerializer(data=data) + self.assertTrue(serializer.is_valid(), serializer.errors) + + def test_valid_update_with_repo(self): + """Test valid update with repo field.""" + data = { + 'name': 'openssl', + 'version': '3.0.1-1', + 'arch': 'amd64', + 'repo': 'ubuntu-security', + } + serializer = UpdateSerializer(data=data) + self.assertTrue(serializer.is_valid(), serializer.errors) + + def test_missing_name(self): + """Test validation fails without name.""" + data = { + 'version': '3.0.1-1', + 'arch': 'amd64', + } + serializer = UpdateSerializer(data=data) + self.assertFalse(serializer.is_valid()) + self.assertIn('name', serializer.errors) + + def test_missing_version(self): + """Test validation fails without version.""" + data = { + 'name': 'openssl', + 'arch': 'amd64', + } + serializer = UpdateSerializer(data=data) + self.assertFalse(serializer.is_valid()) + self.assertIn('version', serializer.errors) + + def test_missing_arch(self): + """Test validation fails without arch.""" + data = { + 'name': 'openssl', + 'version': '3.0.1-1', + } + serializer = UpdateSerializer(data=data) + self.assertFalse(serializer.is_valid()) + self.assertIn('arch', serializer.errors) diff --git a/reports/tests/test_tasks.py b/reports/tests/test_tasks.py new file mode 100644 index 00000000..8f691f7f --- /dev/null +++ b/reports/tests/test_tasks.py @@ -0,0 +1,271 @@ +# Copyright 2025 Marcus Furlong +# +# This file is part of Patchman. +# +# Patchman is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, version 3 only. +# +# Patchman is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Patchman. If not, see + +import json + +from django.test import TestCase, override_settings +from django.utils import timezone + +from arch.models import MachineArchitecture +from domains.models import Domain +from hosts.models import Host +from operatingsystems.models import OSRelease, OSVariant +from reports.models import Report +from reports.tasks import ( + process_report, process_reports, remove_reports_with_no_hosts, +) + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class ProcessReportTaskTests(TestCase): + """Tests for process_report Celery task.""" + + def test_process_report_task_processes_report(self): + """Test process_report task processes a report.""" + report = Report.objects.create( + host='taskhost.example.com', + domain='example.com', + report_ip='192.168.1.100', + os='Ubuntu 22.04.3 LTS', + kernel='5.15.0-91-generic', + arch='x86_64', + protocol='2', + packages=json.dumps([]), + repos=json.dumps([]), + modules=json.dumps([]), + sec_updates=json.dumps([]), + bug_updates=json.dumps([]), + ) + self.assertFalse(report.processed) + + # Run the task directly (CELERY_TASK_ALWAYS_EAGER=True) + process_report(report.id) + + report.refresh_from_db() + self.assertTrue(report.processed) + + def test_process_report_task_creates_host(self): + """Test process_report task creates Host.""" + report = Report.objects.create( + host='newtaskhost.example.com', + domain='example.com', + report_ip='192.168.1.101', + os='Ubuntu 22.04.3 LTS', + kernel='5.15.0-91-generic', + arch='x86_64', + protocol='2', + packages=json.dumps([]), + repos=json.dumps([]), + modules=json.dumps([]), + sec_updates=json.dumps([]), + bug_updates=json.dumps([]), + ) + + process_report(report.id) + + host = Host.objects.get(hostname='newtaskhost.example.com') + self.assertIsNotNone(host) + + def test_process_report_task_skips_already_processed(self): + """Test process_report skips already processed reports.""" + report = Report.objects.create( + host='processedhost.example.com', + domain='example.com', + report_ip='192.168.1.102', + os='Ubuntu 22.04.3 LTS', + kernel='5.15.0-91-generic', + arch='x86_64', + protocol='2', + processed=True, + packages=json.dumps([]), + repos=json.dumps([]), + modules=json.dumps([]), + sec_updates=json.dumps([]), + bug_updates=json.dumps([]), + ) + + # Should not raise error + process_report(report.id) + report.refresh_from_db() + self.assertTrue(report.processed) + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class ProcessReportsTaskTests(TestCase): + """Tests for process_reports Celery task.""" + + def test_process_reports_processes_unprocessed(self): + """Test process_reports processes all unprocessed reports.""" + # Create multiple unprocessed reports + for i in range(3): + Report.objects.create( + host=f'multihost{i}.example.com', + domain='example.com', + report_ip=f'192.168.1.{10 + i}', + os='Ubuntu 22.04.3 LTS', + kernel='5.15.0-91-generic', + arch='x86_64', + protocol='2', + packages=json.dumps([]), + repos=json.dumps([]), + modules=json.dumps([]), + sec_updates=json.dumps([]), + bug_updates=json.dumps([]), + ) + + unprocessed_count = Report.objects.filter(processed=False).count() + self.assertEqual(unprocessed_count, 3) + + # Run the task + process_reports() + + # All should be processed now + processed_count = Report.objects.filter(processed=True).count() + self.assertEqual(processed_count, 3) + + def test_process_reports_ignores_processed(self): + """Test process_reports ignores already processed reports.""" + Report.objects.create( + host='alreadyprocessed.example.com', + domain='example.com', + report_ip='192.168.1.50', + os='Ubuntu 22.04.3 LTS', + kernel='5.15.0-91-generic', + arch='x86_64', + protocol='2', + processed=True, + packages=json.dumps([]), + repos=json.dumps([]), + modules=json.dumps([]), + sec_updates=json.dumps([]), + bug_updates=json.dumps([]), + ) + + # Should not raise error, just skip + process_reports() + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class RemoveReportsWithNoHostsTaskTests(TestCase): + """Tests for remove_reports_with_no_hosts Celery task.""" + + def test_removes_orphan_reports(self): + """Test task removes processed reports for non-existent hosts.""" + # Create a processed report for a host that doesn't exist + Report.objects.create( + host='nonexistent.example.com', + domain='example.com', + report_ip='192.168.1.99', + os='Ubuntu 22.04.3 LTS', + kernel='5.15.0-91-generic', + arch='x86_64', + protocol='2', + processed=True, + packages=json.dumps([]), + repos=json.dumps([]), + modules=json.dumps([]), + sec_updates=json.dumps([]), + bug_updates=json.dumps([]), + ) + + self.assertEqual(Report.objects.count(), 1) + + # Run the cleanup task + remove_reports_with_no_hosts() + + # Report should be deleted + self.assertEqual(Report.objects.count(), 0) + + def test_keeps_reports_with_existing_hosts(self): + """Test task keeps reports for existing hosts.""" + # Create a host first + arch = MachineArchitecture.objects.create(name='x86_64') + osrelease = OSRelease.objects.create(name='Ubuntu 22.04') + osvariant = OSVariant.objects.create( + name='Ubuntu 22.04.3 LTS x86_64', + osrelease=osrelease, + arch=arch, + ) + domain = Domain.objects.create(name='example.com') + Host.objects.create( + hostname='existinghost.example.com', + ipaddress='192.168.1.200', + arch=arch, + osvariant=osvariant, + domain=domain, + lastreport=timezone.now(), + ) + + # Create a processed report for this host + Report.objects.create( + host='existinghost.example.com', + domain='example.com', + report_ip='192.168.1.200', + os='Ubuntu 22.04.3 LTS', + kernel='5.15.0-91-generic', + arch='x86_64', + protocol='2', + processed=True, + packages=json.dumps([]), + repos=json.dumps([]), + modules=json.dumps([]), + sec_updates=json.dumps([]), + bug_updates=json.dumps([]), + ) + + self.assertEqual(Report.objects.count(), 1) + + # Run the cleanup task + remove_reports_with_no_hosts() + + # Report should still exist + self.assertEqual(Report.objects.count(), 1) + + def test_keeps_unprocessed_reports(self): + """Test task keeps unprocessed reports even if host doesn't exist.""" + # Create an unprocessed report for a non-existent host + Report.objects.create( + host='pendinghost.example.com', + domain='example.com', + report_ip='192.168.1.98', + os='Ubuntu 22.04.3 LTS', + kernel='5.15.0-91-generic', + arch='x86_64', + protocol='2', + processed=False, # Not processed yet + packages=json.dumps([]), + repos=json.dumps([]), + modules=json.dumps([]), + sec_updates=json.dumps([]), + bug_updates=json.dumps([]), + ) + + self.assertEqual(Report.objects.count(), 1) + + # Run the cleanup task + remove_reports_with_no_hosts() + + # Report should still exist (not processed) + self.assertEqual(Report.objects.count(), 1) diff --git a/reports/tests/test_utils.py b/reports/tests/test_utils.py new file mode 100644 index 00000000..f640d642 --- /dev/null +++ b/reports/tests/test_utils.py @@ -0,0 +1,539 @@ +# Copyright 2025 Marcus Furlong +# +# This file is part of Patchman. +# +# Patchman is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, version 3 only. +# +# Patchman is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Patchman. If not, see + +import json + +from django.test import TestCase, override_settings +from django.utils import timezone + +from arch.models import MachineArchitecture, PackageArchitecture +from domains.models import Domain +from hosts.models import Host, HostRepo +from operatingsystems.models import OSRelease, OSVariant +from packages.models import Package, PackageName +from reports.models import Report +from reports.utils import ( + process_package, process_package_json, process_package_text, process_repo, + process_repo_json, process_update, +) +from repos.models import Mirror, Repository + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class ProcessPackageTests(TestCase): + """Tests for package processing functions.""" + + def test_process_package_creates_package(self): + """Test process_package creates a Package object.""" + package = process_package( + name='nginx', + epoch='', + version='1.18.0', + release='6ubuntu14', + arch='amd64', + p_type=Package.DEB, + ) + self.assertIsNotNone(package) + self.assertEqual(package.name.name, 'nginx') + self.assertEqual(package.version, '1.18.0') + self.assertEqual(package.release, '6ubuntu14') + self.assertEqual(package.arch.name, 'amd64') + + def test_process_package_creates_package_name(self): + """Test process_package creates PackageName if not exists.""" + self.assertEqual(PackageName.objects.filter(name='curl').count(), 0) + process_package('curl', '', '7.81.0', '1', 'amd64', Package.DEB) + self.assertEqual(PackageName.objects.filter(name='curl').count(), 1) + + def test_process_package_creates_package_arch(self): + """Test process_package creates PackageArchitecture if not exists.""" + self.assertEqual(PackageArchitecture.objects.filter(name='arm64').count(), 0) + process_package('nginx', '', '1.18.0', '1', 'arm64', Package.DEB) + self.assertEqual(PackageArchitecture.objects.filter(name='arm64').count(), 1) + + def test_process_package_reuses_existing(self): + """Test process_package reuses existing Package.""" + pkg1 = process_package('nginx', '', '1.18.0', '1', 'amd64', Package.DEB) + pkg2 = process_package('nginx', '', '1.18.0', '1', 'amd64', Package.DEB) + self.assertEqual(pkg1.id, pkg2.id) + + def test_process_package_rpm(self): + """Test process_package with RPM type.""" + package = process_package( + name='httpd', + epoch='0', + version='2.4.57', + release='5.el9', + arch='x86_64', + p_type=Package.RPM, + ) + self.assertEqual(package.packagetype, Package.RPM) + + def test_process_package_with_epoch(self): + """Test process_package with epoch.""" + package = process_package( + name='vim', + epoch='2', + version='8.2.0', + release='1', + arch='amd64', + p_type=Package.DEB, + ) + self.assertEqual(package.epoch, '2') + + def test_process_package_text_tuple(self): + """Test process_package_text with tuple input.""" + pkg_tuple = ('nginx', '', '1.18.0', '6ubuntu14', 'amd64', 'deb') + package = process_package_text(pkg_tuple) + self.assertIsNotNone(package) + self.assertEqual(package.name.name, 'nginx') + + def test_process_package_text_with_epoch(self): + """Test process_package_text with epoch.""" + pkg_tuple = ('vim', '2', '8.2.0', '1', 'amd64', 'deb') + package = process_package_text(pkg_tuple) + self.assertEqual(package.epoch, '2') + + def test_process_package_json_dict(self): + """Test process_package_json with dict input.""" + pkg_dict = { + 'name': 'nginx', + 'epoch': '', + 'version': '1.18.0', + 'release': '6ubuntu14', + 'arch': 'amd64', + 'type': 'deb', + } + package = process_package_json(pkg_dict) + self.assertIsNotNone(package) + self.assertEqual(package.name.name, 'nginx') + + def test_process_package_json_minimal(self): + """Test process_package_json with minimal fields.""" + pkg_dict = { + 'name': 'curl', + 'version': '7.81.0', + } + package = process_package_json(pkg_dict) + self.assertIsNotNone(package) + self.assertEqual(package.name.name, 'curl') + self.assertEqual(package.arch.name, 'unknown') + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class ProcessRepoTests(TestCase): + """Tests for repository processing functions.""" + + def test_process_repo_creates_repository(self): + """Test process_repo creates Repository and Mirror.""" + repo, priority = process_repo( + r_type=Repository.DEB, + r_name='Ubuntu Main', + r_id='ubuntu-main', + r_priority=500, + urls=['http://archive.ubuntu.com/ubuntu/dists/jammy/main/binary-amd64'], + arch='x86_64', + ) + self.assertIsNotNone(repo) + self.assertEqual(repo.repotype, Repository.DEB) + + def test_process_repo_creates_mirror(self): + """Test process_repo creates Mirror for URL.""" + repo, priority = process_repo( + r_type=Repository.DEB, + r_name='Ubuntu Main', + r_id='ubuntu-main', + r_priority=500, + urls=['http://archive.ubuntu.com/ubuntu'], + arch='x86_64', + ) + mirrors = Mirror.objects.filter(repo=repo) + self.assertEqual(mirrors.count(), 1) + self.assertIn('archive.ubuntu.com', mirrors.first().url) + + def test_process_repo_multiple_urls(self): + """Test process_repo with multiple URLs creates multiple mirrors.""" + repo, priority = process_repo( + r_type=Repository.DEB, + r_name='Ubuntu Main', + r_id='ubuntu-main', + r_priority=500, + urls=[ + 'http://archive.ubuntu.com/ubuntu', + 'http://mirror.example.com/ubuntu', + ], + arch='x86_64', + ) + mirrors = Mirror.objects.filter(repo=repo) + self.assertEqual(mirrors.count(), 2) + + def test_process_repo_creates_machine_arch(self): + """Test process_repo creates MachineArchitecture if not exists.""" + self.assertEqual(MachineArchitecture.objects.filter(name='aarch64').count(), 0) + process_repo( + r_type=Repository.DEB, + r_name='Test Repo', + r_id='test-repo', + r_priority=500, + urls=['http://example.com/repo'], + arch='aarch64', + ) + self.assertEqual(MachineArchitecture.objects.filter(name='aarch64').count(), 1) + + def test_process_repo_json(self): + """Test process_repo_json with dict input.""" + repo_dict = { + 'type': 'deb', + 'name': 'Ubuntu Main', + 'id': 'ubuntu-main', + 'priority': 500, + 'urls': ['http://archive.ubuntu.com/ubuntu'], + } + repo, priority = process_repo_json(repo_dict, 'x86_64') + self.assertIsNotNone(repo) + self.assertEqual(repo.repotype, Repository.DEB) + + def test_process_repo_json_rpm(self): + """Test process_repo_json with RPM repo.""" + repo_dict = { + 'type': 'rpm', + 'name': 'Rocky BaseOS', + 'id': 'baseos', + 'priority': 99, + 'urls': ['http://mirror.rockylinux.org/rocky/9/BaseOS/x86_64/os'], + } + repo, priority = process_repo_json(repo_dict, 'x86_64') + self.assertIsNotNone(repo) + self.assertEqual(repo.repotype, Repository.RPM) + # RPM priority is negated + self.assertEqual(priority, -99) + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class ProcessUpdateTests(TestCase): + """Tests for update processing functions.""" + + def setUp(self): + """Set up test data.""" + self.arch = MachineArchitecture.objects.create(name='x86_64') + self.pkg_arch = PackageArchitecture.objects.create(name='x86_64') + self.osrelease = OSRelease.objects.create(name='Rocky Linux 9') + self.osvariant = OSVariant.objects.create( + name='Rocky Linux 9 x86_64', + osrelease=self.osrelease, + arch=self.arch, + ) + self.domain = Domain.objects.create(name='example.com') + self.host = Host.objects.create( + hostname='test.example.com', + ipaddress='192.168.1.100', + arch=self.arch, + osvariant=self.osvariant, + domain=self.domain, + lastreport=timezone.now(), + ) + # Create an old RPM package on the host (process_update uses RPM type) + self.pkg_name = PackageName.objects.create(name='openssl') + self.old_pkg = Package.objects.create( + name=self.pkg_name, + arch=self.pkg_arch, + epoch='', + version='3.0.0', + release='1.el9', + packagetype=Package.RPM, + ) + self.host.packages.add(self.old_pkg) + + def test_process_update_creates_package_update(self): + """Test process_update creates PackageUpdate.""" + update = process_update( + host=self.host, + name='openssl', + epoch='', + version='3.0.1', + release='1.el9', + arch='x86_64', + repo_id='', + security=True, + ) + self.assertIsNotNone(update) + self.assertEqual(update.oldpackage, self.old_pkg) + + def test_process_update_security_flag(self): + """Test process_update sets security flag correctly.""" + update = process_update( + host=self.host, + name='openssl', + epoch='', + version='3.0.1', + release='1.el9', + arch='x86_64', + repo_id='', + security=True, + ) + self.assertIsNotNone(update) + self.assertTrue(update.security) + + def test_process_update_bugfix(self): + """Test process_update with bugfix (non-security) update.""" + update = process_update( + host=self.host, + name='openssl', + epoch='', + version='3.0.1', + release='1.el9', + arch='x86_64', + repo_id='', + security=False, + ) + self.assertIsNotNone(update) + self.assertFalse(update.security) + + def test_process_update_adds_to_host(self): + """Test process_update adds update to host.""" + update = process_update( + host=self.host, + name='openssl', + epoch='', + version='3.0.1', + release='1.el9', + arch='x86_64', + repo_id='', + security=True, + ) + self.assertIsNotNone(update) + self.host.updates.add(update) + self.assertEqual(self.host.updates.count(), 1) + + def test_process_update_returns_none_if_no_installed_package(self): + """Test process_update returns None if package not installed.""" + update = process_update( + host=self.host, + name='nginx', # Not installed on host + epoch='', + version='1.18.0', + release='1.el9', + arch='x86_64', + repo_id='', + security=True, + ) + self.assertIsNone(update) + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class ReportProcessTests(TestCase): + """Tests for Report.process() method.""" + + def test_report_process_protocol2_creates_host(self): + """Test Report.process() creates Host from Protocol 2 report.""" + report = Report.objects.create( + host='newhost.example.com', + domain='example.com', + report_ip='192.168.1.10', + os='Ubuntu 22.04.3 LTS', + kernel='5.15.0-91-generic', + arch='x86_64', + protocol='2', + packages=json.dumps([ + {'name': 'nginx', 'version': '1.18.0', 'release': '1', 'arch': 'amd64', 'type': 'deb'}, + ]), + repos=json.dumps([]), + modules=json.dumps([]), + sec_updates=json.dumps([]), + bug_updates=json.dumps([]), + ) + report.process(find_updates=False) + + self.assertTrue(report.processed) + host = Host.objects.get(hostname='newhost.example.com') + self.assertIsNotNone(host) + self.assertEqual(host.packages.count(), 1) + + def test_report_process_protocol2_with_packages(self): + """Test Report.process() processes packages correctly.""" + report = Report.objects.create( + host='pkghost.example.com', + domain='example.com', + report_ip='192.168.1.11', + os='Ubuntu 22.04.3 LTS', + kernel='5.15.0-91-generic', + arch='x86_64', + protocol='2', + packages=json.dumps([ + {'name': 'nginx', 'version': '1.18.0', 'release': '1', 'arch': 'amd64', 'type': 'deb'}, + {'name': 'curl', 'version': '7.81.0', 'release': '1', 'arch': 'amd64', 'type': 'deb'}, + {'name': 'vim', 'version': '8.2.0', 'release': '1', 'arch': 'amd64', 'type': 'deb'}, + ]), + repos=json.dumps([]), + modules=json.dumps([]), + sec_updates=json.dumps([]), + bug_updates=json.dumps([]), + ) + report.process(find_updates=False) + + host = Host.objects.get(hostname='pkghost.example.com') + self.assertEqual(host.packages.count(), 3) + pkg_names = [p.name.name for p in host.packages.all()] + self.assertIn('nginx', pkg_names) + self.assertIn('curl', pkg_names) + self.assertIn('vim', pkg_names) + + def test_report_process_protocol2_with_repos(self): + """Test Report.process() processes repos correctly.""" + report = Report.objects.create( + host='repohost.example.com', + domain='example.com', + report_ip='192.168.1.12', + os='Ubuntu 22.04.3 LTS', + kernel='5.15.0-91-generic', + arch='x86_64', + protocol='2', + packages=json.dumps([]), + repos=json.dumps([ + { + 'type': 'deb', + 'name': 'Ubuntu Main', + 'id': 'ubuntu-main', + 'priority': 500, + 'urls': ['http://archive.ubuntu.com/ubuntu'], + }, + ]), + modules=json.dumps([]), + sec_updates=json.dumps([]), + bug_updates=json.dumps([]), + ) + report.process(find_updates=False) + + host = Host.objects.get(hostname='repohost.example.com') + host_repos = HostRepo.objects.filter(host=host) + self.assertEqual(host_repos.count(), 1) + + def test_report_process_protocol2_with_updates(self): + """Test Report.process() processes updates correctly.""" + # This is an integration test that verifies updates flow + # process_updates_json uses process_update which requires RPM packages + # For DEB packages, a different code path is used + # Skip detailed update testing here - covered in ProcessUpdateTests + pass + + def test_report_process_sets_processed_flag(self): + """Test Report.process() sets processed=True.""" + report = Report.objects.create( + host='flaghost.example.com', + domain='example.com', + report_ip='192.168.1.14', + os='Ubuntu 22.04.3 LTS', + kernel='5.15.0-91-generic', + arch='x86_64', + protocol='2', + packages=json.dumps([]), + repos=json.dumps([]), + modules=json.dumps([]), + sec_updates=json.dumps([]), + bug_updates=json.dumps([]), + ) + self.assertFalse(report.processed) + report.process(find_updates=False) + self.assertTrue(report.processed) + + def test_report_process_skips_already_processed(self): + """Test Report.process() skips already processed reports.""" + report = Report.objects.create( + host='skiphost.example.com', + domain='example.com', + report_ip='192.168.1.15', + os='Ubuntu 22.04.3 LTS', + kernel='5.15.0-91-generic', + arch='x86_64', + protocol='2', + processed=True, + packages=json.dumps([]), + repos=json.dumps([]), + modules=json.dumps([]), + sec_updates=json.dumps([]), + bug_updates=json.dumps([]), + ) + # Should not raise an error, just return early + report.process(find_updates=False) + self.assertTrue(report.processed) + + def test_report_process_requires_os_kernel_arch(self): + """Test Report.process() requires os, kernel, and arch.""" + report = Report.objects.create( + host='incomplete.example.com', + domain='example.com', + report_ip='192.168.1.16', + os='', # Missing + kernel='5.15.0-91-generic', + arch='x86_64', + protocol='2', + ) + report.process(find_updates=False) + # Should not be processed due to missing os + self.assertFalse(report.processed) + + def test_report_process_creates_domain(self): + """Test Report.process() creates Domain if not exists.""" + self.assertEqual(Domain.objects.filter(name='newdomain.com').count(), 0) + report = Report.objects.create( + host='host.newdomain.com', + domain='newdomain.com', + report_ip='192.168.1.17', + os='Ubuntu 22.04.3 LTS', + kernel='5.15.0-91-generic', + arch='x86_64', + protocol='2', + packages=json.dumps([]), + repos=json.dumps([]), + modules=json.dumps([]), + sec_updates=json.dumps([]), + bug_updates=json.dumps([]), + ) + report.process(find_updates=False) + self.assertEqual(Domain.objects.filter(name='newdomain.com').count(), 1) + + def test_report_process_creates_osvariant(self): + """Test Report.process() creates OSVariant if not exists.""" + report = Report.objects.create( + host='oshost.example.com', + domain='example.com', + report_ip='192.168.1.18', + os='Rocky Linux 9.3', + kernel='5.14.0-362.el9.x86_64', + arch='x86_64', + protocol='2', + packages=json.dumps([]), + repos=json.dumps([]), + modules=json.dumps([]), + sec_updates=json.dumps([]), + bug_updates=json.dumps([]), + ) + report.process(find_updates=False) + host = Host.objects.get(hostname='oshost.example.com') + self.assertIsNotNone(host.osvariant) + self.assertIn('Rocky', host.osvariant.name) diff --git a/repos/tests/test_managers.py b/repos/tests/test_managers.py new file mode 100644 index 00000000..10e66eed --- /dev/null +++ b/repos/tests/test_managers.py @@ -0,0 +1,170 @@ +# Copyright 2025 Marcus Furlong +# +# This file is part of Patchman. +# +# Patchman is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, version 3 only. +# +# Patchman is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Patchman. If not, see + +from django.test import TestCase, override_settings + +from arch.models import MachineArchitecture +from repos.models import Mirror, Repository + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class RepositoryManagerTests(TestCase): + """Tests for RepositoryManager custom queryset.""" + + def setUp(self): + """Set up test data.""" + self.arch = MachineArchitecture.objects.create(name='x86_64') + + def test_repository_manager_select_related(self): + """Test RepositoryManager uses select_related for efficiency.""" + Repository.objects.create( + name='ubuntu-main', + arch=self.arch, + repotype=Repository.DEB, + ) + # Manager should return queryset with select_related + repos = Repository.objects.all() + self.assertEqual(repos.count(), 1) + + def test_repository_manager_returns_all_repos(self): + """Test RepositoryManager returns all repositories.""" + for name in ['ubuntu-main', 'ubuntu-updates', 'ubuntu-security']: + Repository.objects.create( + name=name, + arch=self.arch, + repotype=Repository.DEB, + ) + self.assertEqual(Repository.objects.count(), 3) + + def test_repository_manager_filter_by_type(self): + """Test RepositoryManager filtering by repo type.""" + Repository.objects.create( + name='ubuntu-main', + arch=self.arch, + repotype=Repository.DEB, + ) + Repository.objects.create( + name='rocky-baseos', + arch=self.arch, + repotype=Repository.RPM, + ) + deb_repos = Repository.objects.filter(repotype=Repository.DEB) + rpm_repos = Repository.objects.filter(repotype=Repository.RPM) + self.assertEqual(deb_repos.count(), 1) + self.assertEqual(rpm_repos.count(), 1) + + def test_repository_manager_filter_by_enabled(self): + """Test RepositoryManager filtering by enabled status.""" + Repository.objects.create( + name='enabled-repo', + arch=self.arch, + repotype=Repository.DEB, + enabled=True, + ) + Repository.objects.create( + name='disabled-repo', + arch=self.arch, + repotype=Repository.DEB, + enabled=False, + ) + enabled_repos = Repository.objects.filter(enabled=True) + disabled_repos = Repository.objects.filter(enabled=False) + self.assertEqual(enabled_repos.count(), 1) + self.assertEqual(disabled_repos.count(), 1) + + def test_repository_manager_filter_by_security(self): + """Test RepositoryManager filtering by security flag.""" + Repository.objects.create( + name='main-repo', + arch=self.arch, + repotype=Repository.DEB, + security=False, + ) + Repository.objects.create( + name='security-repo', + arch=self.arch, + repotype=Repository.DEB, + security=True, + ) + security_repos = Repository.objects.filter(security=True) + non_security_repos = Repository.objects.filter(security=False) + self.assertEqual(security_repos.count(), 1) + self.assertEqual(non_security_repos.count(), 1) + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class MirrorManagerTests(TestCase): + """Tests for Mirror model queries.""" + + def setUp(self): + """Set up test data.""" + self.arch = MachineArchitecture.objects.create(name='x86_64') + self.repo = Repository.objects.create( + name='ubuntu-main', + arch=self.arch, + repotype=Repository.DEB, + ) + + def test_mirror_filter_by_repo(self): + """Test filtering mirrors by repository.""" + Mirror.objects.create( + repo=self.repo, + url='http://archive.ubuntu.com/ubuntu', + ) + Mirror.objects.create( + repo=self.repo, + url='http://mirror.example.com/ubuntu', + ) + repo_mirrors = Mirror.objects.filter(repo=self.repo) + self.assertEqual(repo_mirrors.count(), 2) + + def test_mirror_filter_by_enabled(self): + """Test filtering mirrors by enabled status.""" + Mirror.objects.create( + repo=self.repo, + url='http://archive.ubuntu.com/ubuntu', + enabled=True, + ) + Mirror.objects.create( + repo=self.repo, + url='http://disabled.example.com/ubuntu', + enabled=False, + ) + enabled_mirrors = Mirror.objects.filter(enabled=True) + self.assertEqual(enabled_mirrors.count(), 1) + + def test_mirror_filter_by_last_access_ok(self): + """Test filtering mirrors by last_access_ok status.""" + Mirror.objects.create( + repo=self.repo, + url='http://working.example.com/ubuntu', + last_access_ok=True, + ) + Mirror.objects.create( + repo=self.repo, + url='http://broken.example.com/ubuntu', + last_access_ok=False, + ) + working_mirrors = Mirror.objects.filter(last_access_ok=True) + broken_mirrors = Mirror.objects.filter(last_access_ok=False) + self.assertEqual(working_mirrors.count(), 1) + self.assertEqual(broken_mirrors.count(), 1) diff --git a/repos/tests/test_mirror_sync.py b/repos/tests/test_mirror_sync.py new file mode 100644 index 00000000..ca32fdd9 --- /dev/null +++ b/repos/tests/test_mirror_sync.py @@ -0,0 +1,141 @@ +# Copyright 2025 Marcus Furlong +# +# This file is part of Patchman. +# +# Patchman is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, version 3 only. +# +# Patchman is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Patchman. If not, see + +from django.test import TestCase, override_settings + +from arch.models import MachineArchitecture, PackageArchitecture +from packages.models import Package, PackageName +from repos.models import Mirror, MirrorPackage, Repository + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class MirrorPackageTests(TestCase): + """Tests for MirrorPackage operations.""" + + def setUp(self): + """Set up test data.""" + self.machine_arch = MachineArchitecture.objects.create(name='x86_64') + self.pkg_arch = PackageArchitecture.objects.create(name='x86_64') + self.repo = Repository.objects.create( + name='test-repo', + arch=self.machine_arch, + repotype=Repository.RPM, + ) + self.mirror = Mirror.objects.create( + repo=self.repo, + url='http://mirror.example.com/repo', + ) + + def test_add_package_to_mirror(self): + """Test adding a package to mirror.""" + pkg_name = PackageName.objects.create(name='httpd') + pkg = Package.objects.create( + name=pkg_name, + arch=self.pkg_arch, + epoch='0', + version='2.4.57', + release='1.el9', + packagetype=Package.RPM, + ) + + MirrorPackage.objects.create(mirror=self.mirror, package=pkg) + self.assertEqual(self.mirror.packages.count(), 1) + self.assertIn(pkg, self.mirror.packages.all()) + + def test_remove_package_from_mirror(self): + """Test removing a package from mirror.""" + pkg_name = PackageName.objects.create(name='nginx') + pkg = Package.objects.create( + name=pkg_name, + arch=self.pkg_arch, + epoch='0', + version='1.20.0', + release='1.el9', + packagetype=Package.RPM, + ) + + mp = MirrorPackage.objects.create(mirror=self.mirror, package=pkg) + self.assertEqual(self.mirror.packages.count(), 1) + + mp.delete() + self.assertEqual(self.mirror.packages.count(), 0) + + def test_mirror_package_unique_constraint(self): + """Test that same package can't be added twice to mirror.""" + pkg_name = PackageName.objects.create(name='curl') + pkg = Package.objects.create( + name=pkg_name, + arch=self.pkg_arch, + epoch='0', + version='7.76.0', + release='1.el9', + packagetype=Package.RPM, + ) + + MirrorPackage.objects.create(mirror=self.mirror, package=pkg) + + # get_or_create should return existing, not create new + mp, created = MirrorPackage.objects.get_or_create(mirror=self.mirror, package=pkg) + self.assertFalse(created) + self.assertEqual(self.mirror.packages.count(), 1) + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class MirrorModelTests(TestCase): + """Tests for Mirror model methods.""" + + def setUp(self): + """Set up test data.""" + self.machine_arch = MachineArchitecture.objects.create(name='x86_64') + self.repo = Repository.objects.create( + name='test-repo', + arch=self.machine_arch, + repotype=Repository.RPM, + ) + self.mirror = Mirror.objects.create( + repo=self.repo, + url='http://mirror.example.com/repo', + enabled=True, + refresh=True, + ) + + def test_mirror_fail_increments_count(self): + """Test that fail() increments fail_count.""" + initial_count = self.mirror.fail_count + self.mirror.fail() + self.mirror.refresh_from_db() + self.assertEqual(self.mirror.fail_count, initial_count + 1) + + def test_mirror_fail_disables_after_threshold(self): + """Test that mirror is disabled after too many failures.""" + # Set fail count near threshold + self.mirror.fail_count = 27 # Default threshold is 28 + self.mirror.save() + + self.mirror.fail() + self.mirror.refresh_from_db() + + self.assertFalse(self.mirror.refresh) + + def test_mirror_str(self): + """Test mirror string representation.""" + self.assertIn(self.mirror.url, str(self.mirror)) diff --git a/repos/tests/test_models.py b/repos/tests/test_models.py new file mode 100644 index 00000000..a5dcb987 --- /dev/null +++ b/repos/tests/test_models.py @@ -0,0 +1,193 @@ +# Copyright 2025 Marcus Furlong +# +# This file is part of Patchman. +# +# Patchman is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, version 3 only. +# +# Patchman is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Patchman. If not, see + +from django.test import TestCase, override_settings + +from arch.models import MachineArchitecture, PackageArchitecture +from packages.models import Package, PackageName +from repos.models import Mirror, MirrorPackage, Repository + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class RepositoryMethodTests(TestCase): + """Tests for Repository model methods.""" + + def setUp(self): + """Set up test data.""" + self.arch = MachineArchitecture.objects.create(name='x86_64') + self.repo = Repository.objects.create( + name='ubuntu-main', + arch=self.arch, + repotype='D', + enabled=True, + ) + + def test_repository_get_absolute_url(self): + """Test Repository.get_absolute_url().""" + url = self.repo.get_absolute_url() + self.assertIn(str(self.repo.id), url) + + def test_repository_str(self): + """Test Repository __str__ method.""" + self.assertEqual(str(self.repo), 'ubuntu-main') + + def test_repository_type_constants(self): + """Test Repository type constants.""" + self.assertEqual(Repository.RPM, 'R') + self.assertEqual(Repository.DEB, 'D') + self.assertEqual(Repository.ARCH, 'A') + self.assertEqual(Repository.GENTOO, 'G') + + def test_repository_enabled_default(self): + """Test Repository enabled defaults to True.""" + repo = Repository.objects.create( + name='test-repo', arch=self.arch, repotype='D' + ) + self.assertTrue(repo.enabled) + + def test_repository_security_default(self): + """Test Repository security defaults to False.""" + repo = Repository.objects.create( + name='test-repo', arch=self.arch, repotype='D' + ) + self.assertFalse(repo.security) + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class MirrorMethodTests(TestCase): + """Tests for Mirror model methods.""" + + def setUp(self): + """Set up test data.""" + self.arch = MachineArchitecture.objects.create(name='x86_64') + self.repo = Repository.objects.create( + name='ubuntu-main', + arch=self.arch, + repotype='D', + ) + self.mirror = Mirror.objects.create( + repo=self.repo, + url='http://archive.ubuntu.com/ubuntu/dists/jammy/main/binary-amd64', + ) + + def test_mirror_str(self): + """Test Mirror __str__ method returns URL.""" + self.assertEqual(str(self.mirror), self.mirror.url) + + def test_mirror_get_absolute_url(self): + """Test Mirror.get_absolute_url().""" + url = self.mirror.get_absolute_url() + self.assertIn(str(self.mirror.id), url) + + def test_mirror_enabled_default(self): + """Test Mirror enabled defaults to True.""" + mirror = Mirror.objects.create( + repo=self.repo, + url='http://example.com/repo', + ) + self.assertTrue(mirror.enabled) + + def test_mirror_refresh_default(self): + """Test Mirror refresh defaults to True.""" + mirror = Mirror.objects.create( + repo=self.repo, + url='http://example.com/repo2', + ) + self.assertTrue(mirror.refresh) + + def test_mirror_fail_count_default(self): + """Test Mirror fail_count defaults to 0.""" + mirror = Mirror.objects.create( + repo=self.repo, + url='http://example.com/repo3', + ) + self.assertEqual(mirror.fail_count, 0) + + def test_mirror_last_access_ok_default(self): + """Test Mirror last_access_ok defaults to False.""" + mirror = Mirror.objects.create( + repo=self.repo, + url='http://example.com/repo4', + ) + self.assertFalse(mirror.last_access_ok) + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class MirrorPackageMethodTests(TestCase): + """Tests for MirrorPackage model.""" + + def setUp(self): + """Set up test data.""" + self.arch = MachineArchitecture.objects.create(name='x86_64') + self.pkg_arch = PackageArchitecture.objects.create(name='amd64') + self.repo = Repository.objects.create( + name='ubuntu-main', + arch=self.arch, + repotype='D', + ) + self.mirror = Mirror.objects.create( + repo=self.repo, + url='http://archive.ubuntu.com/ubuntu/dists/jammy/main/binary-amd64', + ) + self.pkg_name = PackageName.objects.create(name='nginx') + self.package = Package.objects.create( + name=self.pkg_name, arch=self.pkg_arch, + epoch='', version='1.18.0', release='1', packagetype='D' + ) + + def test_mirror_package_creation(self): + """Test creating a MirrorPackage relationship.""" + mp = MirrorPackage.objects.create( + mirror=self.mirror, + package=self.package, + enabled=True, + ) + self.assertEqual(mp.mirror, self.mirror) + self.assertEqual(mp.package, self.package) + self.assertTrue(mp.enabled) + + def test_mirror_package_enabled_default(self): + """Test MirrorPackage enabled defaults to True.""" + mp = MirrorPackage.objects.create( + mirror=self.mirror, + package=self.package, + ) + self.assertTrue(mp.enabled) + + def test_mirror_packages_relationship(self): + """Test Mirror.packages ManyToMany via MirrorPackage.""" + MirrorPackage.objects.create( + mirror=self.mirror, + package=self.package, + ) + self.assertIn(self.package, self.mirror.packages.all()) + + def test_package_repo_count(self): + """Test Package.repo_count() method.""" + MirrorPackage.objects.create( + mirror=self.mirror, + package=self.package, + ) + self.assertEqual(self.package.repo_count(), 1) diff --git a/security/tests/test_models.py b/security/tests/test_models.py new file mode 100644 index 00000000..c04acf89 --- /dev/null +++ b/security/tests/test_models.py @@ -0,0 +1,180 @@ +# Copyright 2025 Marcus Furlong +# +# This file is part of Patchman. +# +# Patchman is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, version 3 only. +# +# Patchman is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Patchman. If not, see + +from decimal import Decimal + +from django.test import TestCase, override_settings + +from security.models import CVE, CVSS, CWE, Reference + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class CVEMethodTests(TestCase): + """Tests for CVE model methods.""" + + def test_cve_creation(self): + """Test creating a CVE.""" + cve = CVE.objects.create(cve_id='CVE-2024-12345') + self.assertEqual(cve.cve_id, 'CVE-2024-12345') + + def test_cve_str(self): + """Test CVE __str__ method.""" + cve = CVE.objects.create(cve_id='CVE-2024-12345') + self.assertEqual(str(cve), 'CVE-2024-12345') + + def test_cve_get_absolute_url(self): + """Test CVE.get_absolute_url().""" + cve = CVE.objects.create(cve_id='CVE-2024-12345') + url = cve.get_absolute_url() + self.assertIn(str(cve.id), url) + + def test_cve_unique_id(self): + """Test CVE cve_id is unique.""" + CVE.objects.create(cve_id='CVE-2024-12345') + from django.db import IntegrityError + with self.assertRaises(IntegrityError): + CVE.objects.create(cve_id='CVE-2024-12345') + + def test_cve_with_cvss_score(self): + """Test CVE with associated CVSS scores.""" + cve = CVE.objects.create(cve_id='CVE-2024-99999') + score = CVSS.objects.create( + version=Decimal('3.1'), + vector_string='CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H', + score=Decimal('9.8'), + severity='CRITICAL', + ) + cve.cvss_scores.add(score) + self.assertIn(score, cve.cvss_scores.all()) + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class CWEMethodTests(TestCase): + """Tests for CWE model methods.""" + + def test_cwe_creation(self): + """Test creating a CWE.""" + cwe = CWE.objects.create(cwe_id='CWE-79', name='Cross-site Scripting') + self.assertEqual(cwe.cwe_id, 'CWE-79') + + def test_cwe_str(self): + """Test CWE __str__ method.""" + cwe = CWE.objects.create(cwe_id='CWE-79', name='Cross-site Scripting') + self.assertEqual(str(cwe), 'CWE-79 - Cross-site Scripting') + + def test_cwe_get_absolute_url(self): + """Test CWE.get_absolute_url().""" + cwe = CWE.objects.create(cwe_id='CWE-79', name='XSS') + url = cwe.get_absolute_url() + self.assertIn('CWE-79', url) + + def test_cwe_unique_id(self): + """Test CWE cwe_id is unique.""" + CWE.objects.create(cwe_id='CWE-79', name='XSS') + from django.db import IntegrityError + with self.assertRaises(IntegrityError): + CWE.objects.create(cwe_id='CWE-79', name='Different name') + + def test_cwe_int_id_property(self): + """Test CWE.int_id property extracts numeric part.""" + cwe = CWE.objects.create(cwe_id='CWE-79', name='XSS') + self.assertEqual(cwe.int_id, 79) + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class CVSSMethodTests(TestCase): + """Tests for CVSS model methods.""" + + def test_cvss_creation(self): + """Test creating a CVSS.""" + score = CVSS.objects.create( + version=Decimal('3.1'), + vector_string='CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H', + score=Decimal('9.8'), + severity='CRITICAL', + ) + self.assertEqual(score.version, Decimal('3.1')) + self.assertEqual(score.score, Decimal('9.8')) + + def test_cvss_str(self): + """Test CVSS __str__ method.""" + score = CVSS.objects.create( + version=Decimal('3.1'), + vector_string='CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H', + score=Decimal('9.8'), + severity='CRITICAL', + ) + str_repr = str(score) + self.assertIn('9.8', str_repr) + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class ReferenceMethodTests(TestCase): + """Tests for Reference model methods.""" + + def test_reference_creation(self): + """Test creating a Reference.""" + ref = Reference.objects.create( + url='https://nvd.nist.gov/vuln/detail/CVE-2024-12345', + ref_type='NVD', + ) + self.assertEqual(ref.ref_type, 'NVD') + + def test_reference_str(self): + """Test Reference __str__ method.""" + ref = Reference.objects.create( + url='https://example.com/advisory', + ref_type='VENDOR', + ) + str_repr = str(ref) + self.assertIn('example.com', str_repr) + + def test_reference_unique_together(self): + """Test Reference unique_together constraint.""" + Reference.objects.create( + url='https://example.com/advisory', + ref_type='VENDOR', + ) + from django.db import IntegrityError + with self.assertRaises(IntegrityError): + Reference.objects.create( + url='https://example.com/advisory', + ref_type='VENDOR', + ) + + def test_reference_same_url_different_type(self): + """Test same URL with different ref_type is allowed.""" + Reference.objects.create( + url='https://example.com/advisory', + ref_type='VENDOR', + ) + ref2 = Reference.objects.create( + url='https://example.com/advisory', + ref_type='ADVISORY', + ) + self.assertEqual(ref2.ref_type, 'ADVISORY') diff --git a/util/tests/__init__.py b/util/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/util/tests/test_commands.py b/util/tests/test_commands.py new file mode 100644 index 00000000..45754918 --- /dev/null +++ b/util/tests/test_commands.py @@ -0,0 +1,176 @@ +# Copyright 2025 Marcus Furlong +# +# This file is part of Patchman. +# +# Patchman is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, version 3 only. +# +# Patchman is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Patchman. If not, see + +from io import StringIO + +from django.core.management import call_command +from django.core.management.base import CommandError +from django.test import TestCase, override_settings +from rest_framework_api_key.models import APIKey + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class CreateApiKeyCommandTests(TestCase): + """Tests for create_api_key management command.""" + + def test_create_api_key_creates_key(self): + """Test create_api_key creates a new API key.""" + out = StringIO() + call_command('create_api_key', 'test-client', stdout=out) + + # Key should be created + self.assertEqual(APIKey.objects.count(), 1) + api_key = APIKey.objects.first() + self.assertEqual(api_key.name, 'test-client') + + def test_create_api_key_outputs_key(self): + """Test create_api_key outputs the generated key.""" + out = StringIO() + call_command('create_api_key', 'output-test', stdout=out) + + output = out.getvalue() + self.assertIn('output-test', output) + self.assertIn('Key:', output) + self.assertIn('api_key=', output) + + def test_create_api_key_multiple_keys(self): + """Test creating multiple API keys.""" + call_command('create_api_key', 'client1', stdout=StringIO()) + call_command('create_api_key', 'client2', stdout=StringIO()) + call_command('create_api_key', 'client3', stdout=StringIO()) + + self.assertEqual(APIKey.objects.count(), 3) + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class ListApiKeysCommandTests(TestCase): + """Tests for list_api_keys management command.""" + + def test_list_api_keys_empty(self): + """Test list_api_keys with no keys.""" + out = StringIO() + call_command('list_api_keys', stdout=out) + + output = out.getvalue() + self.assertIn('No API keys found', output) + + def test_list_api_keys_shows_keys(self): + """Test list_api_keys shows created keys.""" + APIKey.objects.create_key(name='test-key-1') + APIKey.objects.create_key(name='test-key-2') + + out = StringIO() + call_command('list_api_keys', stdout=out) + + output = out.getvalue() + self.assertIn('test-key-1', output) + self.assertIn('test-key-2', output) + self.assertIn('Total: 2', output) + + def test_list_api_keys_hides_revoked(self): + """Test list_api_keys hides revoked keys by default.""" + api_key, _ = APIKey.objects.create_key(name='active-key') + revoked_key, _ = APIKey.objects.create_key(name='revoked-key') + revoked_key.revoked = True + revoked_key.save() + + out = StringIO() + call_command('list_api_keys', stdout=out) + + output = out.getvalue() + self.assertIn('active-key', output) + self.assertNotIn('revoked-key', output) + self.assertIn('Total: 1', output) + + def test_list_api_keys_all_shows_revoked(self): + """Test list_api_keys --all shows revoked keys.""" + api_key, _ = APIKey.objects.create_key(name='active-key') + revoked_key, _ = APIKey.objects.create_key(name='revoked-key') + revoked_key.revoked = True + revoked_key.save() + + out = StringIO() + call_command('list_api_keys', '--all', stdout=out) + + output = out.getvalue() + self.assertIn('active-key', output) + self.assertIn('revoked-key', output) + self.assertIn('Total: 2', output) + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class RevokeApiKeyCommandTests(TestCase): + """Tests for revoke_api_key management command.""" + + def test_revoke_api_key_by_name(self): + """Test revoking API key by name.""" + api_key, _ = APIKey.objects.create_key(name='revoke-test') + + out = StringIO() + call_command('revoke_api_key', 'revoke-test', stdout=out) + + api_key.refresh_from_db() + self.assertTrue(api_key.revoked) + + def test_revoke_api_key_by_prefix(self): + """Test revoking API key by prefix.""" + api_key, _ = APIKey.objects.create_key(name='prefix-test') + prefix = api_key.prefix + + out = StringIO() + call_command('revoke_api_key', prefix, stdout=out) + + api_key.refresh_from_db() + self.assertTrue(api_key.revoked) + + def test_revoke_api_key_not_found(self): + """Test revoking non-existent key raises error.""" + with self.assertRaises(CommandError) as context: + call_command('revoke_api_key', 'nonexistent') + + self.assertIn('No API key found', str(context.exception)) + + def test_revoke_api_key_already_revoked(self): + """Test revoking already revoked key shows warning.""" + api_key, _ = APIKey.objects.create_key(name='already-revoked') + api_key.revoked = True + api_key.save() + + out = StringIO() + call_command('revoke_api_key', 'already-revoked', stdout=out) + + output = out.getvalue() + self.assertIn('already revoked', output) + + def test_revoke_api_key_delete(self): + """Test --delete permanently removes the key.""" + api_key, _ = APIKey.objects.create_key(name='delete-test') + + out = StringIO() + call_command('revoke_api_key', 'delete-test', '--delete', stdout=out) + + self.assertEqual(APIKey.objects.filter(name='delete-test').count(), 0) + output = out.getvalue() + self.assertIn('Permanently deleted', output) diff --git a/util/tests/test_utils.py b/util/tests/test_utils.py new file mode 100644 index 00000000..4205150f --- /dev/null +++ b/util/tests/test_utils.py @@ -0,0 +1,261 @@ +# Copyright 2025 Marcus Furlong +# +# This file is part of Patchman. +# +# Patchman is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, version 3 only. +# +# Patchman is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Patchman. If not, see + +import gzip +import hashlib +from io import BytesIO +from unittest.mock import MagicMock + +from django.test import TestCase, override_settings + +from util import ( + Checksum, bunzip2, extract, get_checksum, get_md5, get_sha1, get_sha256, + get_sha512, gunzip, has_setting_of_type, is_epoch_time, response_is_valid, + sanitize_filter_params, tz_aware_datetime, +) + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class ChecksumTests(TestCase): + """Tests for checksum functions.""" + + def test_get_sha256(self): + """Test SHA256 hash generation.""" + data = b'test data for hashing' + expected = hashlib.sha256(data).hexdigest() + result = get_sha256(data) + self.assertEqual(result, expected) + + def test_get_sha1(self): + """Test SHA1 hash generation.""" + data = b'test data for hashing' + expected = hashlib.sha1(data).hexdigest() + result = get_sha1(data) + self.assertEqual(result, expected) + + def test_get_sha512(self): + """Test SHA512 hash generation.""" + data = b'test data for hashing' + expected = hashlib.sha512(data).hexdigest() + result = get_sha512(data) + self.assertEqual(result, expected) + + def test_get_md5(self): + """Test MD5 hash generation.""" + data = b'test data for hashing' + expected = hashlib.md5(data).hexdigest() + result = get_md5(data) + self.assertEqual(result, expected) + + def test_get_checksum_sha256(self): + """Test get_checksum with sha256.""" + data = b'test data' + expected = hashlib.sha256(data).hexdigest() + result = get_checksum(data, Checksum.sha256) + self.assertEqual(result, expected) + + def test_get_checksum_sha1(self): + """Test get_checksum with sha1.""" + data = b'test data' + expected = hashlib.sha1(data).hexdigest() + result = get_checksum(data, Checksum.sha1) + self.assertEqual(result, expected) + + def test_get_checksum_md5(self): + """Test get_checksum with md5.""" + data = b'test data' + expected = hashlib.md5(data).hexdigest() + result = get_checksum(data, Checksum.md5) + self.assertEqual(result, expected) + + def test_get_checksum_sha512(self): + """Test get_checksum with sha512.""" + data = b'test data' + expected = hashlib.sha512(data).hexdigest() + result = get_checksum(data, Checksum.sha512) + self.assertEqual(result, expected) + + def test_get_checksum_empty_data(self): + """Test checksum of empty data.""" + data = b'' + result = get_sha256(data) + expected = hashlib.sha256(b'').hexdigest() + self.assertEqual(result, expected) + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class CompressionTests(TestCase): + """Tests for compression/decompression functions.""" + + def test_gunzip_valid_data(self): + """Test gunzip with valid gzipped data.""" + original = b'Hello, World! This is test data.' + buf = BytesIO() + with gzip.GzipFile(fileobj=buf, mode='wb') as f: + f.write(original) + compressed = buf.getvalue() + + result = gunzip(compressed) + self.assertEqual(result, original) + + def test_gunzip_invalid_data(self): + """Test gunzip with invalid data returns None.""" + result = gunzip(b'not gzipped data') + self.assertIsNone(result) + + def test_bunzip2_invalid_data(self): + """Test bunzip2 with invalid data returns None.""" + result = bunzip2(b'not bzip2 data') + self.assertIsNone(result) + + def test_extract_gzip(self): + """Test extract with gzip format.""" + original = b'test content' + buf = BytesIO() + with gzip.GzipFile(fileobj=buf, mode='wb') as f: + f.write(original) + compressed = buf.getvalue() + + result = extract(compressed, 'gz') + self.assertEqual(result, original) + + def test_extract_unknown_format(self): + """Test extract with unknown format returns original.""" + data = b'unchanged data' + result = extract(data, 'unknown') + self.assertEqual(result, data) + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class URLFetchTests(TestCase): + """Tests for URL fetching functions.""" + + def test_response_is_valid_200(self): + """Test response_is_valid with 200 status.""" + mock_response = MagicMock() + mock_response.ok = True + self.assertTrue(response_is_valid(mock_response)) + + def test_response_is_valid_error(self): + """Test response_is_valid with error status.""" + mock_response = MagicMock() + mock_response.ok = False + self.assertFalse(response_is_valid(mock_response)) + + def test_response_is_valid_none(self): + """Test response_is_valid with None.""" + self.assertFalse(response_is_valid(None)) + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class SanitizeFilterParamsTests(TestCase): + """Tests for sanitize_filter_params function.""" + + def test_sanitize_simple_params(self): + """Test sanitizing simple filter params.""" + params = 'name=test&version=1.0' + result = sanitize_filter_params(params) + self.assertIn('name=test', result) + self.assertIn('version=1.0', result) + + def test_sanitize_empty_params(self): + """Test sanitizing empty params.""" + result = sanitize_filter_params('') + self.assertEqual(result, '') + + def test_sanitize_none_params(self): + """Test sanitizing None params.""" + result = sanitize_filter_params(None) + self.assertEqual(result, '') + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class DateTimeTests(TestCase): + """Tests for datetime utility functions.""" + + def test_is_epoch_time_valid(self): + """Test is_epoch_time with valid epoch timestamp.""" + self.assertTrue(is_epoch_time(1704067200)) # 2024-01-01 00:00:00 + + def test_is_epoch_time_invalid_string(self): + """Test is_epoch_time with non-numeric string.""" + self.assertFalse(is_epoch_time('not-a-number')) + + def test_is_epoch_time_too_large(self): + """Test is_epoch_time with unreasonably large value.""" + self.assertFalse(is_epoch_time(99999999999999)) + + def test_is_epoch_time_negative(self): + """Test is_epoch_time with negative value.""" + self.assertFalse(is_epoch_time(-1)) + + def test_tz_aware_datetime_from_epoch(self): + """Test converting epoch timestamp to timezone-aware datetime.""" + epoch = 1704067200 # 2024-01-01 00:00:00 UTC + result = tz_aware_datetime(epoch) + self.assertIsNotNone(result.tzinfo) + + def test_tz_aware_datetime_from_string(self): + """Test converting string datetime to timezone-aware.""" + date_str = '2024-01-01T12:00:00' + result = tz_aware_datetime(date_str) + self.assertIsNotNone(result.tzinfo) + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class SettingsTests(TestCase): + """Tests for settings utility functions.""" + + @override_settings(TEST_STRING_SETTING='test_value') + def test_has_setting_of_type_string_exists(self): + """Test has_setting_of_type with existing string setting.""" + result = has_setting_of_type('TEST_STRING_SETTING', str) + self.assertTrue(result) + + def test_has_setting_of_type_missing(self): + """Test has_setting_of_type with missing setting.""" + result = has_setting_of_type('NONEXISTENT_SETTING_XYZ', str) + self.assertFalse(result) + + @override_settings(TEST_INT_SETTING=42) + def test_has_setting_of_type_wrong_type(self): + """Test has_setting_of_type with wrong type.""" + result = has_setting_of_type('TEST_INT_SETTING', str) + self.assertFalse(result) + + @override_settings(TEST_BOOL_SETTING=True) + def test_has_setting_of_type_bool(self): + """Test has_setting_of_type with bool setting.""" + result = has_setting_of_type('TEST_BOOL_SETTING', bool) + self.assertTrue(result) From 909a096dde719fc163def3fee9890226f837e974 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Fri, 18 Apr 2025 12:16:19 -0400 Subject: [PATCH 061/146] update README --- README.md | 169 ++++++++++++++++++++++-------------------------------- 1 file changed, 70 insertions(+), 99 deletions(-) diff --git a/README.md b/README.md index 11088556..0cb94199 100644 --- a/README.md +++ b/README.md @@ -31,20 +31,20 @@ time packages are installed or removed on a host. ## Installation -See [the installation guide](https://github.com/furlongm/patchman/blob/master/INSTALL.md) +See [the installation guide](https://github.com/furlongm/patchman/blob/main/INSTALL.md) for installation options. ## Usage The web interface contains a dashboard with items that need attention, and -various pages to manipulate hosts, repositories, packages, operating systems and -reports. +various pages to manipulate and view hosts, repositories and mirrors, packages, +operating system releases and variants, reports, errata and CVEs. To populate the database, simply run the client on some hosts: ```shell -$ patchman-client -s http://patchman.example.org +$ patchman-client -s http://patchman.example.com ``` This should provide some initial data to work with. @@ -56,69 +56,56 @@ the usage: ```shell $ sbin/patchman -h -usage: patchman [-h] [-f] [-q] [-r] [-R REPO] [-lr] [-lh] [-u] [-A] [-H HOST] - [-p] [-c] [-d] [-n] [-a] [-D hostA hostB] +usage: patchman [-h] [-f] [-q] [-r] [-R REPO] [-lr] [-lh] [-dh] [-u] [-A] [-shro | -uhro] [-sdns | -udns] [-H HOST] [-p] [-c] [-d] [-rd] [-n] [-a] [-D hostA hostB] [-e] [-E ERRATUM_TYPE] [-v] [--cve CVE] [--fetch-nist-data] Patchman CLI tool -optional arguments: +options: -h, --help show this help message and exit - -f, --force Ignore stored checksums and force-refresh all mirrors + -f, --force Ignore stored checksums and force-refresh all Mirrors -q, --quiet Quiet mode (e.g. for cronjobs) - -r, --refresh-repos Refresh repositories - -R REPO, --repo REPO Only perform action on a specific repository (repo_id) - -lr, --list-repos List all repositories - -lh, --list-hosts List all hosts - -u, --host-updates Find host updates + -r, --refresh-repos Refresh Repositories + -R REPO, --repo REPO Only perform action on a specific Repository (repo_id) + -lr, --list-repos List all Repositories + -lh, --list-hosts List all Hosts + -dh, --delete-hosts Delete hosts, requires -H, matches substring patterns + -u, --host-updates Find Host updates -A, --host-updates-alt - Find host updates (alternative algorithm that may be - faster when there are many homogeneous hosts) - -H HOST, --host HOST Only perform action on a specific host (fqdn) + Find Host updates (alternative algorithm that may be faster when there are many homogeneous hosts) + -shro, --set-host-repos-only + Set host_repos_only, requires -H, matches substring patterns + -uhro, --unset-host-repos-only + Unset host_repos_only, requires -H, matches substring patterns + -sdns, --set-check-dns + Set check_dns, requires -H, matches substring patterns + -udns, --unset-check-dns + Unset check_dns, requires -H, matches substring patterns + -H HOST, --host HOST Only perform action on a specific Host (fqdn) -p, --process-reports - Process pending reports - -c, --clean-reports Remove all but the last three reports + Process pending Reports + -c, --clean-reports Remove all but the last three Reports -d, --dbcheck Perform some sanity checks and clean unused db entries - -n, --dns-checks Perform reverse DNS checks if enabled for that host - -a, --all Convenience flag for -r -A -p -c -d -n + -rd, --remove-duplicates + Remove duplicates during dbcheck - this may take some time + -n, --dns-checks Perform reverse DNS checks if enabled for that Host + -a, --all Convenience flag for -r -A -p -c -d -n -e -D hostA hostB, --diff hostA hostB - Show differences between two hosts in diff-like output - -e, --errata Download CentOS errata from https://cefs.steve- - meier.de/ + Show differences between two Hosts in diff-like output + -e, --update-errata Update Errata + -E ERRATUM_TYPE, --erratum-type ERRATUM_TYPE + Only update the specified Erratum type (e.g. `yum`, `ubuntu`, `arch`) + -v, --update-cves Update CVEs from https://cve.org + --cve CVE Only update the specified CVE (e.g. CVE-2024-1234) + --fetch-nist-data, -nd + Fetch NIST CVE data in addition to MITRE data (rate-limited to 1 API call every 6 seconds) ``` -## Dependencies +### Client dependencies -### Server-side dependencies - - -``` -python3-django -python3-django-tagging -python3-django-extensions -python3-django-bootstrap3 -python3-djangorestframework -python3-debian -python3-rpm -python3-progressbar -python3-lxml -python3-defusedxml -python3-requests -python3-colorama -python3-magic -python3-humanize -python3-yaml -``` - -The server can optionally make use of celery to asynchronously process the -reports sent by hosts. - - -### Client-side dependencies - -The client-side dependencies are kept to a minimum. `rpm` and `dpkg` are +The client dependencies are kept to a minimum. `rpm` and `dpkg` are required to report packages, `yum`, `dnf`, `zypper` and/or `apt` are required to report repositories. These packages are normally installed by default on -most systems. +most systems. `which`, `mktemp`, `flock` and `curl` are also required. For Protocol 2 (JSON-based reports), `jq` is required. If `jq` is not available, the client will automatically fall back to Protocol 1 (text-based reports). @@ -137,24 +124,26 @@ The default settings will be fine for most people but depending on your setup, there may be some initial work required to logically organise the data sent in the host reports. The following explanations may help in this case. -There are a number of basic objects - Hosts, Repositories, Packages, Operating -Systems and Reports. There are also Operating System Groups (which are optional) -and Mirrors. +There are a number of basic objects: Hosts, Repositories and Mirrors, Packages, +Operating Systems Releases and Variants, Reports and Errata. ### Host -A Host is a single host, e.g. test01.example.org. +A Host is a single host, e.g. test-host-01.example.com. -### Operating System -A Host runs an Operating System, e.g. CentOS 7.7, Debian 10.1, Ubuntu 18.04 +### Operating System Releases and Variants +A Host runs an Operating System Release, e.g. Rocky 10, Debian 13, +Ubuntu 24.04. The particular version running is called a Operating System +Variant. e.g. Debian 13.1, Ubuntu 24.04.4 and Variants are linked to a +Release. For some OS's like Arch Linux, there are no Variants. ### Package A Package is a package that is either installed on a Host, or is available to -download from a Repository mirror, e.g. `strace-4.8-11.el7.x86_64`, -`grub2-tools-2.02-0.34.el7.centos.x86_64`, etc. +download from a Repository mirror, e.g. `strace-4.8-11.el10.x86_64`, +`grub2-tools-2.02-0.34.el10.rocky.x86_64`, etc. ### Mirror -A Mirror is a collection of Packages available on the web, e.g. a `yum`, `yast` -or `apt` repo. +A Mirror is a collection of Packages available on the web, e.g. a `yum` or +`apt` repo. ### Repository A Repository is a collection of Mirrors. Typically all the Mirrors will contain @@ -163,45 +152,27 @@ their Mirrors together. For Debian-based hosts, you may need to link all Mirrors that form a Repository using the web interface. This may reduce the time required to find updates. Repositories can be marked as being security or non-security. This makes most sense with Debian and Ubuntu repositories where -security updates are delivered via security repositories. For CentOS security +security updates are delivered via security repositories. For rpm security updates, see the Erratum section below. -### Report -A Host creates a Report using `patchman-client`. This Report is sent to the -Patchman server. The Report contains the Host's Operating System, and lists -of the installed Packages and enabled Repositories on the Host. The Patchman -server processes and records the list of Packages and Repositories contained in -the Report. - -### Operating System Group (optional) -An OSGroup is a collection of OS's. For example, an OSGroup named "Ubuntu 18.04" -would be comprised of the following OS's: - -``` -Ubuntu 18.04.1 -Ubuntu 18.04.2 -Ubuntu 18.04.5 -``` - -Likewise, an OSGroup named "CentOS 7" would be made up of the following OS's: - -``` -CentOS 7.5 -CentOS 7.7.1511 -``` - -Repositories can be associated with an OSGroup, or with the Host itself. If the -`use_host_repos variable` is set to True for a Host, then updates are found by -looking only at the Repositories that belong to that Host. This is the default -behaviour and does not require OSGroups to be configured. +Repositories can be associated with an OS Release, or with the Host itself. If +the `use_host_repos` variable is set to True for a Host, then updates are found +by looking only at the Repositories that belong to that Host. This is the +default behaviour. If `use_host_repos` is set to False, the update-finding process looks at the -OSGroup that the Host's Operating System is in, and uses the OSGroup's -Repositories to determine the applicable updates. This is useful in environments -where many hosts are homogeneous (e.g. cloud/cluster environments). +OS Release that the Hosts Operating System Variant is associated with, and +uses that Releases Repositories to determine the applicable updates. This is +useful in environments where many hosts are homogeneous. + +### Report +Hosts create Reports using `patchman-client`. This Report is sent to the +Patchman server. The Report contains the Hosts running kernel, Operating System, +installed Packages and enabled Repositories. The Patchman server processes the +Report records the information contained therein. ### Erratum -Errata for CentOS can be downloaded from https://cefs.steve-meier.de/ . -These errata are parsed and stored in the database. If a PackageUpdate -contains a package that is a security update in the errata, then that update is -marked as being a security update. +Errata for many OS's can downloaded by the patchman server. These Errata are +parsed and stored in the database. If a PackageUpdate contains a package that +is a security update in an Erratum, then that update is marked as being a +security update. CVE and CVSS data is used to complement this information. From 1ed23f7d45b2cbe7a1b79a4aca0d9149ac7b13dd Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Fri, 18 Apr 2025 11:56:41 -0400 Subject: [PATCH 062/146] update installation instructions --- BUILD.md | 9 +- INSTALL.md | 324 +++++++++++++++++++-------------- etc/patchman/local_settings.py | 2 +- 3 files changed, 196 insertions(+), 139 deletions(-) diff --git a/BUILD.md b/BUILD.md index ed55358e..447690e7 100644 --- a/BUILD.md +++ b/BUILD.md @@ -5,21 +5,24 @@ vim VERSION.txt # modify version git add VERSION.txt version=$( /usr/share/keyrings/openbytes.gpg -echo "deb [signed-by=/usr/share/keyrings/openbytes.gpg] https://repo.openbytes.ie/patchman/ubuntu jammy main" > /etc/apt/sources.list.d/patchman.list +echo "deb [signed-by=/usr/share/keyrings/openbytes.gpg] https://repo.openbytes.ie/patchman/ubuntu noble-backports main" > /etc/apt/sources.list.d/patchman.list apt update apt -y install python3-patchman patchman-client patchman-manage createsuperuser ``` -### Debian 12 (bookworm) +### Debian 13 (trixie) ```shell curl -sS https://repo.openbytes.ie/openbytes.gpg > /usr/share/keyrings/openbytes.gpg -echo "deb [signed-by=/usr/share/keyrings/openbytes.gpg] https://repo.openbytes.ie/patchman/debian bookworm main" > /etc/apt/sources.list.d/patchman.list +echo "deb [signed-by=/usr/share/keyrings/openbytes.gpg] https://repo.openbytes.ie/patchman/debian trixie main" > /etc/apt/sources.list.d/patchman.list apt update apt -y install python3-patchman patchman-client patchman-manage createsuperuser ``` -### CentOS 9 +### Rocky 10 -This also applies to Rocky/Alma/RHEL +Currently broken due to missing upstream packages: https://github.com/furlongm/patchman/issues/669 + +This also applies to Alma, RHEL, etc. ```shell curl -sS https://repo.openbytes.ie/openbytes.gpg > /etc/pki/rpm-gpg/RPM-GPG-KEY-openbytes cat <> /etc/yum.repos.d/openbytes.repo [openbytes] name=openbytes -baseurl=https://repo.openbytes.ie/patchman/el9 +baseurl=https://repo.openbytes.ie/patchman/el10 enabled=1 gpgcheck=1 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-openbytes EOF -update-crypto-policies --set DEFAULT:SHA1 dnf -y install epel-release dnf makecache dnf -y install patchman patchman-client @@ -60,7 +61,7 @@ TBD - not working yet ```shell apt -y install gcc libxml2-dev libxslt1-dev virtualenv python3-dev zlib1g-dev # (debian/ubuntu) -dnf -y install gcc libxml2-devel libxslt-devel python3-virtualenv # (centos/rocky/alma) +dnf -y install gcc libxml2-devel libxslt-devel python3-virtualenv # (rocky/alma/redhat) mkdir /srv/patchman cd /srv/patchman python3 -m venv .venv @@ -74,7 +75,7 @@ gunicorn patchman.wsgi -b 0.0.0.0:80 ### Source -#### Ubuntu 22.04 (jammy) +#### Ubuntu 24.04 (noble) 1. Install dependencies @@ -107,9 +108,9 @@ cp /srv/patchman/etc/patchman/local_settings.py /etc/patchman/ # Configuration -## Patchman Settings +## Patchman Server Settings -Modify `/etc/patchman/local_settings.py` to configure patchman. +Modify `/etc/patchman/local_settings.py` to configure the patchman server. If installing from source or using virtualenv, the following settings should be configured: @@ -119,36 +120,41 @@ be configured: * STATIC_ROOT - should point to `/srv/patchman/run/static` if installing from source -## Patchman-client Settings +The default settings for errata downloading may include operating systems that +are not relevant to a given deployment. If this is the case, modify the +`ERRATA_OS_UPDATES` setting in `/etc/patchman/local_settings.py`. Further +distribution-specific settings are also available to only download errata +for specific versions/codenames. + +## Patchman Client Settings -The client comes with a default configuration. This configuration will attempt to upload the reports to a server at *patchman.example.com*. This configuration needs to be updated to connect to your own patchman installation. +The client comes with a default configuration that will attempt to upload the +reports to a server at *patchman.example.com*. This configuration needs to be +updated to connect to the correct patchman server. -In `/etc/patchman/patchman-client.conf`, look for the following line(s): +Change the following lines in `/etc/patchman/patchman-client.conf`: ``` # Patchman server -server=https://patchman.example.com +server=https://patchman.example.com # Options to curl curl_options="--insecure --connect-timeout 60 --max-time 300" -... ``` - * *server* needs to point the URL where your patchman server -is running - * *--insecure* in the curl_options tells the client to ignore certificates, if you set them up correctly and are using patchman with "https:/...", you could remove this flag to increase security - - -## Report Protocols + * *server* needs to point the URL where the patchman server is running + * *--insecure* in the curl options tells the client to ignore certificates. + If the patchman server is set up correctly with certificates this flag can + be removed to increase security. Patchman supports two report protocols: -### Protocol 1 (Text) -The original form-based protocol. Uses multipart form data to upload package +### Protocol 1 (text) +The original text-based protocol. Uses multipart form data to upload package and repository information. No additional dependencies required on the client. -### Protocol 2 (JSON) +### Protocol 2 (json) A JSON-based REST API. Provides better error handling, structured validation, and easier debugging. Requires `jq` on the client. @@ -165,70 +171,6 @@ $ patchman-client -s http://patchman.example.org -p 2 ``` -## API Key Authentication - -For Protocol 2, API key authentication is available using -[djangorestframework-api-key](https://florimondmanca.github.io/djangorestframework-api-key/). -Keys are hashed in the database and cannot be retrieved after creation. - -### Server-side setup - -1. Run migrations (first time only): - -```shell -$ patchman-manage migrate -``` - -2. Create an API key: - -```shell -$ patchman-manage create_api_key "clients" -Created API key: clients - - Key: abc123... - -Add this to your patchman-client.conf: - api_key=abc123... - -Save this key as it cannot be retrieved later. -``` - -3. List existing keys: - -```shell -$ patchman-manage list_api_keys -``` - -4. Revoke a key: - -```shell -$ patchman-manage revoke_api_key -``` - -5. To require API keys for all Protocol 2 uploads, set in `local_settings.py`: - -```python -REQUIRE_API_KEY = True -``` - -API keys can also be managed via the Django admin interface. - -### Client-side setup - -Add the API key to `patchman-client.conf`: - -```shell -protocol=2 -api_key=abc123... -``` - -Or use the `-k` command line option: - -```shell -$ patchman-client -s http://patchman.example.org -p 2 -k abc123... -``` - - ## Configure Database The default database backend is sqlite. However, this is not recommended for @@ -238,10 +180,13 @@ production deployments. MySQL or PostgreSQL are better choices. To configure the sqlite database backend: -1. Create the database directory specified in the settings file: +1. Create the database directory specified in the settings file, touch the +database file and set the journal mode to WAL: ```shell mkdir -p /var/lib/patchman/db +touch /var/lib/patchman/db/patchman.db +sqlite3 /var/lib/patchman/db/patchman.db 'PRAGMA journal_mode=WAL;' ``` 2. Modify `/etc/patchman/local_settings.py` as follows: @@ -250,7 +195,7 @@ mkdir -p /var/lib/patchman/db DATABASES = { 'default': { 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': '/var/lib/patchman/db/patchman.db' + 'NAME': '/var/lib/patchman/db/patchman.db', } } ``` @@ -286,16 +231,16 @@ Query OK, 0 rows affected (0.00 sec) ``` DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.mysql', - 'NAME': 'patchman', - 'USER': 'patchman', - 'PASSWORD': 'changeme', - 'HOST': '', - 'PORT': '', - 'STORAGE_ENGINE': 'INNODB', - 'CHARSET' : 'utf8' - } + 'default': { + 'ENGINE': 'django.db.backends.mysql', + 'NAME': 'patchman', + 'USER': 'patchman', + 'PASSWORD': 'changeme', + 'HOST': '', + 'PORT': '', + 'STORAGE_ENGINE': 'INNODB', + 'CHARSET': 'utf8', + } } ``` @@ -331,21 +276,23 @@ postgres=# GRANT ALL PRIVILEGES ON DATABASE patchman to patchman; GRANT postgres=# GRANT ALL ON SCHEMA public TO patchman; GRANT +ALTER DATABASE patchman OWNER TO patchman; +ALTER DATABASE ``` 3. Modify `/etc/patchman/local_settings.py` as follows: ``` DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.postgresql_psycopg2', - 'NAME': 'patchman', - 'USER': 'patchman', - 'PASSWORD': 'changeme', - 'HOST': '127.0.0.1', - 'PORT': '', - 'CHARSET' : 'utf8' - } + 'default': { + 'ENGINE': 'django.db.backends.postgresql_psycopg2', + 'NAME': 'patchman', + 'USER': 'patchman', + 'PASSWORD': 'changeme', + 'HOST': '127.0.0.1', + 'PORT': '', + 'CHARSET': 'utf8', + } } ``` @@ -360,8 +307,7 @@ After configuring a database backend, the django database should be synced: collect static files: ```shell -patchman-manage makemigrations -patchman-manage migrate --run-syncdb --fake-initial +patchman-manage migrate --run-syncdb patchman-manage createsuperuser patchman-manage collectstatic ``` @@ -372,6 +318,27 @@ N.B. To run patchman-manage when installing from source, run `./manage.py` 2. Restart the web server after syncing the database. +### Migrate from sqlite to another database backend + +The prebuilt package installations use sqlite as the default database backend, +but this is not recommended in production. To migrate from sqlite to another +database backend, use the following procedure: + +1. Dump the sqlite database to a json file + +```shell +patchman-manage dumpdata --exclude packages.Packagestring -e contenttypes -e auth.Permission --natural-foreign --natural-primary --indent 4 > patchman-db.json +``` + +2. Create the new database and add the new database settings to `/etc/patchman/local_settings.py` + +3. Sync the new database and load the existing data: + +``` +patchman-manage migrate --run-syncdb +patchman-manage loaddata patchman-db.json +``` + ## Configure Web Server ### Apache @@ -388,7 +355,7 @@ a2enconf patchman ```shell vi /etc/apache2/conf-available/patchman.conf -service apache2 reload +systemctl reload apache2 ``` 3. If installing from source, allow apache access to the settings and to the sqlite db: @@ -408,9 +375,10 @@ The django interface should be available at http://127.0.0.1/patchman/ #### Daily cronjob on patchman server -A daily cronjob on the patchman server should be run to process reports, -perform database maintenance, check for upstream updates, and find updates for -clients. +A daily cronjob on the patchman server can be run to process reports, perform +database maintenance, check for upstream updates, and find updates for clients. +Alternatively, run celery as outlined below for finer granularity over the +timing of these tasks and for increased concurrency. ``` patchman -a @@ -424,16 +392,17 @@ patchman-client ### Celery -Install Celery for realtime processing of reports from clients: +Install Celery for realtime processing of reports from clients and for periodic +maintenance tasks. The celery configuation file is in `/etc/patchman/celery.conf` #### Ubuntu / Debian ```shell apt -y install python3-celery redis python3-redis python-celery-common -C_FORCE_ROOT=1 celery -b redis://127.0.0.1:6379/0 -A patchman worker -l INFO -E +/usr/bin/celery --broker redis://127.0.0.1:6379/0 --app patchman worker --loglevel info --beat --scheduler django_celery_beat.schedulers:DatabaseScheduler --task-events --pool threads ``` -#### CentOS / Rocky / Alma +#### Rocky / Alma / RHEL Currently waiting on https://bugzilla.redhat.com/show_bug.cgi?id=2032543 @@ -442,21 +411,54 @@ dnf -y install python3-celery redis python3-redis systemctl restart redis semanage port -a -t http_port_t -p tcp 6379 setsebool -P httpd_can_network_connect 1 -C_FORCE_ROOT=1 celery -b redis://127.0.0.1:6379/0 -A patchman worker -l INFO -E +/usr/bin/celery --broker redis://127.0.0.1:6379/0 --app patchman worker --loglevel info --beat --scheduler django_celery_beat.schedulers:DatabaseScheduler --task-events --pool threads ``` -Add the last command to an initscript (e.g. /etc/rc.local) to make celery -persistent over reboot. +#### Persistence + +There is a systemd unit file for celery to make the service persistent over reboot: + +`etc/systemd/system/patchman-celery.service` + +If installing from prebuilt packages, this should be enabled by default. -Enable celery by adding `USE_ASYNC_PROCESSING = True` to `/etc/patchman/local_settings.py` -### Memcached +### Caching -Memcached can optionally be run to reduce the load on the server. +Memcached or Redis can optionally be run to reduce the load on the server. +Note that caching may result in the web interface showing results that are +out of date with the database, so this is disabled by default. + + +#### Redis + +Install Redis: + +```shell +apt -y install redis python3-redis # (debian/ubuntu) +dnf -y install redis python3-redis # (rocky/alma/redhat) +systemctl restart redis/redis-server +``` + +and add the following to `/etc/patchman/local_settings.py` + +``` +CACHES = { + 'default': { + 'BACKEND': 'django.core.cache.backends.redis.RedisCache', + 'LOCATION': 'redis://127.0.0.1:6379', + 'TIMEOUT': 30, + } +} +``` + +#### Memcacached + +Install Memcached ```shell apt -y install memcached python3-pymemcache # (debian/ubuntu) -dnf -y install memcached python3-pymemcache # (centos/rocky/alma) +dnf -y install memcached python3-pymemcache # (rocky/alma/redhat) systemctl restart memcached ``` @@ -464,16 +466,68 @@ and add the following to `/etc/patchman/local_settings.py` ``` CACHES = { - 'default': { - 'BACKEND': 'django.core.cache.backends.memcached.PyMemcacheCache', - 'LOCATION': '127.0.0.1:11211', + 'default': { + 'BACKEND': 'django.core.cache.backends.memcached.PyMemcacheCache', + 'LOCATION': '127.0.0.1:11211', + 'TIMEOUT': 30, 'OPTIONS': { 'ignore_exc': True, }, - } + } } ``` +## API Key Authentication + +API key authentication is available. Keys are hashed in the database and +cannot be retrieved after creation. + +### Server-side setup + +1. Create an API key: + +```shell +$ patchman-manage create_api_key "clients" +Created API key: clients + + Key: abc123... + +Add this to your patchman-client.conf: + api_key=abc123... + +Save this key as it cannot be retrieved later. +``` + +2. List existing keys: + +```shell +$ patchman-manage list_api_keys +``` + +3. Revoke a key: + +```shell +$ patchman-manage revoke_api_key +``` + +API keys can also be managed via the Django admin interface. + +### Client-side setup + +Add the API key to `patchman-client.conf`: + +```shell +protocol=2 +api_key=abc123... +``` + +Or use the `-k` command line option: + +```shell +$ patchman-client -s http://patchman.example.org -p 2 -k abc123... +``` + + # Test Installation To test the installation, run the client locally on the patchman server: diff --git a/etc/patchman/local_settings.py b/etc/patchman/local_settings.py index cb19e268..9d0b683b 100644 --- a/etc/patchman/local_settings.py +++ b/etc/patchman/local_settings.py @@ -8,7 +8,7 @@ DATABASES = { 'default': { -# 'ENGINE': 'django.db.backends.sqlite3', # noqa disabled until django 5.1 is in use, see https://blog.pecar.me/django-sqlite-dblock +# 'ENGINE': 'django.db.backends.sqlite3', # noqa - disabled until django 5.1 is in use, see https://blog.pecar.me/django-sqlite-dblock 'ENGINE': 'patchman.sqlite3', 'NAME': '/var/lib/patchman/db/patchman.db', 'OPTIONS': { From c0999196572165d0480bf18935409e93fc325a09 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Mon, 2 Feb 2026 04:33:27 +0000 Subject: [PATCH 063/146] update installation instructions --- INSTALL.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/INSTALL.md b/INSTALL.md index 9e53b074..41aef419 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -39,7 +39,7 @@ Currently broken due to missing upstream packages: https://github.com/furlongm/p This also applies to Alma, RHEL, etc. ```shell -curl -sS https://repo.openbytes.ie/openbytes.gpg > /etc/pki/rpm-gpg/RPM-GPG-KEY-openbytes +curl -sS https://repo.openbytes.ie/openbytes-2.gpg > /etc/pki/rpm-gpg/RPM-GPG-KEY-openbytes cat <> /etc/yum.repos.d/openbytes.repo [openbytes] name=openbytes @@ -140,7 +140,6 @@ server=https://patchman.example.com # Options to curl curl_options="--insecure --connect-timeout 60 --max-time 300" - ``` * *server* needs to point the URL where the patchman server is running From 21c7f91d1111e1c73078a2b2b4faafb9aa3ab3d3 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Mon, 2 Feb 2026 04:34:27 +0000 Subject: [PATCH 064/146] auto-commit to update version skip-checks: true --- VERSION.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION.txt b/VERSION.txt index 25875f01..fcdb2e10 100644 --- a/VERSION.txt +++ b/VERSION.txt @@ -1 +1 @@ -3.0.19 +4.0.0 From a8b82a0cca7c5e67c4d34eab4e664e7d5759fc61 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Mon, 2 Feb 2026 04:34:46 +0000 Subject: [PATCH 065/146] auto-commit to update debian changelog skip-checks: true --- debian/changelog | 275 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 275 insertions(+) diff --git a/debian/changelog b/debian/changelog index 452ac0b4..efc0eb49 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,278 @@ +patchman (4.0.0-1) stable; urgency=medium + + [ grumo35 ] + * Update __init__.py + + [ pyup-bot ] + * Update version-utils from 0.3.0 to 0.3.2 + * Update python-magic from 0.4.25 to 0.4.27 + * Update colorama from 0.4.4 to 0.4.6 + * Update python-debian from 0.1.43 to 0.1.49 + * Update chardet from 4.0.0 to 5.1.0 + * Update django-extensions from 3.1.5 to 3.2.3 + * Update pyyaml from 6.0.1 to 6.0.2 + * Update humanize from 3.13.1 to 4.11.0 + + [ Marcus Furlong ] + * support for timezone-aware datetimes + * add function to check django settings + * update os and osgroup to use os codenames + * switch from obsolete django-tagging to django-taggit + * additional errata sources and processing + * gentoo client support + * operatingsystem updates + * template, filter and view updates + * flake8 fixes + * improve requests error handling + * clean up repo and mirror handling + * use timezone-aware datetimes by default + * flake8 fixes + * workaround smart_str usage + * standardize fstrings + * lowercase boolean querystring args + * remove extra comma + * add icon to external links + * return response for 404 + * disable centos updates + * switch to redis for caching and add celery tasks + * standardize f-strings + * remove tagging requirement + * submit CPE_NAME with client + * remove unused os templates + * simplify apt repo naming + * improvements to errata downloading and processing + * template and view updates + * celery and task updates + * correctly filter periodic reports + * celery tasks use id instead of non-serializable json + * add patchman-celery service + * enable WAL mode for sqlite3 db + * simplify osvariant deletion + * add missing f-strings + * add migrations for os and cves/cwes + * create db cleaning celery task + * ubuntu errata concurrent processing + * concurrrent arch linux errata processing + * add helper function to get settings + * celery task updates + * update manage.py to match upstream + * use BEGIN IMMEDIATE on sqlite3 + * remove extra space + * concurrent errata processing updates + * simplify report cleaning + * switch to tqdm + * add task to find host updates + * bugfix for errata tasks + * show date without time + * make get requests args consistent + * add space before glyphicon + * flake8 + * improve package handling and duplicate removal + * use thread pool for celery to allow child concurrency + * add button to find host updates from ui + * imports at top of functions + * improve mirror list processing + * handle amazon linux repos not adding package count + * use bootstrap template tags consistently + * pyflakes + * add migrations for tz-aware DataTimeFields + * update TODO list + * update erratum creation function + * update progress bar type + * don't retry ConnectionErrors + * consistent formatting + * add more cases to tz_aware_datetime + * add category to package duplicate detection + * update progressbars + * update erratum creation + * add task to update rpm errata + * formatting updates + * rename checksum and add module/errata checksums + * refactor repo handling to add rpm errata support + * add missing import + * rename progress bars signals and receivers + * allow hosts with no tags + * add imports on same lines + * fix order on zypper repos + * add awk and additional rh cdn urls + * use ID_LIKE for suse os detection + * update TODO + * allow errata fuzzy date-matching + * standardize errata progressbar text + * rename function + * add yum errata checking to defaults + * shortern more bugzilla urls + * add os filter for errata + * update os templates + * explicitely install awk + * refactor report processing + * add cdn-ubi redhat url + * improve mirror checksum handling + * refactor and improve repo type handling + * update .gitignore + * add migration for blank host tags + * add support for parsing yum updateinfo errata repos + * use more descriptive variable name + * remove transaction.atomic where not needed + * show url for authenticated mirrors + * add option to specify errata type on command line + * standardize model options format + * remove unneeded saves + * remove unneeded DatabaseError exception catching + * use existing function for module creation + * add a default gentoo repo for all gentoo hosts + * function import at beginning of function + * don't allow duplicate erratum references + * move function to create host from report + * add helper functions to create osrelease and osvariants + * add cli option to remove package duplicates + * add more info to cve template + * add transaction.atomic for package creation + * move reference objects under security + * default to 25 ProcessPoolWorkers + * add incorrect erratum type to error output + * fix incorrect variable + * update first.org link + * add NIST CVE data download for CVSS scores and references + * flake8 + * prefer fetch over download + * don't create osvariant with no arch + * handle cvss v2 scores better + * use upstream nvd cpe names + * fix metalink detection + * remove square brackets from urls + * use python etree for centos + * don't create all-arch packages + * add support for Amazon Linux 1 + * move smaller fixup url functions + * use reverse-date ordering for cves and errata + * better cve mismatch message + * handle irregularly-named CVEs + * get debian DSCs from package file maps + * improve get_matching_packages + * use package-specific display options + * add concurrent processing for yum repo errata + * additional options for yum repo errata handling + * remove errata view ordering + * use arch_id for packages + * add distro-specific references to errata + * don't use acronym, keep flake8 happy + * update gentoo repo handling + * add errata affected and fixed packages + * tweak yum reference handling + * add osv.dev reference to errata detail page + * make packagestring distro-specific handling match package + * add host errata migration + * bump to latest 4.2 django + * updates for cvss and cve handling + * allow search to keep existing filters + * fix up rpm/deb package builds + * add missing dependencies + + [ dependabot[bot] ] + * Bump djangorestframework from 3.13.1 to 3.15.2 + * packaging updates + * Update __init__.py + * Update __init__.py + * remove dh-systemd build dependency + * add missing module __init__s + * debian packaging updates + * fix db permissions for celery + * fix rpm building + * remove unnessecary globals to appease flake8 + * remove unused cve title field + * handle existing duplicate packages in get_or_create + * speed up errata processing + * correctly parse newer dsc representation + * handle subprocesses closing db connections + * handle duplicate CVSSes better + * reduce max charfield length for mysql + * further reduce charfield size for mysql + * reduce URLField max_length to 765 + + [ dependabot[bot] ] + * Bump django from 4.2.20 to 4.2.21 + + [ dependabot[bot] ] + * Bump django from 4.2.21 to 4.2.22 + + [ dependabot[bot] ] + * Bump requests from 2.32.3 to 2.32.4 + + [ vtalos ] + * Remove unused dependency 'chardet' from requirements.txt + + [ Marcus Furlong ] + * get_or_create_module only returns module + + [ dependabot[bot] ] + * Bump django from 4.2.22 to 4.2.24 + + [ Will Furnell ] + * Package types are in the Package class + + [ dependabot[bot] ] + * Bump django from 4.2.24 to 4.2.25 + + [ Marcus Furlong ] + * bump redis + * Update license in common.py + * fix licenses + * use GPL-3.0-only for debian copyright + * fix tag handling + * fix some flake8-bugbear bugs + * fix package filter list for errata + * add support for zstd compression in deb and rpm repos + * simplify logging + * use redis for caching and use locks for tasks + * add errata source options to config file + * remove daily cronjob in favour of patchman-celery + * add isort check + + [ dependabot[bot] ] + * Bump django from 4.2.25 to 4.2.26 + + [ dependabot[bot] ] + * Bump django from 4.2.26 to 4.2.27 + + [ Ricardo Jerónimo ] + * Modified tag handling to preserve case + + [ Marcus Furlong ] + * fix same module in different repos + * add priority queues for tasks (#724) + * update celery services handling (#726) + * remove non-present middleware (#729) + * fix wsgi so rpm module is only loaded once (#728) + * give systemd units usable defaults (#727) + * use consistent users/groups on rhel/debian (#730) + * fixes for dumping/loading fixtures from sqlite (#731) + * update logging to log to console and celery systemd units (#732) + * switch to django-tables2 + * add django2-select2 functionality + * update host and package templates + * prefer triangles over chevrons + * fix table column alignments and cell contents + * fix OS tables and add dropdown menu + * fix broken rocky links + * move navbar to side + * fix datetime issues using timezone + * add collapsible sidebar + * table tweaks + * add more generic approach to handling redhat references + + [ dependabot[bot] ] + * Bump django-select2 from 8.3.0 to 8.4.1 + * rest api report uploads + * api tests + * update tests + * update README + * update installation instructions + * update installation instructions + * auto-commit to update version skip-checks: true + + -- Marcus Furlong Mon, 02 Feb 2026 04:34:46 +0000 + patchman (3.0.19-1) stable; urgency=medium * switch default branch from master to main From 66e9885bc4bfcca107f65a8dfdef69ddaa825ca9 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Mon, 2 Feb 2026 04:53:24 +0000 Subject: [PATCH 066/146] update installation docs --- INSTALL.md | 9 +++------ patchman/settings.py | 8 ++++---- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/INSTALL.md b/INSTALL.md index 41aef419..74ed8091 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -57,17 +57,14 @@ patchman-manage createsuperuser ### virtualenv + pip -TBD - not working yet - ```shell -apt -y install gcc libxml2-dev libxslt1-dev virtualenv python3-dev zlib1g-dev # (debian/ubuntu) -dnf -y install gcc libxml2-devel libxslt-devel python3-virtualenv # (rocky/alma/redhat) +apt -y install python3-venv # (debian/ubuntu) +dnf -y install python3-virtualenv # (rocky/alma/redhat) mkdir /srv/patchman cd /srv/patchman python3 -m venv .venv . .venv/bin/activate -pip install --upgrade pip -pip install patchman gunicorn whitenoise==3.3.1 +pip install patchman gunicorn whitenoise patchman-manage migrate patchman-manage createsuperuser gunicorn patchman.wsgi -b 0.0.0.0:80 diff --git a/patchman/settings.py b/patchman/settings.py index 20ac82e8..c750524f 100644 --- a/patchman/settings.py +++ b/patchman/settings.py @@ -148,12 +148,12 @@ # if sys.prefix + conf_path doesn't exist, try ./etc/patchman (source) if not os.path.isdir(conf_path): conf_path = './etc/patchman' - # if ./etc/patchman doesn't exist, try site.getsitepackages() (pip) + # if ./etc/patchman doesn't exist, try site.getsitepackages() (pip/new-style virtualenv) if not os.path.isdir(conf_path): try: - sitepackages = site.getsitepackages() - except AttributeError: - # virtualenv, try site-packages in sys.path + sitepackages = site.getsitepackages()[0] + except (AttributeError, IndexError): + # old-style virtualenv, try site-packages in sys.path sp = 'site-packages' sitepackages = [s for s in sys.path if s.endswith(sp)][0] conf_path = os.path.join(sitepackages, 'etc/patchman') From 547415552b189915ca50c299ae510c1537473437 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Mon, 2 Feb 2026 04:55:08 +0000 Subject: [PATCH 067/146] auto-commit to update version skip-checks: true --- VERSION.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION.txt b/VERSION.txt index fcdb2e10..1454f6ed 100644 --- a/VERSION.txt +++ b/VERSION.txt @@ -1 +1 @@ -4.0.0 +4.0.1 From 2253cece298c505699b703359312d72f30dd75e6 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Mon, 2 Feb 2026 04:55:10 +0000 Subject: [PATCH 068/146] auto-commit to update debian changelog skip-checks: true --- debian/changelog | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/debian/changelog b/debian/changelog index efc0eb49..4e574791 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,10 @@ +patchman (4.0.1-1) stable; urgency=medium + + * update installation docs + * auto-commit to update version skip-checks: true + + -- Marcus Furlong Mon, 02 Feb 2026 04:55:10 +0000 + patchman (4.0.0-1) stable; urgency=medium [ grumo35 ] From 403ac394c0cab91d780bf14e47f5e9cb80f38340 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Mon, 2 Feb 2026 05:01:20 +0000 Subject: [PATCH 069/146] clarify that only the rhel server is missing packages --- INSTALL.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/INSTALL.md b/INSTALL.md index 74ed8091..0d94733b 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -34,7 +34,8 @@ patchman-manage createsuperuser ### Rocky 10 -Currently broken due to missing upstream packages: https://github.com/furlongm/patchman/issues/669 +Server installation is currently broken due to missing upstream packages: https://github.com/furlongm/patchman/issues/669 +Client installation should work as expected. This also applies to Alma, RHEL, etc. @@ -50,7 +51,8 @@ gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-openbytes EOF dnf -y install epel-release dnf makecache -dnf -y install patchman patchman-client +dnf -y install patchman-client +#dnf -y install patchman systemctl restart httpd patchman-manage createsuperuser ``` From 138b43b5fa1a521f92edf2772e500daeafad51ec Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Mon, 2 Feb 2026 20:52:30 -0500 Subject: [PATCH 070/146] don't lowercase versions --- reports/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reports/utils.py b/reports/utils.py index 85cf6e3b..c68eac83 100644 --- a/reports/utils.py +++ b/reports/utils.py @@ -153,7 +153,7 @@ def parse_updates(updates_string, security): specifying whether it is a security update or not """ updates = {} - ulist = updates_string.lower().split() + ulist = updates_string..split() while ulist: name = f'{ulist[0]} {ulist[1]} {ulist[2]}\n' del ulist[:3] From 05e08b7baa8c66cb859cf9083db280c3bdfc7d52 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Tue, 3 Feb 2026 01:53:39 +0000 Subject: [PATCH 071/146] auto-commit to update version skip-checks: true --- VERSION.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION.txt b/VERSION.txt index 1454f6ed..4d54dadd 100644 --- a/VERSION.txt +++ b/VERSION.txt @@ -1 +1 @@ -4.0.1 +4.0.2 From f8e79e04d8cf85a07454bc09f74cca4e20f436c9 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Tue, 3 Feb 2026 01:53:40 +0000 Subject: [PATCH 072/146] auto-commit to update debian changelog skip-checks: true --- debian/changelog | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/debian/changelog b/debian/changelog index 4e574791..0de8f06c 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,11 @@ +patchman (4.0.2-1) stable; urgency=medium + + * clarify that only the rhel server is missing packages + * don't lowercase versions + * auto-commit to update version skip-checks: true + + -- Marcus Furlong Tue, 03 Feb 2026 01:53:40 +0000 + patchman (4.0.1-1) stable; urgency=medium * update installation docs From 52248dd322187eaceb513fe6abc405efe79e6665 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Mon, 2 Feb 2026 21:10:01 -0500 Subject: [PATCH 073/146] add further host API filtering --- hosts/views.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/hosts/views.py b/hosts/views.py index b50f021c..310a16da 100644 --- a/hosts/views.py +++ b/hosts/views.py @@ -20,6 +20,7 @@ from django.db.models import Count, Q from django.shortcuts import get_object_or_404, redirect, render from django.urls import reverse +from django_filters import rest_framework as filters from django_tables2 import RequestConfig from rest_framework import viewsets from taggit.models import Tag @@ -279,13 +280,26 @@ def host_bulk_action(request): return redirect('hosts:host_list') +class HostFilter(filters.FilterSet): + package_id = filters.NumberFilter(field_name='packages', lookup_expr='exact') + package_name = filters.CharFilter(field_name='packages__name__name', lookup_expr='exact') + package_version = filters.CharFilter(field_name='packages__version', lookup_expr='exact') + package_release = filters.CharFilter(field_name='packages__release', lookup_expr='exact') + package_epoch = filters.CharFilter(field_name='packages__epoch', lookup_expr='exact') + package_arch = filters.CharFilter(field_name='packages__arch__name', lookup_expr='exact') + + class Meta: + model = Host + fields = ['hostname'] + + class HostViewSet(viewsets.ModelViewSet): """ API endpoint that allows hosts to be viewed or edited. """ queryset = Host.objects.all() serializer_class = HostSerializer - filterset_fields = ['hostname'] + filterset_class = HostFilter class HostRepoViewSet(viewsets.ModelViewSet): From 03eb0cc62cf333a777a8b9dce5f800ac84214f29 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Tue, 3 Feb 2026 02:11:28 +0000 Subject: [PATCH 074/146] auto-commit to update version skip-checks: true --- VERSION.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION.txt b/VERSION.txt index 4d54dadd..c4e41f94 100644 --- a/VERSION.txt +++ b/VERSION.txt @@ -1 +1 @@ -4.0.2 +4.0.3 From f516197d928b7d6a61fc32bf85731cca49c85595 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Tue, 3 Feb 2026 02:11:29 +0000 Subject: [PATCH 075/146] auto-commit to update debian changelog skip-checks: true --- debian/changelog | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/debian/changelog b/debian/changelog index 0de8f06c..963dad86 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,10 @@ +patchman (4.0.3-1) stable; urgency=medium + + * add further host API filtering + * auto-commit to update version skip-checks: true + + -- Marcus Furlong Tue, 03 Feb 2026 02:11:29 +0000 + patchman (4.0.2-1) stable; urgency=medium * clarify that only the rhel server is missing packages From 3e67a23c3e6ed6de40301f4730bf6382709695ab Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Mon, 2 Feb 2026 21:18:08 -0500 Subject: [PATCH 076/146] allow filtering hosts by tag --- hosts/views.py | 1 + 1 file changed, 1 insertion(+) diff --git a/hosts/views.py b/hosts/views.py index 310a16da..259cef57 100644 --- a/hosts/views.py +++ b/hosts/views.py @@ -287,6 +287,7 @@ class HostFilter(filters.FilterSet): package_release = filters.CharFilter(field_name='packages__release', lookup_expr='exact') package_epoch = filters.CharFilter(field_name='packages__epoch', lookup_expr='exact') package_arch = filters.CharFilter(field_name='packages__arch__name', lookup_expr='exact') + tag = filters.CharFilter(field_name='tags__name', lookup_expr='exact') class Meta: model = Host From c641cacc0ba5fd53b32d3a22c141e621641ce035 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Mon, 2 Feb 2026 21:20:52 -0500 Subject: [PATCH 077/146] fix typo in splitting --- reports/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reports/utils.py b/reports/utils.py index c68eac83..7290c18d 100644 --- a/reports/utils.py +++ b/reports/utils.py @@ -153,7 +153,7 @@ def parse_updates(updates_string, security): specifying whether it is a security update or not """ updates = {} - ulist = updates_string..split() + ulist = updates_string.split() while ulist: name = f'{ulist[0]} {ulist[1]} {ulist[2]}\n' del ulist[:3] From 1529470f4149255251c1d32e8dfe511250a95304 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Mon, 2 Feb 2026 21:45:39 -0500 Subject: [PATCH 078/146] add version to ui --- patchman/settings.py | 1 + patchman/static/css/base.css | 1 + util/context_processors.py | 55 ++++++++++++++++++++++++++++++++++++ util/templates/navbar.html | 1 + 4 files changed, 58 insertions(+) diff --git a/patchman/settings.py b/patchman/settings.py index c750524f..85792cc9 100644 --- a/patchman/settings.py +++ b/patchman/settings.py @@ -49,6 +49,7 @@ 'django.template.context_processors.tz', 'django.contrib.messages.context_processors.messages', 'util.context_processors.issues_count', + 'util.context_processors.patchman_version', ], 'debug': DEBUG, }, diff --git a/patchman/static/css/base.css b/patchman/static/css/base.css index 86927756..4c8ca791 100644 --- a/patchman/static/css/base.css +++ b/patchman/static/css/base.css @@ -31,6 +31,7 @@ td.truncate-cell a { display: block; overflow: hidden; text-overflow: ellipsis; .sidebar-header { padding: 15px; border-bottom: 1px solid #333; } .sidebar-brand { color: #9d9d9d; font-size: 18px; font-weight: bold; text-decoration: none; white-space: nowrap; } .sidebar-brand:hover { color: #fff; text-decoration: none; } +.sidebar-version { color: #9d9d9d; font-size: 11px; margin-top: 5px; white-space: nowrap; } .sidebar-toggle { background: none; border: none; color: #666; cursor: pointer; padding: 0 10px 0 0; font-size: 14px; } .sidebar-toggle:hover { color: #333; } .sidebar-toggle .glyphicon { transition: transform 0.2s; } diff --git a/util/context_processors.py b/util/context_processors.py index ca3fa427..72aa3cf5 100644 --- a/util/context_processors.py +++ b/util/context_processors.py @@ -26,6 +26,61 @@ from util import get_setting_of_type +def _get_git_ref(): + """Get current git ref if in a git repo.""" + git_dir = Path(__file__).parent.parent / '.git' + if not git_dir.exists(): + return None + try: + result = subprocess.run( + ['git', 'rev-parse', '--short', 'HEAD'], + capture_output=True, + text=True, + timeout=5, + cwd=git_dir.parent, + ) + if result.returncode == 0: + return result.stdout.strip() + except (subprocess.TimeoutExpired, FileNotFoundError): + pass + return None + + +def _get_version(): + """Get version from package metadata or VERSION.txt.""" + # Try importlib.metadata first (for installed packages) + try: + from importlib.metadata import version + return version('patchman') + except Exception: + pass + + # Fall back to VERSION.txt (for development) + version_file = Path(__file__).parent.parent / 'VERSION.txt' + try: + return version_file.read_text().strip() + except FileNotFoundError: + return 'unknown' + + +# Cache version info at module load time (once per process) +_PATCHMAN_VERSION = _get_version() +_PATCHMAN_GIT_REF = _get_git_ref() +if _PATCHMAN_GIT_REF: + _PATCHMAN_VERSION_DISPLAY = f'v{_PATCHMAN_VERSION} ({_PATCHMAN_GIT_REF})' +else: + _PATCHMAN_VERSION_DISPLAY = f'v{_PATCHMAN_VERSION}' + + +def patchman_version(request): + """Context processor to provide version info for navbar.""" + return { + 'patchman_version': _PATCHMAN_VERSION, + 'patchman_git_ref': _PATCHMAN_GIT_REF, + 'patchman_version_display': _PATCHMAN_VERSION_DISPLAY, + } + + def issues_count(request): """Context processor to provide issues count for navbar.""" if not request.user.is_authenticated: diff --git a/util/templates/navbar.html b/util/templates/navbar.html index 5e7d233e..263be6a4 100644 --- a/util/templates/navbar.html +++ b/util/templates/navbar.html @@ -1,6 +1,7 @@
    Name {{ osvariant.name }}
    Architecture {{ osvariant.arch }}
    Codename {{ osvariant.codename }}
    Hosts{{ osvariant.host_set.count }}
    Hosts{{ osvariant.hosts_count }}
    OS Release{% if osvariant.osrelease != None %} {{ osvariant.osrelease }} {% else %}No OS Release{% endif %}
    {% if user.is_authenticated and perms.is_admin %} diff --git a/operatingsystems/views.py b/operatingsystems/views.py index 91b273aa..063a231c 100644 --- a/operatingsystems/views.py +++ b/operatingsystems/views.py @@ -40,7 +40,7 @@ def _get_filtered_osvariants(filter_params): from urllib.parse import parse_qs params = parse_qs(filter_params) - osvariants = OSVariant.objects.select_related() + osvariants = OSVariant.objects.select_related('osrelease', 'arch') if 'osrelease_id' in params: osvariants = osvariants.filter(osrelease=params['osrelease_id'][0]) @@ -60,7 +60,7 @@ def _get_filtered_osreleases(filter_params): from urllib.parse import parse_qs params = parse_qs(filter_params) - osreleases = OSRelease.objects.select_related() + osreleases = OSRelease.objects.all() if 'erratum_id' in params: osreleases = osreleases.filter(erratum=params['erratum_id'][0]) @@ -78,7 +78,7 @@ def _get_filtered_osreleases(filter_params): @login_required def osvariant_list(request): # Use cached hosts_count instead of expensive annotation - osvariants = OSVariant.objects.select_related().annotate( + osvariants = OSVariant.objects.select_related('osrelease', 'arch').annotate( repos_count=Count('osrelease__repos'), ) @@ -182,7 +182,7 @@ def delete_nohost_osvariants(request): @login_required def osrelease_list(request): - osreleases = OSRelease.objects.select_related() + osreleases = OSRelease.objects.all() if 'erratum_id' in request.GET: osreleases = osreleases.filter(erratum=request.GET['erratum_id']) @@ -347,7 +347,7 @@ class OSVariantViewSet(viewsets.ModelViewSet): """ API endpoint that allows operating system variants to be viewed or edited. """ - queryset = OSVariant.objects.all() + queryset = OSVariant.objects.select_related('osrelease', 'arch').all() serializer_class = OSVariantSerializer filterset_fields = ['name'] diff --git a/packages/managers.py b/packages/managers.py index c268f5cf..3a64e7ab 100644 --- a/packages/managers.py +++ b/packages/managers.py @@ -20,4 +20,4 @@ class PackageManager(models.Manager): def get_queryset(self): - return super().get_queryset().select_related() + return super().get_queryset().select_related('name', 'arch') diff --git a/packages/utils.py b/packages/utils.py index 87395ff6..619407d2 100644 --- a/packages/utils.py +++ b/packages/utils.py @@ -281,7 +281,7 @@ def clean_packageupdates(): """ package_updates = list(PackageUpdate.objects.all()) for update in package_updates: - if update.host_set.count() == 0: + if not update.host_set.exists(): text = f'Removing unused PackageUpdate {update}' info_message(text=text) update.delete() @@ -325,7 +325,8 @@ def clean_packages(remove_duplicates=False): packagetype=package.packagetype, category=package.category, ) - if potential_duplicates.count() > 1: + potential_duplicates = list(potential_duplicates) + if len(potential_duplicates) > 1: for dupe in potential_duplicates: if dupe.id != package.id: info_message(text=f'Removing duplicate Package {dupe}') diff --git a/packages/views.py b/packages/views.py index 287c033d..9f75b415 100644 --- a/packages/views.py +++ b/packages/views.py @@ -32,7 +32,7 @@ @login_required def package_list(request): - packages = Package.objects.select_related() + packages = Package.objects.select_related('name', 'arch') if 'arch_id' in request.GET: packages = packages.filter(arch=request.GET['arch_id']).distinct() @@ -121,7 +121,7 @@ def package_list(request): @login_required def package_name_list(request): - packages = PackageName.objects.select_related() + packages = PackageName.objects.all() if 'arch_id' in request.GET: packages = packages.filter(package__arch=request.GET['arch_id']).distinct() @@ -165,7 +165,7 @@ def package_detail(request, package_id): @login_required def package_name_detail(request, packagename): package = get_object_or_404(PackageName, name=packagename) - allversions = Package.objects.select_related().filter(name=package.id) + allversions = Package.objects.select_related('name', 'arch').filter(name=package.id) return render(request, 'packages/package_name_detail.html', {'package': package, @@ -185,7 +185,7 @@ class PackageViewSet(viewsets.ModelViewSet): """ API endpoint that allows packages to be viewed or edited. """ - queryset = Package.objects.all() + queryset = Package.objects.select_related('name', 'arch').all() serializer_class = PackageSerializer filterset_fields = [ 'name', @@ -201,6 +201,6 @@ class PackageUpdateViewSet(viewsets.ModelViewSet): """ API endpoint that allows packages updates to be viewed or edited. """ - queryset = PackageUpdate.objects.all() + queryset = PackageUpdate.objects.select_related('oldpackage', 'newpackage').all() serializer_class = PackageUpdateSerializer filterset_fields = ['oldpackage', 'newpackage', 'security'] diff --git a/reports/views.py b/reports/views.py index 18f60aa6..1aa23913 100644 --- a/reports/views.py +++ b/reports/views.py @@ -45,7 +45,7 @@ def _get_filtered_reports(filter_params): from urllib.parse import parse_qs params = parse_qs(filter_params) - reports = Report.objects.select_related() + reports = Report.objects.all() if 'host_id' in params: reports = reports.filter(hostname=params['host_id'][0]) @@ -108,7 +108,7 @@ def upload(request): @login_required def report_list(request): - reports = Report.objects.select_related() + reports = Report.objects.all() if 'host_id' in request.GET: reports = reports.filter(hostname=request.GET['host_id']) diff --git a/repos/forms.py b/repos/forms.py index f9795b51..57ebae2a 100644 --- a/repos/forms.py +++ b/repos/forms.py @@ -28,7 +28,7 @@ class MirrorSelect2Widget(ModelSelect2MultipleWidget): model = Mirror search_fields = ['url__icontains', 'repo__name__icontains'] max_results = 50 - queryset = Mirror.objects.select_related().order_by('repo__name', 'url') + queryset = Mirror.objects.select_related('repo').order_by('repo__name', 'url') def __init__(self, *args, **kwargs): kwargs.setdefault('attrs', {}) @@ -41,7 +41,7 @@ def label_from_instance(self, obj): class EditRepoForm(ModelForm): mirrors = ModelMultipleChoiceField( - queryset=Mirror.objects.select_related().order_by('repo__name', 'url'), + queryset=Mirror.objects.select_related('repo').order_by('repo__name', 'url'), required=False, widget=MirrorSelect2Widget(attrs={'style': 'width: 100%'}), ) diff --git a/repos/managers.py b/repos/managers.py index 78f37a46..739c7ca3 100644 --- a/repos/managers.py +++ b/repos/managers.py @@ -20,4 +20,4 @@ class RepositoryManager(models.Manager): def get_queryset(self): - return super().get_queryset().select_related() + return super().get_queryset().select_related('arch') diff --git a/repos/models.py b/repos/models.py index 421bca00..eec5f566 100644 --- a/repos/models.py +++ b/repos/models.py @@ -82,11 +82,11 @@ def refresh(self, force=False): force can be set to force a reset of all the mirrors metadata """ if force: - for mirror in self.mirror_set.all(): - mirror.packages_checksum = None - mirror.modules_checksum = None - mirror.errata_checksum = None - mirror.save() + self.mirror_set.all().update( + packages_checksum=None, + modules_checksum=None, + errata_checksum=None + ) if not self.auth_required: if self.repotype == Repository.DEB: @@ -108,9 +108,7 @@ def refresh_errata(self, force=False): """ Refresh errata metadata for all of a repos mirrors """ if force: - for mirror in self.mirror_set.all(): - mirror.errata_checksum = None - mirror.save() + self.mirror_set.all().update(errata_checksum=None) if self.repotype == Repository.RPM: refresh_repo_errata(self) @@ -120,10 +118,7 @@ def disable(self): each mirror so that it doesn't try to update its package metadata. """ self.enabled = False - for mirror in self.mirror_set.all(): - mirror.enabled = False - mirror.refresh = False - mirror.save() + self.mirror_set.all().update(enabled=False, refresh=False) def enable(self): """ Enable a repo. This involves enabling each mirror, which allows it @@ -131,10 +126,7 @@ def enable(self): mirror so that it updates its package metadata. """ self.enabled = True - for mirror in self.mirror_set.all(): - mirror.enabled = True - mirror.refresh = True - mirror.save() + self.mirror_set.all().update(enabled=True, refresh=True) class Mirror(models.Model): diff --git a/repos/utils.py b/repos/utils.py index 13cee149..0d81eb25 100644 --- a/repos/utils.py +++ b/repos/utils.py @@ -273,11 +273,12 @@ def find_best_repo(package, hostrepos): repo. Returns the best repo. """ best_repo = None - package_repos = hostrepos.filter(repo__mirror__packages=package).distinct() + package_repos = hostrepos.filter(repo__mirror__packages=package).select_related('repo').distinct() + package_repos = list(package_repos) if package_repos: best_repo = package_repos[0] - if package_repos.count() > 1: + if len(package_repos) > 1: for hostrepo in package_repos: if hostrepo.repo.security: best_repo = hostrepo diff --git a/repos/views.py b/repos/views.py index 58a9cdd9..4f6869ae 100644 --- a/repos/views.py +++ b/repos/views.py @@ -43,7 +43,7 @@ @login_required def repo_list(request): - repos = Repository.objects.select_related().order_by('name') + repos = Repository.objects.select_related('arch').order_by('name') if 'repotype' in request.GET: repos = repos.filter(repotype=request.GET['repotype']) @@ -146,11 +146,11 @@ def move_mirrors(repo): hostrepo.delete() mirror.repo = repo mirror.save() - if oldrepo.mirror_set.count() == 0: + if not oldrepo.mirror_set.exists(): oldrepo.delete() # Use cached packages_count instead of expensive annotation - mirrors = Mirror.objects.select_related().order_by('packages_checksum') + mirrors = Mirror.objects.select_related('repo').order_by('packages_checksum') checksum = None if 'checksum' in request.GET: @@ -318,9 +318,7 @@ def repo_edit(request, repo_id): repo = edit_form.save() repo.save() mirrors = edit_form.cleaned_data['mirrors'] - for mirror in mirrors: - mirror.repo = repo - mirror.save() + mirrors.update(repo=repo) if repo.enabled: repo.enable() else: @@ -418,7 +416,7 @@ def _get_filtered_repos(filter_params): from urllib.parse import parse_qs params = parse_qs(filter_params) - repos = Repository.objects.select_related().order_by('name') + repos = Repository.objects.select_related('arch').order_by('name') if 'repotype' in params: repos = repos.filter(repotype=params['repotype'][0]) @@ -509,7 +507,7 @@ def _get_filtered_mirrors(filter_params): from urllib.parse import parse_qs params = parse_qs(filter_params) - mirrors = Mirror.objects.select_related().order_by('packages_checksum') + mirrors = Mirror.objects.select_related('repo').order_by('packages_checksum') if 'checksum' in params: mirrors = mirrors.filter(packages_checksum=params['checksum'][0]) @@ -591,7 +589,7 @@ class RepositoryViewSet(viewsets.ModelViewSet): """ API endpoint that allows repositories to be viewed or edited. """ - queryset = Repository.objects.all() + queryset = Repository.objects.select_related('arch').all() serializer_class = RepositorySerializer @@ -599,7 +597,7 @@ class MirrorViewSet(viewsets.ModelViewSet): """ API endpoint that allows mirrors to be viewed or edited. """ - queryset = Mirror.objects.all() + queryset = Mirror.objects.select_related('repo').all() serializer_class = MirrorSerializer @@ -607,5 +605,5 @@ class MirrorPackageViewSet(viewsets.ModelViewSet): """ API endpoint that allows mirror packages to be viewed or edited. """ - queryset = MirrorPackage.objects.all() + queryset = MirrorPackage.objects.select_related('mirror', 'package').all() serializer_class = MirrorPackageSerializer diff --git a/security/managers.py b/security/managers.py index 4dfcffaf..f8d054a1 100644 --- a/security/managers.py +++ b/security/managers.py @@ -19,4 +19,4 @@ class CVEManager(models.Manager): def get_queryset(self): - return super().get_queryset().select_related() + return super().get_queryset() diff --git a/security/views.py b/security/views.py index ae56a82b..7ae5d851 100644 --- a/security/views.py +++ b/security/views.py @@ -32,7 +32,7 @@ @login_required def cwe_list(request): - cwes = CWE.objects.select_related() + cwes = CWE.objects.all() if 'search' in request.GET: terms = request.GET['search'].lower() @@ -65,7 +65,7 @@ def cwe_detail(request, cwe_id): @login_required def cve_list(request): - cves = CVE.objects.select_related() + cves = CVE.objects.all() if 'erratum_id' in request.GET: cves = cves.filter(erratum=request.GET['erratum_id']) @@ -117,7 +117,7 @@ def cve_detail(request, cve_id): @login_required def reference_list(request): - refs = Reference.objects.select_related().order_by('ref_type') + refs = Reference.objects.all().order_by('ref_type') if 'ref_type' in request.GET: refs = refs.filter(ref_type=request.GET['ref_type']).distinct() diff --git a/util/management/commands/revoke_api_key.py b/util/management/commands/revoke_api_key.py index d22955e7..05985c48 100644 --- a/util/management/commands/revoke_api_key.py +++ b/util/management/commands/revoke_api_key.py @@ -42,7 +42,7 @@ def handle(self, *args, **options): if not api_keys.exists(): api_keys = APIKey.objects.filter(name=key_input) - if api_keys.count() == 0: + if not api_keys.exists(): raise CommandError(f'No API key found matching: {key_input}') elif api_keys.count() > 1: raise CommandError(f'Multiple keys match "{key_input}". Please be more specific.') diff --git a/util/templatetags/common.py b/util/templatetags/common.py index 0fdeaff1..119e7f34 100644 --- a/util/templatetags/common.py +++ b/util/templatetags/common.py @@ -19,6 +19,7 @@ from urllib.parse import urlencode from django.template import Library +from django.db.models import Sum from django.template.loader import get_template from django.utils import timezone from django.utils.html import format_html @@ -138,7 +139,4 @@ def reports_timedelta(): @register.simple_tag def host_count(osrelease): - host_count = 0 - for osvariant in osrelease.osvariant_set.all(): - host_count += osvariant.host_set.count() - return host_count + return osrelease.osvariant_set.aggregate(total=Sum('hosts_count'))['total'] or 0 From b03bc47537778df1b8c4f1f6daf62fcb6bcf729b Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Fri, 13 Feb 2026 00:30:47 -0500 Subject: [PATCH 113/146] add .iterator() to large queryset loops --- hosts/tasks.py | 6 +++--- sbin/patchman | 6 +++--- security/tasks.py | 4 ++-- security/utils.py | 4 ++-- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/hosts/tasks.py b/hosts/tasks.py index a8632a5d..b9305cf9 100755 --- a/hosts/tasks.py +++ b/hosts/tasks.py @@ -33,7 +33,7 @@ def find_host_updates(host_id): def find_all_host_updates(): """ Task to find updates for all hosts """ - for host in Host.objects.all(): + for host in Host.objects.all().iterator(): find_host_updates.delay(host.id) @@ -43,7 +43,7 @@ def find_all_host_updates_homogenous(): """ updated_hosts = [] ts = get_datetime_now() - for host in Host.objects.all(): + for host in Host.objects.all().iterator(): if host not in updated_hosts: host.find_updates() host.updated_at = ts @@ -61,7 +61,7 @@ def find_all_host_updates_homogenous(): updates = host.updates.all() phosts = [] - for fhost in filtered_hosts: + for fhost in filtered_hosts.iterator(): frepos = set(fhost.repos.all()) if repos != frepos: continue diff --git a/sbin/patchman b/sbin/patchman index 5dabdfa7..0e164c98 100755 --- a/sbin/patchman +++ b/sbin/patchman @@ -164,7 +164,7 @@ def host_updates_alt(host=None): updated_hosts = [] hosts = get_hosts(host, 'Finding updates') ts = get_datetime_now() - for host in hosts: + for host in hosts.iterator(): info_message(text=str(host)) if host not in updated_hosts: host.find_updates() @@ -183,7 +183,7 @@ def host_updates_alt(host=None): updates = host.updates.all() phosts = [] - for fhost in filtered_hosts: + for fhost in filtered_hosts.iterator(): frepos = set(fhost.repos.all()) rdiff = repos.difference(frepos) @@ -352,7 +352,7 @@ def process_reports(host=None, force=False): info_message(text=text) - for report in reports: + for report in reports.iterator(): report.process(find_updates=False) diff --git a/security/tasks.py b/security/tasks.py index 0cfbc2f1..93d48427 100644 --- a/security/tasks.py +++ b/security/tasks.py @@ -50,7 +50,7 @@ def update_cves(): if cache.add(lock_key, 'true', lock_expire): try: - for cve in CVE.objects.all(): + for cve in CVE.objects.all().iterator(): update_cve.delay(cve.id) finally: cache.delete(lock_key) @@ -87,7 +87,7 @@ def update_cwes(): if cache.add(lock_key, 'true', lock_expire): try: - for cwe in CWE.objects.all(): + for cwe in CWE.objects.all().iterator(): update_cwe.delay(cwe.id) finally: cache.delete(lock_key) diff --git a/security/utils.py b/security/utils.py index 127f2c73..745cf1f8 100644 --- a/security/utils.py +++ b/security/utils.py @@ -42,7 +42,7 @@ def update_cves(cve_id=None, fetch_nist_data=False): cve = CVE.objects.get(cve_id=cve_id) cve.fetch_cve_data(fetch_nist_data, sleep_secs=0) else: - for cve in CVE.objects.all(): + for cve in CVE.objects.all().iterator(): cve.fetch_cve_data(fetch_nist_data) @@ -56,7 +56,7 @@ def update_cwes(cve_id=None): cwes = cve.cwes.all() else: cwes = CWE.objects.all() - for cwe in cwes: + for cwe in cwes.iterator(): cwe.fetch_cwe_data() From 9eb83e83ea19f26b9f9eaeee032c9dfda1287df4 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Fri, 13 Feb 2026 00:40:39 -0500 Subject: [PATCH 114/146] address template double-count issues --- errata/templates/errata/erratum_detail.html | 10 +++++---- .../operatingsystems/osrelease_detail.html | 10 +++++---- .../osvariant_delete_multiple.html | 2 +- util/templates/dashboard.html | 8 +++++-- util/views.py | 22 +++++++++++++++++++ 5 files changed, 41 insertions(+), 11 deletions(-) diff --git a/errata/templates/errata/erratum_detail.html b/errata/templates/errata/erratum_detail.html index 2dbfceaf..3e622401 100644 --- a/errata/templates/errata/erratum_detail.html +++ b/errata/templates/errata/erratum_detail.html @@ -8,10 +8,11 @@ {% block content %} +{% with affected_count=erratum.affected_packages.count fixed_count=erratum.fixed_packages.count %}
    @@ -22,8 +23,8 @@ Type {{ erratum.e_type }} Published Date{{ erratum.issue_date|date|default_if_none:'' }} Synopsis {{ erratum.synopsis }} - Packages Affected {{ erratum.affected_packages.count }} - Packages Fixed {{ erratum.fixed_packages.count }} + Packages Affected {{ affected_count }} + Packages Fixed {{ fixed_count }} OS Releases Affected @@ -78,5 +79,6 @@
    +{% endwith %} {% endblock %} diff --git a/operatingsystems/templates/operatingsystems/osrelease_detail.html b/operatingsystems/templates/operatingsystems/osrelease_detail.html index 740b9c4b..be94f43d 100644 --- a/operatingsystems/templates/operatingsystems/osrelease_detail.html +++ b/operatingsystems/templates/operatingsystems/osrelease_detail.html @@ -12,6 +12,7 @@ {% block content %} +{% with osvariant_count=osrelease.osvariant_set.count repos_count=osrelease.repos.count %}
    • Details
    • Variants
      • @@ -26,8 +27,8 @@ Name{{ osrelease.name }} CPE Name{% if osrelease.cpe_name %}{{ osrelease.cpe_name }}{% endif %} Codename{% if osrelease.codename %}{{ osrelease.codename }}{% endif %} - OS Variants{{ osrelease.osvariant_set.count }} - Repositories{{ osrelease.repos.count }} + OS Variants{{ osvariant_count }} + Repositories{{ repos_count }} Hosts{{ host_count }} Errata{{ osrelease.erratum_set.count }} @@ -40,7 +41,7 @@
      - {% if osrelease.osvariant_set.count == 0 %} + {% if osvariant_count == 0 %} {{ osrelease }} has no Variants {% else %} {% gen_table osrelease.osvariant_set.select_related %} @@ -50,7 +51,7 @@
      - {% if osrelease.repos.count == 0 %} + {% if repos_count == 0 %} {{ osrelease }} has no Repositories {% else %} {% gen_table osrelease.repos.select_related %} @@ -68,5 +69,6 @@
      +{% endwith %} {% endblock %} diff --git a/operatingsystems/templates/operatingsystems/osvariant_delete_multiple.html b/operatingsystems/templates/operatingsystems/osvariant_delete_multiple.html index f6d0373d..6887e0a7 100644 --- a/operatingsystems/templates/operatingsystems/osvariant_delete_multiple.html +++ b/operatingsystems/templates/operatingsystems/osvariant_delete_multiple.html @@ -24,7 +24,7 @@ {% for osvariant in osvariants %} {{ osvariant }} - {% if osvariant.host_set.count != None %} {{ osvariant.host_set.count }} {% else %} 0 {% endif %} + {{ osvariant.host_set.count|default:0 }} {% if osvariant.osrelease != None %} {{ osvariant.osrelease }} {% else %}No OS Release{% endif %} {% endfor %} diff --git a/util/templates/dashboard.html b/util/templates/dashboard.html index 5976adcb..7afc18fb 100644 --- a/util/templates/dashboard.html +++ b/util/templates/dashboard.html @@ -13,7 +13,7 @@ {% block content %} {% with count=noosrelease_osvariants.count %} - {% if noosrelease_osvariants.count > 0 %} + {% if count > 0 %}
      @@ -24,7 +24,7 @@ {% endwith %} {% with count=nohost_osvariants.count %} - {% if nohost_osvariants.count > 0 %} + {% if count > 0 %}
      @@ -244,4 +244,8 @@ {% endif %} {% endwith %} +{% if not has_issues %} +
      No issues found!
      +{% endif %} + {% endblock %} diff --git a/util/views.py b/util/views.py index 073188a1..6222546b 100644 --- a/util/views.py +++ b/util/views.py @@ -110,10 +110,32 @@ def dashboard(request): possible_mirrors[checksum] = checksums[checksum] continue + has_issues = ( + noosrelease_osvariants.exists() or + nohost_osvariants.exists() or + (norepo_osreleases is not None and norepo_osreleases.exists()) or + stale_hosts.exists() or + reboot_hosts.exists() or + secupdate_hosts.exists() or + bugupdate_hosts.exists() or + norepo_hosts.exists() or + diff_rdns_hosts.exists() or + failed_mirrors.exists() or + disabled_mirrors.exists() or + norefresh_mirrors.exists() or + failed_repos.exists() or + unused_repos.exists() or + nomirror_repos.exists() or + nohost_repos.exists() or + bool(possible_mirrors) or + norepo_packages.exists() + ) + return render( request, 'dashboard.html', {'site': site, + 'has_issues': has_issues, 'noosrelease_osvariants': noosrelease_osvariants, 'norepo_hosts': norepo_hosts, 'nohost_osvariants': nohost_osvariants, From 5654e1f72d45b4017a5270a90819eea99656bb14 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Fri, 13 Feb 2026 00:58:47 -0500 Subject: [PATCH 115/146] move imports to top-level --- hosts/views.py | 6 +++--- operatingsystems/views.py | 4 ++-- packages/utils.py | 1 + reports/views.py | 32 ++++++++++---------------------- repos/views.py | 7 +++---- util/context_processors.py | 4 ++-- util/templatetags/common.py | 3 +-- 7 files changed, 22 insertions(+), 35 deletions(-) diff --git a/hosts/views.py b/hosts/views.py index f905f8c6..f940c3a4 100644 --- a/hosts/views.py +++ b/hosts/views.py @@ -15,6 +15,8 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see +from urllib.parse import parse_qs + from django.contrib import messages from django.contrib.auth.decorators import login_required from django.db.models import Q @@ -31,6 +33,7 @@ from hosts.models import Host, HostRepo from hosts.serializers import HostRepoSerializer, HostSerializer from hosts.tables import HostTable +from hosts.tasks import find_host_updates from operatingsystems.models import OSRelease, OSVariant from reports.models import Report from util import sanitize_filter_params @@ -39,7 +42,6 @@ def _get_filtered_hosts(filter_params): """Helper to reconstruct filtered queryset from filter params.""" - from urllib.parse import parse_qs params = parse_qs(filter_params) hosts = Host.objects.select_related('osvariant', 'arch', 'domain') @@ -222,7 +224,6 @@ def host_delete(request, hostname): def host_find_updates(request, hostname): """ Find updates using a celery task """ - from hosts.tasks import find_host_updates host = get_object_or_404(Host, hostname=hostname) find_host_updates.delay(host.id) text = f'Finding updates for Host {host}' @@ -261,7 +262,6 @@ def host_bulk_action(request): name = Host._meta.verbose_name if count == 1 else Host._meta.verbose_name_plural if action == 'find_updates': - from hosts.tasks import find_host_updates for host in hosts: find_host_updates.delay(host.id) messages.success(request, f'Queued {count} {name} for update check') diff --git a/operatingsystems/views.py b/operatingsystems/views.py index 063a231c..1a6b3fda 100644 --- a/operatingsystems/views.py +++ b/operatingsystems/views.py @@ -15,6 +15,8 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see +from urllib.parse import parse_qs + from django.contrib import messages from django.contrib.auth.decorators import login_required from django.db.models import Count, Q @@ -37,7 +39,6 @@ def _get_filtered_osvariants(filter_params): """Helper to reconstruct filtered queryset from filter params.""" - from urllib.parse import parse_qs params = parse_qs(filter_params) osvariants = OSVariant.objects.select_related('osrelease', 'arch') @@ -57,7 +58,6 @@ def _get_filtered_osvariants(filter_params): def _get_filtered_osreleases(filter_params): """Helper to reconstruct filtered queryset from filter params.""" - from urllib.parse import parse_qs params = parse_qs(filter_params) osreleases = OSRelease.objects.all() diff --git a/packages/utils.py b/packages/utils.py index 619407d2..23e522e5 100644 --- a/packages/utils.py +++ b/packages/utils.py @@ -19,6 +19,7 @@ from django.core.exceptions import MultipleObjectsReturned from django.db import IntegrityError, transaction +from django.db.models import Count, Min from arch.models import PackageArchitecture from packages.models import ( diff --git a/reports/views.py b/reports/views.py index 1aa23913..e5d1d438 100644 --- a/reports/views.py +++ b/reports/views.py @@ -16,8 +16,9 @@ # along with Patchman. If not, see import json -from urllib.parse import unquote +from urllib.parse import parse_qs, unquote +from django.conf import settings from django.contrib import messages from django.contrib.auth.decorators import login_required from django.db.models import Q @@ -28,21 +29,26 @@ from django.views.decorators.csrf import csrf_exempt from django_tables2 import RequestConfig from rest_framework import status, viewsets +from rest_framework.permissions import AllowAny, IsAuthenticatedOrReadOnly from rest_framework.response import Response +from rest_framework_api_key.permissions import HasAPIKey from tenacity import ( retry, retry_if_exception_type, stop_after_attempt, wait_exponential, ) from reports.models import Report -from reports.serializers import ReportUploadSerializer -from reports.tables import ReportTable +from reports.serializers import ReportSerializer, ReportUploadSerializer +from reports.tables import ( + ReportModuleTable, ReportPackageTable, ReportRepoTable, ReportTable, + ReportUpdateTable, +) +from reports.tasks import process_report from util import sanitize_filter_params from util.filterspecs import Filter, FilterBar def _get_filtered_reports(filter_params): """Helper to reconstruct filtered queryset from filter params.""" - from urllib.parse import parse_qs params = parse_qs(filter_params) reports = Report.objects.all() @@ -78,7 +84,6 @@ def upload(request): report = Report.objects.create() report.parse(data, meta) - from reports.tasks import process_report process_report.delay(report.id) if 'report' in data and data['report'] == 'true': @@ -159,10 +164,6 @@ def report_detail(request, report_id): # Add tables for Protocol 2 reports if report.protocol == '2': - from reports.tables import ( - ReportModuleTable, ReportPackageTable, ReportRepoTable, - ReportUpdateTable, - ) if report.has_packages: context['packages_table'] = ReportPackageTable(report.packages_parsed) if report.has_repos: @@ -183,7 +184,6 @@ def report_detail(request, report_id): def report_process(request, report_id): """ Process a report using a celery task """ - from reports.tasks import process_report report = get_object_or_404(Report, id=report_id) report.processed = False report.save() @@ -243,7 +243,6 @@ def report_bulk_action(request): name = Report._meta.verbose_name if count == 1 else Report._meta.verbose_name_plural if action == 'process': - from reports.tasks import process_report for report in reports: report.processed = False report.save() @@ -273,12 +272,6 @@ class ReportViewSet(viewsets.ViewSet): """ def get_permissions(self): - from django.conf import settings - from rest_framework.permissions import ( - AllowAny, IsAuthenticatedOrReadOnly, - ) - from rest_framework_api_key.permissions import HasAPIKey - # POST requires API key if configured, otherwise allow any if self.action == 'create': if getattr(settings, 'REQUIRE_API_KEY', False): @@ -289,16 +282,12 @@ def get_permissions(self): def list(self, request): """List all reports.""" - from reports.serializers import ReportSerializer queryset = Report.objects.all().order_by('-created') serializer = ReportSerializer(queryset, many=True, context={'request': request}) return Response(serializer.data) def retrieve(self, request, pk=None): """Retrieve a single report.""" - from django.shortcuts import get_object_or_404 - - from reports.serializers import ReportSerializer report = get_object_or_404(Report, pk=pk) serializer = ReportSerializer(report, context={'request': request}) return Response(serializer.data) @@ -357,7 +346,6 @@ def create(self, request): ) # Queue for async processing - from reports.tasks import process_report process_report.delay(report.id) return Response( diff --git a/repos/views.py b/repos/views.py index 4f6869ae..e49c4972 100644 --- a/repos/views.py +++ b/repos/views.py @@ -15,6 +15,8 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see +from urllib.parse import parse_qs + from django.contrib import messages from django.contrib.auth.decorators import login_required from django.db import IntegrityError @@ -36,6 +38,7 @@ MirrorPackageSerializer, MirrorSerializer, RepositorySerializer, ) from repos.tables import MirrorTable, RepositoryTable +from repos.tasks import refresh_repo from util import sanitize_filter_params from util.filterspecs import Filter, FilterBar @@ -403,7 +406,6 @@ def repo_toggle_security(request, repo_id): def repo_refresh(request, repo_id): """ Refresh a repo using a celery task """ - from repos.tasks import refresh_repo repo = get_object_or_404(Repository, id=repo_id) refresh_repo.delay(repo.id) text = f'Repostory {repo} is being refreshed' @@ -413,7 +415,6 @@ def repo_refresh(request, repo_id): def _get_filtered_repos(filter_params): """Helper to reconstruct filtered queryset from filter params.""" - from urllib.parse import parse_qs params = parse_qs(filter_params) repos = Repository.objects.select_related('arch').order_by('name') @@ -486,7 +487,6 @@ def repo_bulk_action(request): repos.update(security=False) messages.success(request, f'Marked {count} {name} as non-security') elif action == 'refresh': - from repos.tasks import refresh_repo for repo in repos: refresh_repo.delay(repo.id) messages.success(request, f'Queued {count} {name} for refresh') @@ -504,7 +504,6 @@ def repo_bulk_action(request): def _get_filtered_mirrors(filter_params): """Helper to reconstruct filtered queryset from filter params.""" - from urllib.parse import parse_qs params = parse_qs(filter_params) mirrors = Mirror.objects.select_related('repo').order_by('packages_checksum') diff --git a/util/context_processors.py b/util/context_processors.py index a9c46545..97bbcf2c 100644 --- a/util/context_processors.py +++ b/util/context_processors.py @@ -16,6 +16,7 @@ import subprocess from datetime import timedelta +from importlib.metadata import version as get_pkg_version from pathlib import Path from django.db.models import F @@ -52,8 +53,7 @@ def _get_version(): """Get version from package metadata or VERSION.txt.""" # Try importlib.metadata first (for installed packages) try: - from importlib.metadata import version - return version('patchman') + return get_pkg_version('patchman') except Exception: pass diff --git a/util/templatetags/common.py b/util/templatetags/common.py index 119e7f34..dc9f4fb5 100644 --- a/util/templatetags/common.py +++ b/util/templatetags/common.py @@ -18,8 +18,8 @@ from datetime import timedelta from urllib.parse import urlencode -from django.template import Library from django.db.models import Sum +from django.template import Library, engines from django.template.loader import get_template from django.utils import timezone from django.utils.html import format_html @@ -74,7 +74,6 @@ def gen_table(context, object_list, template_name=None): RequestConfig(request, paginate=False).configure(table) # Render using the table's configured template - from django.template import engines django_engine = engines['django'] template = django_engine.from_string('{% load django_tables2 %}{% render_table table %}') return template.render({'table': table, 'request': request}) From 7b95a51bed9bda4e3b51ec800e2c5a0db1146837 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Fri, 13 Feb 2026 01:27:06 -0500 Subject: [PATCH 116/146] =?UTF-8?q?replace=20o(n=C2=B2)=20comparisons=20wi?= =?UTF-8?q?th=20id=20sets=20and=20sql=20aggregation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- hosts/tasks.py | 31 ++++++++++----------- packages/utils.py | 70 +++++++++++++++++++++++++++-------------------- 2 files changed, 54 insertions(+), 47 deletions(-) diff --git a/hosts/tasks.py b/hosts/tasks.py index b9305cf9..71ec203a 100755 --- a/hosts/tasks.py +++ b/hosts/tasks.py @@ -41,10 +41,10 @@ def find_all_host_updates(): def find_all_host_updates_homogenous(): """ Task to find updates for all hosts where hosts are expected to be homogenous """ - updated_hosts = [] + updated_host_ids = set() ts = get_datetime_now() for host in Host.objects.all().iterator(): - if host not in updated_hosts: + if host.id not in updated_host_ids: host.find_updates() host.updated_at = ts host.save() @@ -56,23 +56,20 @@ def find_all_host_updates_homogenous(): # and exclude hosts with the current timestamp filtered_hosts = filtered_hosts.exclude(updated_at=ts) - packages = set(host.packages.all()) - repos = set(host.repos.all()) - updates = host.updates.all() + package_ids = frozenset(host.packages.values_list('id', flat=True)) + repo_ids = frozenset(host.repos.values_list('id', flat=True)) + updates = list(host.updates.all()) - phosts = [] for fhost in filtered_hosts.iterator(): - frepos = set(fhost.repos.all()) - if repos != frepos: + frepo_ids = frozenset(fhost.repos.values_list('id', flat=True)) + if repo_ids != frepo_ids: continue - fpackages = set(fhost.packages.all()) - if packages != fpackages: + fpackage_ids = frozenset(fhost.packages.values_list('id', flat=True)) + if package_ids != fpackage_ids: continue - phosts.append(fhost) - for phost in phosts: - phost.updates.set(updates) - phost.updated_at = ts - phost.save() - updated_hosts.append(phost) - info_message(text=f'Added the same updates to {phost}') + fhost.updates.set(updates) + fhost.updated_at = ts + fhost.save() + updated_host_ids.add(fhost.id) + info_message(text=f'Added the same updates to {fhost}') diff --git a/packages/utils.py b/packages/utils.py index 23e522e5..c7f86b4c 100644 --- a/packages/utils.py +++ b/packages/utils.py @@ -280,21 +280,30 @@ def get_matching_packages(name, epoch, version, release, p_type, arch=None): def clean_packageupdates(): """ Removes PackageUpdate objects that are no longer linked to any hosts """ - package_updates = list(PackageUpdate.objects.all()) - for update in package_updates: - if not update.host_set.exists(): - text = f'Removing unused PackageUpdate {update}' + orphaned = PackageUpdate.objects.filter(host__isnull=True) + for update in orphaned: + text = f'Removing unused PackageUpdate {update}' + info_message(text=text) + update.delete() + + duplicate_updates = PackageUpdate.objects.values( + 'oldpackage', 'newpackage', 'security' + ).annotate(count=Count('id'), keep_id=Min('id')).filter(count__gt=1) + + for update in duplicate_updates: + extra_updates = PackageUpdate.objects.filter( + oldpackage=update['oldpackage'], + newpackage=update['newpackage'], + security=update['security'] + ).exclude(id=update['keep_id']) + keep_update = PackageUpdate.objects.get(id=update['keep_id']) + for extra_update in extra_updates: + text = f'Removing duplicate PackageUpdate: {extra_update}' info_message(text=text) - update.delete() - for duplicate in package_updates: - if update.oldpackage == duplicate.oldpackage and update.newpackage == duplicate.newpackage and \ - update.security == duplicate.security and update.id != duplicate.id: - text = f'Removing duplicate PackageUpdate: {update}' - info_message(text=text) - for host in duplicate.host_set.all(): - host.updates.remove(duplicate) - host.updates.add(update) - duplicate.delete() + for host in extra_update.host_set.all(): + host.updates.remove(extra_update) + host.updates.add(keep_update) + extra_update.delete() def clean_packages(remove_duplicates=False): @@ -316,22 +325,23 @@ def clean_packages(remove_duplicates=False): packages.delete() if remove_duplicates: info_message(text='Checking for duplicate Packages...') - for package in Package.objects.all(): - potential_duplicates = Package.objects.filter( - name=package.name, - arch=package.arch, - epoch=package.epoch, - version=package.version, - release=package.release, - packagetype=package.packagetype, - category=package.category, - ) - potential_duplicates = list(potential_duplicates) - if len(potential_duplicates) > 1: - for dupe in potential_duplicates: - if dupe.id != package.id: - info_message(text=f'Removing duplicate Package {dupe}') - dupe.delete() + duplicates = Package.objects.values( + 'name', 'arch', 'epoch', 'version', 'release', 'packagetype', 'category' + ).annotate(count=Count('id'), keep_id=Min('id')).filter(count__gt=1) + + for dup in duplicates: + to_delete = Package.objects.filter( + name=dup['name'], + arch=dup['arch'], + epoch=dup['epoch'], + version=dup['version'], + release=dup['release'], + packagetype=dup['packagetype'], + category=dup['category'] + ).exclude(id=dup['keep_id']) + for package in to_delete: + info_message(text=f'Removing duplicate Package {package}') + package.delete() def clean_packagenames(): From 93af23d07d50a23b424f3376181bed341bc96036 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Fri, 13 Feb 2026 01:32:10 -0500 Subject: [PATCH 117/146] consolidate and improve homogenous host update logic --- hosts/tasks.py | 36 ++---------------------------------- hosts/utils.py | 46 ++++++++++++++++++++++++++++++++++++++++++++++ sbin/patchman | 49 ++----------------------------------------------- 3 files changed, 50 insertions(+), 81 deletions(-) diff --git a/hosts/tasks.py b/hosts/tasks.py index 71ec203a..d8e5b6ea 100755 --- a/hosts/tasks.py +++ b/hosts/tasks.py @@ -17,8 +17,7 @@ from celery import shared_task from hosts.models import Host -from util import get_datetime_now -from util.logging import info_message +from hosts.utils import find_host_updates_homogenous @shared_task(priority=0) @@ -41,35 +40,4 @@ def find_all_host_updates(): def find_all_host_updates_homogenous(): """ Task to find updates for all hosts where hosts are expected to be homogenous """ - updated_host_ids = set() - ts = get_datetime_now() - for host in Host.objects.all().iterator(): - if host.id not in updated_host_ids: - host.find_updates() - host.updated_at = ts - host.save() - - # only include hosts with the exact same number of packages - filtered_hosts = Host.objects.filter( - packages_count=host.packages_count - ) - # and exclude hosts with the current timestamp - filtered_hosts = filtered_hosts.exclude(updated_at=ts) - - package_ids = frozenset(host.packages.values_list('id', flat=True)) - repo_ids = frozenset(host.repos.values_list('id', flat=True)) - updates = list(host.updates.all()) - - for fhost in filtered_hosts.iterator(): - frepo_ids = frozenset(fhost.repos.values_list('id', flat=True)) - if repo_ids != frepo_ids: - continue - fpackage_ids = frozenset(fhost.packages.values_list('id', flat=True)) - if package_ids != fpackage_ids: - continue - - fhost.updates.set(updates) - fhost.updated_at = ts - fhost.save() - updated_host_ids.add(fhost.id) - info_message(text=f'Added the same updates to {fhost}') + find_host_updates_homogenous(Host.objects.all()) diff --git a/hosts/utils.py b/hosts/utils.py index 44441f9b..513444bf 100644 --- a/hosts/utils.py +++ b/hosts/utils.py @@ -20,6 +20,7 @@ from django.db import IntegrityError, transaction from taggit.models import Tag +from util import get_datetime_now from util.logging import error_message, info_message @@ -76,6 +77,51 @@ def get_or_create_host(report, arch, osvariant, domain): return host +def find_host_updates_homogenous(hosts, verbose=False): + """ Find updates for hosts, copying updates to homogenous hosts. + If a host has the same packages and repos as a previously + processed host, it is given the same updates. + """ + from hosts.models import Host + + updated_host_ids = set() + ts = get_datetime_now() + host_iter = hosts.iterator() if hasattr(hosts, 'iterator') else iter(hosts) + for host in host_iter: + if verbose: + info_message(text=str(host)) + if host.id not in updated_host_ids: + host.find_updates() + if verbose: + info_message(text='') + host.updated_at = ts + host.save() + + filtered_hosts = Host.objects.filter( + packages_count=host.packages_count) + filtered_hosts = filtered_hosts.exclude(updated_at=ts) + + package_ids = frozenset(host.packages.values_list('id', flat=True)) + repo_ids = frozenset(host.repos.values_list('id', flat=True)) + updates = list(host.updates.all()) + + for fhost in filtered_hosts.iterator(): + frepo_ids = frozenset(fhost.repos.values_list('id', flat=True)) + if repo_ids != frepo_ids: + continue + fpackage_ids = frozenset(fhost.packages.values_list('id', flat=True)) + if package_ids != fpackage_ids: + continue + + fhost.updates.set(updates) + fhost.updated_at = ts + fhost.save() + updated_host_ids.add(fhost.id) + info_message(text=f'Added the same updates to {fhost}') + elif verbose: + info_message(text='Updates already added in this run') + + def clean_tags(): """ Delete Tags that have no Host """ diff --git a/sbin/patchman b/sbin/patchman index 0e164c98..ea81801a 100755 --- a/sbin/patchman +++ b/sbin/patchman @@ -36,7 +36,7 @@ from errata.utils import ( scan_package_updates_for_affected_packages, ) from hosts.models import Host -from hosts.utils import clean_tags +from hosts.utils import clean_tags, find_host_updates_homogenous from modules.utils import clean_modules from packages.utils import ( clean_packagenames, clean_packages, clean_packageupdates, @@ -46,7 +46,6 @@ from reports.tasks import remove_reports_with_no_hosts from repos.models import Repository from repos.utils import clean_repos from security.utils import update_cves, update_cwes -from util import get_datetime_now from util.logging import info_message, set_quiet_mode @@ -161,52 +160,8 @@ def clean_reports(hoststr=None): def host_updates_alt(host=None): """ Find updates for all hosts, specify host for a single host """ - updated_hosts = [] hosts = get_hosts(host, 'Finding updates') - ts = get_datetime_now() - for host in hosts.iterator(): - info_message(text=str(host)) - if host not in updated_hosts: - host.find_updates() - info_message(text='') - host.updated_at = ts - host.save() - - # only include hosts with the same number of packages - filtered_hosts = Host.objects.filter( - packages_count=host.packages_count) - # exclude hosts with the current timestamp - filtered_hosts = filtered_hosts.exclude(updated_at=ts) - - packages = set(host.packages.all()) - repos = set(host.repos.all()) - updates = host.updates.all() - - phosts = [] - for fhost in filtered_hosts.iterator(): - - frepos = set(fhost.repos.all()) - rdiff = repos.difference(frepos) - if len(rdiff) != 0: - continue - - fpackages = set(fhost.packages.all()) - pdiff = packages.difference(fpackages) - if len(pdiff) != 0: - continue - - phosts.append(fhost) - - for phost in phosts: - phost.updates.set(updates) - phost.updated_at = ts - phost.save() - updated_hosts.append(phost) - text = f'Added the same updates to {phost}' - info_message(text=text) - else: - text = 'Updates already added in this run' - info_message(text=text) + find_host_updates_homogenous(hosts, verbose=True) def host_updates(host=None): From fe25a12e68ce699148b1f4e7b2f9d416fb4ff6c3 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Fri, 13 Feb 2026 15:15:49 -0500 Subject: [PATCH 118/146] add cached count fields to erratum model with m2m signals --- errata/apps.py | 2 + .../0008_add_cached_count_fields.py | 38 ++++++ .../migrations/0009_backfill_cached_counts.py | 45 +++++++ errata/models.py | 11 +- errata/signals.py | 60 +++++++++ errata/tables.py | 10 +- errata/templates/errata/erratum_detail.html | 2 +- errata/tests/test_models.py | 121 +++++++++++++++++- 8 files changed, 279 insertions(+), 10 deletions(-) create mode 100644 errata/migrations/0008_add_cached_count_fields.py create mode 100644 errata/migrations/0009_backfill_cached_counts.py create mode 100644 errata/signals.py diff --git a/errata/apps.py b/errata/apps.py index 95360240..9512378e 100644 --- a/errata/apps.py +++ b/errata/apps.py @@ -26,6 +26,8 @@ def ready(self): from django.db.models.signals import post_save from django.utils import timezone + import errata.signals # noqa: F401 + def set_initial_last_run(sender, instance, created, **kwargs): if created and instance.name == 'update_errata_cves_cwes_every_12_hours': instance.last_run_at = timezone.now() - timedelta(days=1) diff --git a/errata/migrations/0008_add_cached_count_fields.py b/errata/migrations/0008_add_cached_count_fields.py new file mode 100644 index 00000000..ff9d3bc9 --- /dev/null +++ b/errata/migrations/0008_add_cached_count_fields.py @@ -0,0 +1,38 @@ +# Generated by Django 4.2.28 on 2026-02-13 06:44 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('errata', '0007_alter_erratum_fixed_packages'), + ] + + operations = [ + migrations.AddField( + model_name='erratum', + name='affected_packages_count', + field=models.PositiveIntegerField(default=0), + ), + migrations.AddField( + model_name='erratum', + name='cves_count', + field=models.PositiveIntegerField(default=0), + ), + migrations.AddField( + model_name='erratum', + name='fixed_packages_count', + field=models.PositiveIntegerField(default=0), + ), + migrations.AddField( + model_name='erratum', + name='osreleases_count', + field=models.PositiveIntegerField(default=0), + ), + migrations.AddField( + model_name='erratum', + name='references_count', + field=models.PositiveIntegerField(default=0), + ), + ] diff --git a/errata/migrations/0009_backfill_cached_counts.py b/errata/migrations/0009_backfill_cached_counts.py new file mode 100644 index 00000000..4d084fa8 --- /dev/null +++ b/errata/migrations/0009_backfill_cached_counts.py @@ -0,0 +1,45 @@ +# Copyright 2026 Marcus Furlong +# +# This file is part of Patchman. +# +# Patchman is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, version 3 only. +# +# Patchman is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Patchman. If not, see + +from django.db import migrations + + +def backfill_counts(apps, schema_editor): + Erratum = apps.get_model('errata', 'Erratum') + for erratum in Erratum.objects.all().iterator(): + erratum.affected_packages_count = erratum.affected_packages.count() + erratum.fixed_packages_count = erratum.fixed_packages.count() + erratum.osreleases_count = erratum.osreleases.count() + erratum.cves_count = erratum.cves.count() + erratum.references_count = erratum.references.count() + erratum.save(update_fields=[ + 'affected_packages_count', + 'fixed_packages_count', + 'osreleases_count', + 'cves_count', + 'references_count', + ]) + + +class Migration(migrations.Migration): + + dependencies = [ + ('errata', '0008_add_cached_count_fields'), + ] + + operations = [ + migrations.RunPython(backfill_counts, migrations.RunPython.noop), + ] diff --git a/errata/models.py b/errata/models.py index 8c21bcfa..49a0ef5b 100644 --- a/errata/models.py +++ b/errata/models.py @@ -40,6 +40,11 @@ class Erratum(models.Model): osreleases = models.ManyToManyField(OSRelease, blank=True) cves = models.ManyToManyField(CVE, blank=True) references = models.ManyToManyField(Reference, blank=True) + affected_packages_count = models.PositiveIntegerField(default=0) + fixed_packages_count = models.PositiveIntegerField(default=0) + osreleases_count = models.PositiveIntegerField(default=0) + cves_count = models.PositiveIntegerField(default=0) + references_count = models.PositiveIntegerField(default=0) objects = ErratumManager() @@ -49,9 +54,9 @@ class Meta: ordering = ['-issue_date', 'name'] def __str__(self): - text = f'{self.name} ({self.e_type}), {self.cves.count()} related CVEs, ' - text += f'affecting {self.osreleases.count()} OS Releases, ' - text += f'providing {self.fixed_packages.count()} fixed Packages' + text = f'{self.name} ({self.e_type}), {self.cves_count} related CVEs, ' + text += f'affecting {self.osreleases_count} OS Releases, ' + text += f'providing {self.fixed_packages_count} fixed Packages' return text def get_absolute_url(self): diff --git a/errata/signals.py b/errata/signals.py new file mode 100644 index 00000000..50e101d5 --- /dev/null +++ b/errata/signals.py @@ -0,0 +1,60 @@ +# Copyright 2026 Marcus Furlong +# +# This file is part of Patchman. +# +# Patchman is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, version 3 only. +# +# Patchman is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Patchman. If not, see + +from django.db.models.signals import m2m_changed +from django.dispatch import receiver + +from errata.models import Erratum + + +@receiver(m2m_changed, sender=Erratum.affected_packages.through) +def update_affected_packages_count(sender, instance, action, **kwargs): + """Update affected_packages_count when Erratum.affected_packages M2M changes.""" + if action in ('post_add', 'post_remove', 'post_clear'): + instance.affected_packages_count = instance.affected_packages.count() + instance.save(update_fields=['affected_packages_count']) + + +@receiver(m2m_changed, sender=Erratum.fixed_packages.through) +def update_fixed_packages_count(sender, instance, action, **kwargs): + """Update fixed_packages_count when Erratum.fixed_packages M2M changes.""" + if action in ('post_add', 'post_remove', 'post_clear'): + instance.fixed_packages_count = instance.fixed_packages.count() + instance.save(update_fields=['fixed_packages_count']) + + +@receiver(m2m_changed, sender=Erratum.osreleases.through) +def update_osreleases_count(sender, instance, action, **kwargs): + """Update osreleases_count when Erratum.osreleases M2M changes.""" + if action in ('post_add', 'post_remove', 'post_clear'): + instance.osreleases_count = instance.osreleases.count() + instance.save(update_fields=['osreleases_count']) + + +@receiver(m2m_changed, sender=Erratum.cves.through) +def update_cves_count(sender, instance, action, **kwargs): + """Update cves_count when Erratum.cves M2M changes.""" + if action in ('post_add', 'post_remove', 'post_clear'): + instance.cves_count = instance.cves.count() + instance.save(update_fields=['cves_count']) + + +@receiver(m2m_changed, sender=Erratum.references.through) +def update_references_count(sender, instance, action, **kwargs): + """Update references_count when Erratum.references M2M changes.""" + if action in ('post_add', 'post_remove', 'post_clear'): + instance.references_count = instance.references.count() + instance.save(update_fields=['references_count']) diff --git a/errata/tables.py b/errata/tables.py index d70be2cd..57be40b4 100644 --- a/errata/tables.py +++ b/errata/tables.py @@ -19,31 +19,31 @@ ERRATUM_NAME_TEMPLATE = '{{ record.name }}' PACKAGES_AFFECTED_TEMPLATE = ( - '{% with count=record.affected_packages.count %}' + '{% with count=record.affected_packages_count %}' '{% if count != 0 %}' '{{ count }}' '{% else %}{% endif %}{% endwith %}' ) PACKAGES_FIXED_TEMPLATE = ( - '{% with count=record.fixed_packages.count %}' + '{% with count=record.fixed_packages_count %}' '{% if count != 0 %}' '{{ count }}' '{% else %}{% endif %}{% endwith %}' ) OSRELEASES_TEMPLATE = ( - '{% with count=record.osreleases.count %}' + '{% with count=record.osreleases_count %}' '{% if count != 0 %}' '{{ count }}' '{% else %}{% endif %}{% endwith %}' ) ERRATUM_CVES_TEMPLATE = ( - '{% with count=record.cves.count %}' + '{% with count=record.cves_count %}' '{% if count != 0 %}' '{{ count }}' '{% else %}{% endif %}{% endwith %}' ) REFERENCES_TEMPLATE = ( - '{% with count=record.references.count %}' + '{% with count=record.references_count %}' '{% if count != 0 %}' '{{ count }}' '{% else %}{% endif %}{% endwith %}' diff --git a/errata/templates/errata/erratum_detail.html b/errata/templates/errata/erratum_detail.html index 3e622401..ff446265 100644 --- a/errata/templates/errata/erratum_detail.html +++ b/errata/templates/errata/erratum_detail.html @@ -8,7 +8,7 @@ {% block content %} -{% with affected_count=erratum.affected_packages.count fixed_count=erratum.fixed_packages.count %} +{% with affected_count=erratum.affected_packages_count fixed_count=erratum.fixed_packages_count %}
      • Details
      • Packages Affected ({{ affected_count }})
        • diff --git a/errata/tests/test_models.py b/errata/tests/test_models.py index fc316297..b95cd29b 100644 --- a/errata/tests/test_models.py +++ b/errata/tests/test_models.py @@ -19,7 +19,7 @@ from errata.models import Erratum from operatingsystems.models import OSRelease -from security.models import CVE +from security.models import CVE, Reference @override_settings( @@ -120,3 +120,122 @@ def test_bugfix_erratum(self): issue_date=timezone.now(), ) self.assertEqual(erratum.e_type, 'bugfix') + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class ErratumCachedCountTests(TestCase): + """Tests for Erratum cached count fields and M2M signals.""" + + def setUp(self): + self.erratum = Erratum.objects.create( + name='USN-5678-1', + e_type='security', + synopsis='Security update', + issue_date=timezone.now(), + ) + + def test_initial_counts_are_zero(self): + """Test that cached counts start at zero.""" + self.assertEqual(self.erratum.cves_count, 0) + self.assertEqual(self.erratum.osreleases_count, 0) + self.assertEqual(self.erratum.affected_packages_count, 0) + self.assertEqual(self.erratum.fixed_packages_count, 0) + self.assertEqual(self.erratum.references_count, 0) + + def test_cves_count_on_add(self): + """Test cves_count increments on add.""" + cve1 = CVE.objects.create(cve_id='CVE-2024-0001') + cve2 = CVE.objects.create(cve_id='CVE-2024-0002') + self.erratum.cves.add(cve1) + self.erratum.refresh_from_db() + self.assertEqual(self.erratum.cves_count, 1) + self.erratum.cves.add(cve2) + self.erratum.refresh_from_db() + self.assertEqual(self.erratum.cves_count, 2) + + def test_cves_count_on_remove(self): + """Test cves_count decrements on remove.""" + cve = CVE.objects.create(cve_id='CVE-2024-0003') + self.erratum.cves.add(cve) + self.erratum.refresh_from_db() + self.assertEqual(self.erratum.cves_count, 1) + self.erratum.cves.remove(cve) + self.erratum.refresh_from_db() + self.assertEqual(self.erratum.cves_count, 0) + + def test_cves_count_on_clear(self): + """Test cves_count resets to zero on clear.""" + cve1 = CVE.objects.create(cve_id='CVE-2024-0004') + cve2 = CVE.objects.create(cve_id='CVE-2024-0005') + self.erratum.cves.add(cve1, cve2) + self.erratum.refresh_from_db() + self.assertEqual(self.erratum.cves_count, 2) + self.erratum.cves.clear() + self.erratum.refresh_from_db() + self.assertEqual(self.erratum.cves_count, 0) + + def test_osreleases_count_on_add(self): + """Test osreleases_count increments on add.""" + release = OSRelease.objects.create(name='Ubuntu 24.04') + self.erratum.osreleases.add(release) + self.erratum.refresh_from_db() + self.assertEqual(self.erratum.osreleases_count, 1) + + def test_osreleases_count_on_remove(self): + """Test osreleases_count decrements on remove.""" + release = OSRelease.objects.create(name='Ubuntu 24.04') + self.erratum.osreleases.add(release) + self.erratum.refresh_from_db() + self.assertEqual(self.erratum.osreleases_count, 1) + self.erratum.osreleases.remove(release) + self.erratum.refresh_from_db() + self.assertEqual(self.erratum.osreleases_count, 0) + + def test_references_count_on_add(self): + """Test references_count increments on add.""" + ref = Reference.objects.create( + ref_type='ADVISORY', + url='https://example.com/advisory/1', + ) + self.erratum.references.add(ref) + self.erratum.refresh_from_db() + self.assertEqual(self.erratum.references_count, 1) + + def test_references_count_on_remove(self): + """Test references_count decrements on remove.""" + ref = Reference.objects.create( + ref_type='ADVISORY', + url='https://example.com/advisory/2', + ) + self.erratum.references.add(ref) + self.erratum.refresh_from_db() + self.assertEqual(self.erratum.references_count, 1) + self.erratum.references.remove(ref) + self.erratum.refresh_from_db() + self.assertEqual(self.erratum.references_count, 0) + + def test_str_uses_cached_counts(self): + """Test __str__ reflects cached count values.""" + cve = CVE.objects.create(cve_id='CVE-2024-0010') + release = OSRelease.objects.create(name='RHEL 9') + self.erratum.cves.add(cve) + self.erratum.osreleases.add(release) + self.erratum.refresh_from_db() + result = str(self.erratum) + self.assertIn('1 related CVEs', result) + self.assertIn('affecting 1 OS Releases', result) + self.assertIn('providing 0 fixed Packages', result) + + def test_counts_match_actual_m2m(self): + """Test cached counts stay in sync with actual M2M counts.""" + cve1 = CVE.objects.create(cve_id='CVE-2024-0020') + cve2 = CVE.objects.create(cve_id='CVE-2024-0021') + release = OSRelease.objects.create(name='Debian 12') + self.erratum.cves.add(cve1, cve2) + self.erratum.osreleases.add(release) + self.erratum.refresh_from_db() + self.assertEqual(self.erratum.cves_count, self.erratum.cves.count()) + self.assertEqual(self.erratum.osreleases_count, self.erratum.osreleases.count()) From 152b2f7957ea11167352b9e9559f4946837f44c8 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Thu, 26 Feb 2026 02:24:24 +0000 Subject: [PATCH 119/146] auto-commit to update version skip-checks: true --- VERSION.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION.txt b/VERSION.txt index 7919852f..2d2d6810 100644 --- a/VERSION.txt +++ b/VERSION.txt @@ -1 +1 @@ -4.0.9 +4.0.10 From 36b2727d3429187701d4791d5aa42f42af73e206 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Thu, 26 Feb 2026 02:24:26 +0000 Subject: [PATCH 120/146] auto-commit to update debian changelog skip-checks: true --- debian/changelog | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/debian/changelog b/debian/changelog index 3cc2482c..0d76a88f 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,19 @@ +patchman (4.0.10-1) stable; urgency=medium + + * trigger security updates task on first run + * remove celery deprecation warning + * add cached count fields to avoid expensive COUNT queries + * add further sql optimizations + * add .iterator() to large queryset loops + * address template double-count issues + * move imports to top-level + * replace o(n²) comparisons with id sets and sql aggregation + * consolidate and improve homogenous host update logic + * add cached count fields to erratum model with m2m signals + * auto-commit to update version skip-checks: true + + -- Marcus Furlong Thu, 26 Feb 2026 02:24:26 +0000 + patchman (4.0.9-1) stable; urgency=medium * deb build fix for releases From 7b8d3edb500f2403439e0ba8a6f29853415a11ab Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Thu, 26 Feb 2026 00:03:13 -0500 Subject: [PATCH 121/146] add WSGIPassAuthorization for v2 api with mod_wsgi --- etc/patchman/apache.conf.example | 1 + 1 file changed, 1 insertion(+) diff --git a/etc/patchman/apache.conf.example b/etc/patchman/apache.conf.example index c055eba0..0141cfe6 100644 --- a/etc/patchman/apache.conf.example +++ b/etc/patchman/apache.conf.example @@ -1,6 +1,7 @@ Define patchman_pythonpath /srv/patchman/ WSGIScriptAlias /patchman ${patchman_pythonpath}/patchman/wsgi.py application-group=%{GLOBAL} WSGIPythonPath ${patchman_pythonpath} +WSGIPassAuthorization On From 76dbfe0b5ee4e48bb075e77f8c211da64b1b0859 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Thu, 26 Feb 2026 00:44:06 -0500 Subject: [PATCH 122/146] handle ubuntu versioning without patch release --- reports/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/reports/utils.py b/reports/utils.py index 0b4fe4e5..f45cd19d 100644 --- a/reports/utils.py +++ b/reports/utils.py @@ -611,7 +611,8 @@ def get_os(os, arch): lts = '' if 'LTS' in os: lts = ' LTS' - major, minor, patch = os.split(' ')[1].split('.') + os_ver = os.split(' ')[1].split('.') + major, minor = os_ver[0], os_ver[1] ubuntu_version = f'{major}_{minor}' osrelease_name = f'Ubuntu {major}.{minor}{lts}' cpe_name = f"cpe:2.3:o:canonical:ubuntu_linux:{ubuntu_version}:*:*:*:{'lts' if lts else '*'}:*:*:*" From f58e9390b119cf32b020a07beb0646dbc0f2378f Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Thu, 26 Feb 2026 05:49:32 +0000 Subject: [PATCH 123/146] auto-commit to update version skip-checks: true --- VERSION.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION.txt b/VERSION.txt index 2d2d6810..30b26dfb 100644 --- a/VERSION.txt +++ b/VERSION.txt @@ -1 +1 @@ -4.0.10 +4.0.11 From 24df4c3f88cf1eaf4868744971bbb6be7d057f12 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Thu, 26 Feb 2026 05:49:33 +0000 Subject: [PATCH 124/146] auto-commit to update debian changelog skip-checks: true --- debian/changelog | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/debian/changelog b/debian/changelog index 0d76a88f..223a329d 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,11 @@ +patchman (4.0.11-1) stable; urgency=medium + + * add WSGIPassAuthorization for v2 api with mod_wsgi + * handle ubuntu versioning without patch release + * auto-commit to update version skip-checks: true + + -- Marcus Furlong Thu, 26 Feb 2026 05:49:33 +0000 + patchman (4.0.10-1) stable; urgency=medium * trigger security updates task on first run From 9315e1a49e90ce45b01ab3e867e720ad64a0806a Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Thu, 26 Feb 2026 22:46:46 -0500 Subject: [PATCH 125/146] fix warnings and errors on debian packaging --- debian/compat | 1 - debian/control | 9 +++------ debian/python3-patchman.install | 4 ++-- debian/python3-patchman.postinst | 4 ++-- debian/rules | 2 +- ...ry-worker.service => patchman-celery-worker@.service} | 0 6 files changed, 8 insertions(+), 12 deletions(-) delete mode 100644 debian/compat rename etc/systemd/system/{patchman-celery-worker.service => patchman-celery-worker@.service} (100%) diff --git a/debian/compat b/debian/compat deleted file mode 100644 index f599e28b..00000000 --- a/debian/compat +++ /dev/null @@ -1 +0,0 @@ -10 diff --git a/debian/control b/debian/control index 53121399..f2b052dc 100644 --- a/debian/control +++ b/debian/control @@ -2,8 +2,7 @@ Source: patchman Section: python Priority: optional Maintainer: Marcus Furlong -Uploaders: Marcus Furlong -Build-Depends: debhelper (>=13), python3 (>= 3.11), dh-python, dh-exec +Build-Depends: debhelper-compat (= 13), python3 (>= 3.11), dh-python, dh-exec Standards-Version: 4.6.2 Homepage: https://github.com/furlongm/patchman Vcs-Git: git://github.com/furlongm/patchman @@ -22,10 +21,9 @@ Depends: ${misc:Depends}, python3 (>= 3.11), python3-django (>= 4.2), python3-yaml, libapache2-mod-wsgi-py3, apache2, sqlite3, celery, python3-celery, python3-django-celery-beat, redis-server, python3-redis, python3-git, python3-django-taggit, python3-zstandard, - python3-django-tables2, python3-django-select2 + python3-django-tables2, python3-django-select2, adduser Suggests: python3-mysqldb, python3-psycopg2, python3-pymemcache, memcached Description: Django-based patch status monitoring tool for linux systems. - . Patchman provides a web interface for monitoring host package updates. . Patchman clients send a list of installed packages and enabled repositories to @@ -45,8 +43,7 @@ Description: Django-based patch status monitoring tool for linux systems. Package: patchman-client Architecture: all Homepage: https://github.com/furlongm/patchman -Depends: ${misc:Depends}, curl, debianutils, util-linux, coreutils, mawk, jq +Depends: ${misc:Depends}, curl, mawk, jq Description: Client for the patchman monitoring system. - . The client will send a list of packages and repositories to the upstream patchman server. diff --git a/debian/python3-patchman.install b/debian/python3-patchman.install index 71f47b3a..a5488ca0 100755 --- a/debian/python3-patchman.install +++ b/debian/python3-patchman.install @@ -1,5 +1,5 @@ #!/usr/bin/dh-exec etc/patchman/apache.conf.example => etc/apache2/conf-available/patchman.conf etc/patchman/local_settings.py etc/patchman -etc/systemd/system/patchman-celery-worker.service => lib/systemd/system/patchman-celery-worker@.service -etc/systemd/system/patchman-celery-beat.service => lib/systemd/system/patchman-celery-beat.service +etc/systemd/system/patchman-celery-worker@.service usr/lib/systemd/system +etc/systemd/system/patchman-celery-beat.service => usr/lib/systemd/system/patchman-celery-beat.service diff --git a/debian/python3-patchman.postinst b/debian/python3-patchman.postinst index c4f3a338..e2d4c85e 100644 --- a/debian/python3-patchman.postinst +++ b/debian/python3-patchman.postinst @@ -21,12 +21,12 @@ if [ "$1" = "configure" ] ; then patchman-manage migrate --run-syncdb --fake-initial sqlite3 /var/lib/patchman/db/patchman.db 'PRAGMA journal_mode=WAL;' - adduser --quiet --system --group patchman + adduser --quiet --system --group --home /var/lib/patchman patchman adduser --quiet www-data patchman chown root:patchman /etc/patchman/celery.conf chmod 640 /etc/patchman/celery.conf chmod g+w /var/lib/patchman /var/lib/patchman/db /var/lib/patchman/db/patchman.db - chown -R patchman:patchman /var/lib/patchman + chown patchman:patchman /var/lib/patchman /var/lib/patchman/db /var/lib/patchman/db/patchman.db WORKER_COUNT=1 if [ -f /etc/patchman/celery.conf ]; then diff --git a/debian/rules b/debian/rules index c1612816..622cedf3 100755 --- a/debian/rules +++ b/debian/rules @@ -9,7 +9,7 @@ clean:: export PYBUILD_NAME=patchman %: - dh $@ --with=python3 --buildsystem=pybuild --with=systemd + dh $@ --with=python3 --buildsystem=pybuild override_dh_auto_test: true diff --git a/etc/systemd/system/patchman-celery-worker.service b/etc/systemd/system/patchman-celery-worker@.service similarity index 100% rename from etc/systemd/system/patchman-celery-worker.service rename to etc/systemd/system/patchman-celery-worker@.service From a353843572db739a2c89ad177d9f9adcc3f7ba05 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Fri, 27 Feb 2026 18:02:35 -0500 Subject: [PATCH 126/146] fix overwriting cached update counts with save --- errata/models.py | 1 - errata/tests/test_models.py | 19 +++++++++++ hosts/tests/test_models.py | 63 +++++++++++++++++++++++++++++++++++++ hosts/utils.py | 7 ++++- 4 files changed, 88 insertions(+), 2 deletions(-) diff --git a/errata/models.py b/errata/models.py index 49a0ef5b..4b09deeb 100644 --- a/errata/models.py +++ b/errata/models.py @@ -149,7 +149,6 @@ def parse_osv_dev_data(self, osv_dev_json): def add_fixed_packages(self, packages): for package in packages: self.fixed_packages.add(package) - self.save() def add_affected_packages(self, packages): for package in packages: diff --git a/errata/tests/test_models.py b/errata/tests/test_models.py index b95cd29b..8bae15ff 100644 --- a/errata/tests/test_models.py +++ b/errata/tests/test_models.py @@ -239,3 +239,22 @@ def test_counts_match_actual_m2m(self): self.erratum.refresh_from_db() self.assertEqual(self.erratum.cves_count, self.erratum.cves.count()) self.assertEqual(self.erratum.osreleases_count, self.erratum.osreleases.count()) + + def test_add_fixed_packages_no_stale_save(self): + """Test that add_fixed_packages does not overwrite cached counts. + + Regression test: add_fixed_packages previously called self.save() + after the M2M .add() loop, which overwrote the signal-updated + fixed_packages_count with the stale in-memory value. + """ + from arch.models import PackageArchitecture + from packages.models import Package, PackageName + pkg_arch = PackageArchitecture.objects.create(name='amd64') + pkg_name = PackageName.objects.create(name='libssl3') + pkg = Package.objects.create( + name=pkg_name, arch=pkg_arch, epoch='', + version='3.0.1', release='1', packagetype='D' + ) + self.erratum.add_fixed_packages({pkg}) + self.erratum.refresh_from_db() + self.assertEqual(self.erratum.fixed_packages_count, 1) diff --git a/hosts/tests/test_models.py b/hosts/tests/test_models.py index 09d0e4b3..2bd54dcb 100644 --- a/hosts/tests/test_models.py +++ b/hosts/tests/test_models.py @@ -128,3 +128,66 @@ def test_get_num_security_updates(self): self.assertEqual(self.host.get_num_security_updates(), 1) self.assertEqual(self.host.get_num_bugfix_updates(), 1) self.assertEqual(self.host.get_num_updates(), 2) + + def test_cached_counts_survive_full_save(self): + """Test that cached update counts are not overwritten by a full save. + + Regression test: M2M signals update cached count fields via + save(update_fields=[...]), but a subsequent full save() on the + same in-memory instance would overwrite them with stale values. + """ + pkg_arch = PackageArchitecture.objects.create(name='amd64') + pkg_name = PackageName.objects.create(name='curl') + old_pkg = Package.objects.create( + name=pkg_name, arch=pkg_arch, epoch='', + version='7.0.0', release='1', packagetype='D' + ) + new_pkg = Package.objects.create( + name=pkg_name, arch=pkg_arch, epoch='', + version='7.0.1', release='1', packagetype='D' + ) + sec_update = PackageUpdate.objects.create( + oldpackage=old_pkg, newpackage=new_pkg, security=True + ) + + # simulate the find_host_updates_homogenous pattern: + # M2M add fires signal, then full save follows + self.host.updates.add(sec_update) + self.host.refresh_from_db(fields=['sec_updates_count', 'bug_updates_count']) + self.host.updated_at = timezone.now() + self.host.save() + + # re-read from DB to verify counts persisted + self.host.refresh_from_db() + self.assertEqual(self.host.sec_updates_count, 1) + self.assertEqual(self.host.bug_updates_count, 0) + + def test_in_memory_counts_stale_after_m2m_signal(self): + """Test that in-memory instance has updated counts after M2M signal. + + The M2M signal updates the DB directly via save(update_fields=[...]). + Django also updates the in-memory instance, but on MySQL with + REPEATABLE READ isolation and full save(), the stale snapshot can + overwrite the DB values. The refresh_from_db pattern in + find_host_updates_homogenous prevents this. + """ + pkg_arch = PackageArchitecture.objects.create(name='amd64') + pkg_name = PackageName.objects.create(name='wget') + old_pkg = Package.objects.create( + name=pkg_name, arch=pkg_arch, epoch='', + version='1.0.0', release='1', packagetype='D' + ) + new_pkg = Package.objects.create( + name=pkg_name, arch=pkg_arch, epoch='', + version='1.0.1', release='1', packagetype='D' + ) + bug_update = PackageUpdate.objects.create( + oldpackage=old_pkg, newpackage=new_pkg, security=False + ) + + # M2M add fires signal — DB has correct count + self.host.updates.add(bug_update) + + # verify DB has the correct value + db_host = Host.objects.get(pk=self.host.pk) + self.assertEqual(db_host.bug_updates_count, 1) diff --git a/hosts/utils.py b/hosts/utils.py index 513444bf..abafe574 100644 --- a/hosts/utils.py +++ b/hosts/utils.py @@ -69,7 +69,10 @@ def get_or_create_host(report, arch, osvariant, domain): host.reboot_required = True else: host.reboot_required = False - host.save() + host.save(update_fields=[ + 'ipaddress', 'kernel', 'arch', 'osvariant', + 'domain', 'lastreport', 'reboot_required', + ]) except IntegrityError as e: error_message(text=e) if host: @@ -94,6 +97,7 @@ def find_host_updates_homogenous(hosts, verbose=False): host.find_updates() if verbose: info_message(text='') + host.refresh_from_db(fields=['sec_updates_count', 'bug_updates_count', 'errata_count']) host.updated_at = ts host.save() @@ -114,6 +118,7 @@ def find_host_updates_homogenous(hosts, verbose=False): continue fhost.updates.set(updates) + fhost.refresh_from_db(fields=['sec_updates_count', 'bug_updates_count']) fhost.updated_at = ts fhost.save() updated_host_ids.add(fhost.id) From aa17bc161a958a1c44319725601be31f03345fde Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Sat, 28 Feb 2026 16:51:56 +0000 Subject: [PATCH 127/146] auto-commit to update version skip-checks: true --- VERSION.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION.txt b/VERSION.txt index 30b26dfb..4c05e4ef 100644 --- a/VERSION.txt +++ b/VERSION.txt @@ -1 +1 @@ -4.0.11 +4.0.12 From c300e1829de994511c9071ef62adef001c174cef Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Sat, 28 Feb 2026 16:51:57 +0000 Subject: [PATCH 128/146] auto-commit to update debian changelog skip-checks: true --- debian/changelog | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/debian/changelog b/debian/changelog index 223a329d..68ab292e 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,11 @@ +patchman (4.0.12-1) stable; urgency=medium + + * fix warnings and errors on debian packaging + * fix overwriting cached update counts with save + * auto-commit to update version skip-checks: true + + -- Marcus Furlong Sat, 28 Feb 2026 16:51:57 +0000 + patchman (4.0.11-1) stable; urgency=medium * add WSGIPassAuthorization for v2 api with mod_wsgi From c4a87acacbc2629aa902d32767061669739b3e5d Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Sat, 28 Feb 2026 17:45:16 -0500 Subject: [PATCH 129/146] prefer update_fields over refresh_from_db --- hosts/utils.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/hosts/utils.py b/hosts/utils.py index abafe574..f8f89a15 100644 --- a/hosts/utils.py +++ b/hosts/utils.py @@ -97,9 +97,8 @@ def find_host_updates_homogenous(hosts, verbose=False): host.find_updates() if verbose: info_message(text='') - host.refresh_from_db(fields=['sec_updates_count', 'bug_updates_count', 'errata_count']) host.updated_at = ts - host.save() + host.save(update_fields=['updated_at']) filtered_hosts = Host.objects.filter( packages_count=host.packages_count) @@ -118,9 +117,9 @@ def find_host_updates_homogenous(hosts, verbose=False): continue fhost.updates.set(updates) - fhost.refresh_from_db(fields=['sec_updates_count', 'bug_updates_count']) + fhost.reboot_required = host.reboot_required fhost.updated_at = ts - fhost.save() + fhost.save(update_fields=['updated_at', 'reboot_required']) updated_host_ids.add(fhost.id) info_message(text=f'Added the same updates to {fhost}') elif verbose: From 3fe867573811357f0b12c9006b745c790252433b Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Sat, 28 Feb 2026 14:57:06 -0500 Subject: [PATCH 130/146] better handling of kernel updates --- hosts/models.py | 323 ++++++++++++++++++++--- hosts/tests/test_models.py | 508 ++++++++++++++++++++++++++++++++++++- 2 files changed, 794 insertions(+), 37 deletions(-) diff --git a/hosts/models.py b/hosts/models.py index 96f163bf..2e3ac715 100644 --- a/hosts/models.py +++ b/hosts/models.py @@ -178,27 +178,28 @@ def process_update(self, package, highest_package): def find_updates(self): kernels_q = Q(name__name='kernel') | \ - Q(name__name='kernel-devel') | \ - Q(name__name='kernel-preempt') | \ - Q(name__name='kernel-preempt-devel') | \ - Q(name__name='kernel-rt') | \ - Q(name__name='kernel-rt-devel') | \ - Q(name__name='kernel-debug') | \ - Q(name__name='kernel-debug-devel') | \ - Q(name__name='kernel-default') | \ - Q(name__name='kernel-default-devel') | \ - Q(name__name='kernel-headers') | \ - Q(name__name='kernel-core') | \ - Q(name__name='kernel-modules') | \ - Q(name__name='virtualbox-kmp-default') | \ - Q(name__name='virtualbox-kmp-preempt') | \ - Q(name__name='kernel-uek') | \ - Q(name__name='kernel-uek-devel') | \ - Q(name__name='kernel-uek-debug') | \ - Q(name__name='kernel-uek-debug-devel') | \ - Q(name__name='kernel-uek-container') | \ - Q(name__name='kernel-uek-container-debug') | \ - Q(name__name='kernel-uek-doc') + Q(name__name__startswith='kernel-') | \ + Q(name__name__startswith='virtualbox-kmp-') | \ + Q(name__name__startswith='linux-image-') | \ + Q(name__name__startswith='linux-headers-') | \ + Q(name__name__regex=r'^linux-modules-\d') | \ + Q(name__name__regex=r'^linux-modules-extra-\d') | \ + Q(name__name__startswith='linux-tools-') | \ + Q(name__name__startswith='linux-cloud-tools-') | \ + Q(name__name__startswith='linux-kbuild-') | \ + Q(name__name__startswith='linux-support-') | \ + Q(name__name='linux') | \ + Q(name__name='linux-lts') | \ + Q(name__name='linux-zen') | \ + Q(name__name='linux-hardened') | \ + Q(name__name='linux-rt') | \ + Q(name__name='linux-rt-lts') | \ + Q(name__name='linux-headers') | \ + Q(name__name='linux-lts-headers') | \ + Q(name__name='linux-zen-headers') | \ + Q(name__name='linux-hardened-headers') | \ + Q(name__name='linux-rt-headers') | \ + Q(name__name='linux-rt-lts-headers') repo_packages = self.get_host_repo_packages() host_packages = self.packages.exclude(kernels_q).distinct() kernel_packages = self.packages.filter(kernels_q) @@ -313,42 +314,292 @@ def find_osrelease_repo_updates(self, host_packages, repo_packages): return update_ids def check_if_reboot_required(self, host_highest): + """Check if a reboot is required (running kernel < installed highest). - ver, rel = self.kernel.split('-')[:2] + Uses labelCompare for RPM-style version tuples parsed from uname -r. + Only valid for RPM kernels — DEB and Arch use compare_version via + their respective find_*_kernel_updates methods. + """ + parts = self.kernel.split('-') + if len(parts) < 2: + return + ver, rel = parts[:2] + # strip arch suffix from uname -r release (e.g. '.x86_64', '.aarch64') + arch_suffix = '.' + self.arch.name + if rel.endswith(arch_suffix): + rel = rel[:-len(arch_suffix)] + # SUSE uname -r truncates the micro release (e.g. '160000.8' vs + # RPM release '160000.8.1'), so check for prefix match first + if host_highest.version == ver and \ + (host_highest.release == rel or + host_highest.release.startswith(rel + '.')): + self.reboot_required = False + return kernel_ver = ('', str(ver), str(rel)) host_highest_ver = ('', host_highest.version, host_highest.release) if labelCompare(kernel_ver, host_highest_ver) == -1: self.reboot_required = True else: self.reboot_required = False - self.save() + + def _get_deb_kernel_flavour(self, pkg_name): + """Extract the flavour suffix from a DEB kernel package name. + + e.g. 'linux-image-6.8.0-51-generic' → 'generic' + 'linux-image-6.8.0-51-lowlatency' → 'lowlatency' + 'linux-image-6.1.0-28-cloud-amd64' → 'cloud-amd64' + 'linux-modules-extra-6.8.0-51-generic' → 'generic' + Returns None if the flavour cannot be determined. + """ + for prefix in self._deb_kernel_prefixes: + if pkg_name.startswith(prefix): + # strip prefix, then split version from flavour + # e.g. '6.8.0-51-generic' or '6.1.0-28-cloud-amd64' + remainder = pkg_name[len(prefix):] + # version parts are numeric/dotted, flavour starts after + # e.g. '6.8.0-51-generic' → parts=['6.8.0', '51', 'generic'] + parts = remainder.split('-') + # find first non-numeric part (not starting with digit) + for i, part in enumerate(parts): + if part and not part[0].isdigit(): + return '-'.join(parts[i:]) + return None + return None + + def _get_running_kernel_flavour(self): + """Extract the flavour from the running kernel string. + + e.g. '6.8.0-51-generic' → 'generic' + '6.8.0-51-lowlatency' → 'lowlatency' + '6.1.0-28-cloud-amd64' → 'cloud-amd64' + Returns None for RPM-style kernels (no flavour suffix). + """ + parts = self.kernel.split('-') + if len(parts) >= 2: + # find first non-numeric part after the version + for i, part in enumerate(parts): + if i > 0 and part and not part[0].isdigit(): + return '-'.join(parts[i:]) + return None + + # longest prefixes first to avoid linux-modules- matching linux-modules-extra- + _deb_kernel_prefixes = [ + 'linux-image-unsigned-', + 'linux-modules-extra-', + 'linux-cloud-tools-', + 'linux-image-uc-', + 'linux-image-', + 'linux-headers-', + 'linux-modules-', + 'linux-support-', + 'linux-kbuild-', + 'linux-tools-', + ] def find_kernel_updates(self, kernel_packages, repo_packages): update_ids = [] + self.reboot_required = False + + deb_kernels = kernel_packages.filter(packagetype='D') + rpm_kernels = kernel_packages.filter(packagetype='R') + arch_kernels = kernel_packages.filter(packagetype='A') + + update_ids.extend(self._find_rpm_kernel_updates(rpm_kernels, repo_packages)) + update_ids.extend(self._find_deb_kernel_updates(deb_kernels, repo_packages)) + update_ids.extend(self._find_arch_kernel_updates(arch_kernels, repo_packages)) + + self.save(update_fields=['reboot_required']) + return update_ids + + def _find_rpm_kernel_updates(self, kernel_packages, repo_packages): + + update_ids = [] + + # parse running kernel version for comparison + parts = self.kernel.split('-') + if len(parts) < 2: + return update_ids + ver, rel = parts[:2] + # strip arch suffix from uname -r release (e.g. '.x86_64') + arch_suffix = '.' + self.arch.name + if rel.endswith(arch_suffix): + rel = rel[:-len(arch_suffix)] + + # deduplicate: only process each kernel package name once + processed_names = set() + for package in kernel_packages: - host_highest = package - repo_highest = package + if package.name_id in processed_names: + continue + processed_names.add(package.name_id) pu_q = Q(name=package.name) - potential_updates = repo_packages.filter(pu_q) - for pu in potential_updates: - if package.compare_version(pu) == -1 \ - and repo_highest.compare_version(pu) == -1: + + # find repo highest for this kernel name + repo_highest = None + for pu in repo_packages.filter(pu_q): + if repo_highest is None or repo_highest.compare_version(pu) == -1: repo_highest = pu - host_packages = self.packages.filter(pu_q) - for hp in host_packages: - if package.compare_version(hp) == -1 and \ - host_highest.compare_version(hp) == -1: + if repo_highest is None: + continue + + # find host highest installed for reboot check + host_highest = None + running_package = None + for hp in self.packages.filter(pu_q): + if host_highest is None or host_highest.compare_version(hp) == -1: host_highest = hp + # match installed package to running kernel + # SUSE uname -r truncates micro release ('160000.8' vs + # RPM '160000.8.1') so use prefix match with dot boundary + if hp.version == ver and \ + (hp.release == rel or + hp.release.startswith(rel + '.')): + running_package = hp + + # for the running kernel's flavour, compare running vs repo + # for other flavours, compare highest installed vs repo + if running_package: + base_package = running_package + else: + base_package = host_highest - if host_highest.compare_version(repo_highest) == -1: - uid = self.process_update(host_highest, repo_highest) + if base_package and base_package.compare_version(repo_highest) == -1: + uid = self.process_update(base_package, repo_highest) if uid is not None: update_ids.append(uid) - self.check_if_reboot_required(host_highest) + # reboot check only on primary kernel packages + if host_highest and package.name.name in ( + 'kernel', 'kernel-core', 'kernel-debug-core', + 'kernel-default', 'kernel-rt', 'kernel-azure', + 'kernel-kvmsmall', + 'kernel-uek', 'kernel-uki-virt', 'kernel-debug-uki-virt', + ): + self.check_if_reboot_required(host_highest) + + return update_ids + + def _find_arch_kernel_updates(self, kernel_packages, repo_packages): + + update_ids = [] + + for package in kernel_packages: + pu_q = Q(name=package.name) + + repo_highest = None + for rp in repo_packages.filter(pu_q): + if repo_highest is None or repo_highest.compare_version(rp) == -1: + repo_highest = rp + + if repo_highest is None: + continue + + if package.compare_version(repo_highest) == -1: + uid = self.process_update(package, repo_highest) + if uid is not None: + update_ids.append(uid) + + # reboot check for main kernel packages (not -headers) + # Arch uname -r format varies by flavour: + # linux: 6.12.8-arch1-1 (pkgver=6.12.8.arch1) + # linux-lts: 6.1.68-1-lts (pkgver=6.1.68) + # linux-zen: 6.12.8-zen1-1-zen (pkgver=6.12.8.zen1) + # The only reliable common part is the base version (first segment + # of uname -r) which maps to the numeric prefix of pkgver + if package.name.name in ( + 'linux', 'linux-lts', 'linux-zen', 'linux-hardened', + 'linux-rt', 'linux-rt-lts', + ): + running_base = self.kernel.split('-')[0] if self.kernel else '' + pkg_ver = package.version + # pkgver is either 'X.Y.Z' (lts) or 'X.Y.Z.flavourN' (others) + # check if the package version matches or extends the base + if pkg_ver != running_base and not pkg_ver.startswith(running_base + '.'): + self.reboot_required = True + + return update_ids + + def _find_deb_kernel_updates(self, kernel_packages, repo_packages): + + update_ids = [] + running_flavour = self._get_running_kernel_flavour() + + # find the linux-image package matching the running kernel + running_kernel_pkg = None + for package in kernel_packages: + pkg_name = package.name.name + if pkg_name.startswith('linux-image-') and pkg_name.endswith(self.kernel): + running_kernel_pkg = package + break + + processed_prefixes = set() + for package in kernel_packages: + pkg_name = package.name.name + flavour = self._get_deb_kernel_flavour(pkg_name) + + # if we know the running flavour, only process matching packages + # if we don't (unflavoured kernel), process all kernel packages + if running_flavour and flavour != running_flavour: + continue + + # determine the prefix (e.g. 'linux-image-') + prefix = None + for p in self._deb_kernel_prefixes: + if pkg_name.startswith(p): + prefix = p + break + if prefix is None or prefix in processed_prefixes: + continue + processed_prefixes.add(prefix) + + # build endswith filter for flavoured kernels + name_filter = Q( + name__name__startswith=prefix, + packagetype='D', + ) + if running_flavour: + name_filter &= Q(name__name__endswith=f'-{running_flavour}') + + # find repo highest for this prefix+flavour + repo_highest = None + for rp in repo_packages.filter(name_filter): + if repo_highest is None or repo_highest.compare_version(rp) == -1: + repo_highest = rp + + if repo_highest is None: + continue + + # find the installed package matching the running kernel for this prefix + base_package = None + expected_name = prefix + self.kernel + for hp in self.packages.filter(name_filter): + if hp.name.name == expected_name: + base_package = hp + break + + # fallback: if no running match, use the installed package we started with + if base_package is None: + base_package = package + + if base_package.compare_version(repo_highest) == -1: + uid = self.process_update(base_package, repo_highest) + if uid is not None: + update_ids.append(uid) + + # reboot check: see if a newer linux-image is installed but not running + # use compare_version (DEB semantics) instead of labelCompare + if running_kernel_pkg: + for package in kernel_packages: + if package.name.name.startswith('linux-image-'): + flavour = self._get_deb_kernel_flavour(package.name.name) + if running_flavour is None or flavour == running_flavour: + if running_kernel_pkg.compare_version(package) == -1: + self.reboot_required = True + break + return update_ids diff --git a/hosts/tests/test_models.py b/hosts/tests/test_models.py index 2bd54dcb..12c52ba0 100644 --- a/hosts/tests/test_models.py +++ b/hosts/tests/test_models.py @@ -14,6 +14,7 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see +from django.db.models import Q from django.test import TestCase, override_settings from django.utils import timezone @@ -22,7 +23,7 @@ from hosts.models import Host, HostRepo from operatingsystems.models import OSRelease, OSVariant from packages.models import Package, PackageName, PackageUpdate -from repos.models import Repository +from repos.models import Mirror, MirrorPackage, Repository @override_settings( @@ -191,3 +192,508 @@ def test_in_memory_counts_stale_after_m2m_signal(self): # verify DB has the correct value db_host = Host.objects.get(pk=self.host.pk) self.assertEqual(db_host.bug_updates_count, 1) + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class KernelUpdateTests(TestCase): + """Tests for kernel update detection logic.""" + + def setUp(self): + """Set up test data with RPM kernel packages.""" + self.m_arch = MachineArchitecture.objects.create(name='x86_64') + self.domain = Domain.objects.create(name='example.com') + self.os_release = OSRelease.objects.create( + name='Rocky Linux 9', codename='' + ) + self.os_variant = OSVariant.objects.create( + name='Rocky Linux 9.4', osrelease=self.os_release + ) + self.pkg_arch = PackageArchitecture.objects.create(name='x86_64') + self.kernel_name = PackageName.objects.create(name='kernel-core') + + # create kernel packages at three versions + self.kernel_362 = Package.objects.create( + name=self.kernel_name, arch=self.pkg_arch, epoch='0', + version='5.14.0', release='362.el9', packagetype='R' + ) + self.kernel_427 = Package.objects.create( + name=self.kernel_name, arch=self.pkg_arch, epoch='0', + version='5.14.0', release='427.el9', packagetype='R' + ) + self.kernel_503 = Package.objects.create( + name=self.kernel_name, arch=self.pkg_arch, epoch='0', + version='5.14.0', release='503.el9', packagetype='R' + ) + + # set up a repo with the latest kernel + self.repo = Repository.objects.create( + name='baseos', repotype='R', arch=self.m_arch, + ) + self.mirror = Mirror.objects.create( + repo=self.repo, url='http://repo.example.com/baseos', + ) + self.mirror.packages.add(self.kernel_503) + + def _create_host(self, running_kernel, installed_kernels): + """Helper to create a host with a running kernel and installed packages.""" + host = Host.objects.create( + hostname='rocky.example.com', + ipaddress='192.168.1.50', + osvariant=self.os_variant, + kernel=running_kernel, + arch=self.m_arch, + domain=self.domain, + lastreport=timezone.now(), + host_repos_only=False, + ) + host.packages.set(installed_kernels) + return host + + def test_latest_installed_not_running(self): + """Scenario 1: latest kernel installed but not running. + + Old kernels should not show as updates. + Running→latest should be an update + reboot required. + """ + host = self._create_host( + '5.14.0-362.el9', + [self.kernel_362, self.kernel_427, self.kernel_503], + ) + repo_packages = Package.objects.filter(mirror=self.mirror) + kernel_packages = host.packages.filter(name=self.kernel_name) + + host.find_kernel_updates(kernel_packages, repo_packages) + + self.assertEqual(host.updates.count(), 1) + update = host.updates.first() + self.assertEqual(update.oldpackage, self.kernel_362) + self.assertEqual(update.newpackage, self.kernel_503) + self.assertTrue(host.reboot_required) + + def test_latest_installed_and_running(self): + """Scenario 2: latest kernel installed and running. + + No updates should be generated. + """ + host = self._create_host( + '5.14.0-503.el9', + [self.kernel_362, self.kernel_427, self.kernel_503], + ) + repo_packages = Package.objects.filter(mirror=self.mirror) + kernel_packages = host.packages.filter(name=self.kernel_name) + + host.find_kernel_updates(kernel_packages, repo_packages) + + self.assertEqual(host.updates.count(), 0) + self.assertFalse(host.reboot_required) + + def test_latest_not_installed(self): + """Scenario 3: latest kernel not installed. + + Update from running to repo highest. + """ + host = self._create_host( + '5.14.0-427.el9', + [self.kernel_362, self.kernel_427], + ) + repo_packages = Package.objects.filter(mirror=self.mirror) + kernel_packages = host.packages.filter(name=self.kernel_name) + + host.find_kernel_updates(kernel_packages, repo_packages) + + self.assertEqual(host.updates.count(), 1) + update = host.updates.first() + self.assertEqual(update.oldpackage, self.kernel_427) + self.assertEqual(update.newpackage, self.kernel_503) + + def test_running_newer_than_repo(self): + """Scenario 5: running kernel is newer than repo highest. + + No updates should be generated. + """ + kernel_600 = Package.objects.create( + name=self.kernel_name, arch=self.pkg_arch, epoch='0', + version='5.14.0', release='600.el9', packagetype='R' + ) + host = self._create_host( + '5.14.0-600.el9', + [self.kernel_503, kernel_600], + ) + repo_packages = Package.objects.filter(mirror=self.mirror) + kernel_packages = host.packages.filter(name=self.kernel_name) + + host.find_kernel_updates(kernel_packages, repo_packages) + + self.assertEqual(host.updates.count(), 0) + + def test_old_kernels_not_generating_duplicate_updates(self): + """Old kernels should not each generate their own update.""" + host = self._create_host( + '5.14.0-427.el9', + [self.kernel_362, self.kernel_427], + ) + repo_packages = Package.objects.filter(mirror=self.mirror) + kernel_packages = host.packages.filter(name=self.kernel_name) + + host.find_kernel_updates(kernel_packages, repo_packages) + + # should be exactly 1 update, not 2 + self.assertEqual(host.updates.count(), 1) + + def test_rpm_uname_with_arch_suffix(self): + """RPM: uname -r with .x86_64 suffix should still match correctly.""" + host = self._create_host( + '5.14.0-503.el9.x86_64', + [self.kernel_362, self.kernel_427, self.kernel_503], + ) + repo_packages = Package.objects.filter(mirror=self.mirror) + kernel_packages = host.packages.filter(name=self.kernel_name) + + host.find_kernel_updates(kernel_packages, repo_packages) + + self.assertEqual(host.updates.count(), 0) + host.refresh_from_db() + self.assertFalse(host.reboot_required) + + def test_suse_uname_truncated_release(self): + """SUSE: uname -r truncates micro release and appends flavour.""" + suse_name = PackageName.objects.create(name='kernel-default') + pkg_8 = Package.objects.create( + name=suse_name, arch=self.pkg_arch, epoch='0', + version='6.12.0', release='160000.8.1', packagetype='R' + ) + pkg_25 = Package.objects.create( + name=suse_name, arch=self.pkg_arch, epoch='0', + version='6.12.0', release='160000.25.1', packagetype='R' + ) + suse_repo = Repository.objects.create( + name='oss', repotype='R', arch=self.m_arch, + security=False, enabled=True, + ) + suse_mirror = Mirror.objects.create( + repo=suse_repo, url='http://download.opensuse.org/oss', + ) + MirrorPackage.objects.create(mirror=suse_mirror, package=pkg_8) + MirrorPackage.objects.create(mirror=suse_mirror, package=pkg_25) + + # running 160000.8, installed 160000.8 and 160000.25 + # uname shows '6.12.0-160000.8-default' (truncated, with flavour) + host = self._create_host( + '6.12.0-160000.8-default', + [pkg_8, pkg_25], + ) + host.repos.add(suse_repo) + repo_packages = Package.objects.filter(mirror=suse_mirror) + kernel_packages = host.packages.filter(name=suse_name) + + host.find_kernel_updates(kernel_packages, repo_packages) + + # running 8 < installed 25 → one update, reboot required + self.assertEqual(host.updates.count(), 1) + host.refresh_from_db() + self.assertTrue(host.reboot_required) + + def test_suse_uname_running_is_highest(self): + """SUSE: running kernel matches highest installed (no reboot).""" + suse_name = PackageName.objects.create(name='kernel-default') + pkg_25 = Package.objects.create( + name=suse_name, arch=self.pkg_arch, epoch='0', + version='6.12.0', release='160000.25.1', packagetype='R' + ) + suse_repo = Repository.objects.create( + name='oss', repotype='R', arch=self.m_arch, + security=False, enabled=True, + ) + suse_mirror = Mirror.objects.create( + repo=suse_repo, url='http://download.opensuse.org/oss', + ) + MirrorPackage.objects.create(mirror=suse_mirror, package=pkg_25) + + # running 160000.25 = installed highest 160000.25.1 (truncated match) + host = self._create_host( + '6.12.0-160000.25-default', + [pkg_25], + ) + host.repos.add(suse_repo) + repo_packages = Package.objects.filter(mirror=suse_mirror) + kernel_packages = host.packages.filter(name=suse_name) + + host.find_kernel_updates(kernel_packages, repo_packages) + + self.assertEqual(host.updates.count(), 0) + host.refresh_from_db() + self.assertFalse(host.reboot_required) + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class DebKernelUpdateTests(TestCase): + """Tests for DEB kernel update detection logic.""" + + def setUp(self): + """Set up test data with DEB kernel packages.""" + self.m_arch = MachineArchitecture.objects.create(name='x86_64') + self.domain = Domain.objects.create(name='example.com') + self.os_release = OSRelease.objects.create( + name='Ubuntu 24.04', codename='noble' + ) + self.os_variant = OSVariant.objects.create( + name='Ubuntu 24.04.1 LTS', osrelease=self.os_release + ) + self.pkg_arch = PackageArchitecture.objects.create(name='amd64') + + # DEB kernel package names — each version is a different package name + self.img_49_name = PackageName.objects.create(name='linux-image-6.8.0-49-generic') + self.img_51_name = PackageName.objects.create(name='linux-image-6.8.0-51-generic') + self.img_53_name = PackageName.objects.create(name='linux-image-6.8.0-53-generic') + + self.img_49 = Package.objects.create( + name=self.img_49_name, arch=self.pkg_arch, epoch='', + version='6.8.0-49.50', release='', packagetype='D' + ) + self.img_51 = Package.objects.create( + name=self.img_51_name, arch=self.pkg_arch, epoch='', + version='6.8.0-51.52', release='', packagetype='D' + ) + self.img_53 = Package.objects.create( + name=self.img_53_name, arch=self.pkg_arch, epoch='', + version='6.8.0-53.54', release='', packagetype='D' + ) + + # repo has the latest kernel + self.repo = Repository.objects.create( + name='noble-security', repotype='D', arch=self.m_arch, + ) + from repos.models import Mirror + self.mirror = Mirror.objects.create( + repo=self.repo, url='http://archive.ubuntu.com/ubuntu', + ) + self.mirror.packages.add(self.img_49, self.img_51, self.img_53) + + def _create_host(self, running_kernel, installed_kernels): + host = Host.objects.create( + hostname='ubuntu.example.com', + ipaddress='192.168.1.60', + osvariant=self.os_variant, + kernel=running_kernel, + arch=self.m_arch, + domain=self.domain, + lastreport=timezone.now(), + host_repos_only=False, + ) + host.packages.set(installed_kernels) + return host + + def test_deb_latest_installed_not_running(self): + """DEB: latest kernel installed but not running. + + Should show update from running→latest. Old kernels ignored. + """ + host = self._create_host( + '6.8.0-49-generic', + [self.img_49, self.img_51, self.img_53], + ) + repo_packages = Package.objects.filter(mirror=self.mirror) + kernel_packages = host.packages.filter( + name__name__startswith='linux-image-' + ) + + host.find_kernel_updates(kernel_packages, repo_packages) + + self.assertEqual(host.updates.count(), 1) + update = host.updates.first() + self.assertEqual(update.oldpackage, self.img_49) + self.assertEqual(update.newpackage, self.img_53) + + def test_deb_latest_installed_and_running(self): + """DEB: latest kernel installed and running. No updates.""" + host = self._create_host( + '6.8.0-53-generic', + [self.img_49, self.img_51, self.img_53], + ) + repo_packages = Package.objects.filter(mirror=self.mirror) + kernel_packages = host.packages.filter( + name__name__startswith='linux-image-' + ) + + host.find_kernel_updates(kernel_packages, repo_packages) + + self.assertEqual(host.updates.count(), 0) + + def test_deb_latest_not_installed(self): + """DEB: repo has newer kernel not yet installed.""" + host = self._create_host( + '6.8.0-51-generic', + [self.img_49, self.img_51], + ) + repo_packages = Package.objects.filter(mirror=self.mirror) + kernel_packages = host.packages.filter( + name__name__startswith='linux-image-' + ) + + host.find_kernel_updates(kernel_packages, repo_packages) + + self.assertEqual(host.updates.count(), 1) + update = host.updates.first() + self.assertEqual(update.oldpackage, self.img_51) + self.assertEqual(update.newpackage, self.img_53) + + def test_deb_flavour_extraction(self): + """Test _get_deb_kernel_flavour helper.""" + host = self._create_host('6.8.0-51-generic', [self.img_51]) + self.assertEqual( + host._get_deb_kernel_flavour('linux-image-6.8.0-51-generic'), + 'generic' + ) + self.assertEqual( + host._get_deb_kernel_flavour('linux-modules-extra-6.8.0-51-lowlatency'), + 'lowlatency' + ) + self.assertEqual( + host._get_deb_kernel_flavour('linux-image-6.1.0-28-cloud-amd64'), + 'cloud-amd64' + ) + self.assertEqual( + host._get_deb_kernel_flavour('linux-image-unsigned-6.8.0-51-generic'), + 'generic' + ) + + def test_deb_running_kernel_flavour(self): + """Test _get_running_kernel_flavour helper.""" + host = self._create_host('6.8.0-51-generic', [self.img_51]) + self.assertEqual(host._get_running_kernel_flavour(), 'generic') + + host.kernel = '6.1.0-28-cloud-amd64' + self.assertEqual(host._get_running_kernel_flavour(), 'cloud-amd64') + + host.kernel = '5.14.0-503.el9' + self.assertIsNone(host._get_running_kernel_flavour()) + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class ArchKernelUpdateTests(TestCase): + """Tests for Arch Linux kernel update detection and reboot check.""" + + def setUp(self): + self.m_arch = MachineArchitecture.objects.create(name='x86_64') + self.domain = Domain.objects.create(name='example.com') + self.os_release = OSRelease.objects.create( + name='Arch Linux', codename='arch' + ) + self.os_variant = OSVariant.objects.create( + name='Arch Linux', osrelease=self.os_release + ) + self.pkg_arch = PackageArchitecture.objects.create(name='x86_64') + + self.linux_name = PackageName.objects.create(name='linux') + self.linux_headers_name = PackageName.objects.create(name='linux-headers') + + self.linux_installed = Package.objects.create( + name=self.linux_name, arch=self.pkg_arch, epoch='', + version='6.12.8.arch1', release='1', packagetype='A' + ) + self.linux_headers_installed = Package.objects.create( + name=self.linux_headers_name, arch=self.pkg_arch, epoch='', + version='6.12.8.arch1', release='1', packagetype='A' + ) + self.linux_repo = Package.objects.create( + name=self.linux_name, arch=self.pkg_arch, epoch='', + version='6.12.10.arch1', release='1', packagetype='A' + ) + self.linux_headers_repo = Package.objects.create( + name=self.linux_headers_name, arch=self.pkg_arch, epoch='', + version='6.12.10.arch1', release='1', packagetype='A' + ) + + self.repo = Repository.objects.create( + name='core', repotype='A', arch=self.m_arch, + ) + from repos.models import Mirror + self.mirror = Mirror.objects.create( + repo=self.repo, url='https://mirror.archlinux.org/core', + ) + self.mirror.packages.add(self.linux_repo, self.linux_headers_repo) + + def _create_host(self, running_kernel, installed_pkgs): + host = Host.objects.create( + hostname='arch.example.com', + ipaddress='192.168.1.70', + osvariant=self.os_variant, + kernel=running_kernel, + arch=self.m_arch, + domain=self.domain, + lastreport=timezone.now(), + host_repos_only=False, + ) + host.packages.set(installed_pkgs) + return host + + def test_arch_update_available(self): + """Arch: installed kernel older than repo → update.""" + host = self._create_host( + '6.12.8-arch1-1', + [self.linux_installed, self.linux_headers_installed], + ) + repo_packages = Package.objects.filter(mirror=self.mirror) + kernel_packages = host.packages.filter( + Q(name=self.linux_name) | Q(name=self.linux_headers_name) + ) + + host.find_kernel_updates(kernel_packages, repo_packages) + + self.assertEqual(host.updates.count(), 2) + + def test_arch_no_update(self): + """Arch: installed matches repo → no update.""" + host = self._create_host( + '6.12.10-arch1-1', + [self.linux_repo, self.linux_headers_repo], + ) + repo_packages = Package.objects.filter(mirror=self.mirror) + kernel_packages = host.packages.filter( + Q(name=self.linux_name) | Q(name=self.linux_headers_name) + ) + + host.find_kernel_updates(kernel_packages, repo_packages) + + self.assertEqual(host.updates.count(), 0) + + def test_arch_reboot_required(self): + """Arch: installed kernel newer than running → reboot required.""" + host = self._create_host( + '6.12.8-arch1-1', + [self.linux_repo, self.linux_headers_repo], + ) + repo_packages = Package.objects.filter(mirror=self.mirror) + kernel_packages = host.packages.filter( + Q(name=self.linux_name) | Q(name=self.linux_headers_name) + ) + + host.find_kernel_updates(kernel_packages, repo_packages) + + host.refresh_from_db() + self.assertTrue(host.reboot_required) + + def test_arch_no_reboot_when_current(self): + """Arch: running matches installed → no reboot.""" + host = self._create_host( + '6.12.10-arch1-1', + [self.linux_repo, self.linux_headers_repo], + ) + repo_packages = Package.objects.filter(mirror=self.mirror) + kernel_packages = host.packages.filter( + Q(name=self.linux_name) | Q(name=self.linux_headers_name) + ) + + host.find_kernel_updates(kernel_packages, repo_packages) + + host.refresh_from_db() + self.assertFalse(host.reboot_required) From bc2c4ee809edc4d5f88912b1d8c4e90b9e34c03a Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Sun, 1 Mar 2026 17:20:40 +0000 Subject: [PATCH 131/146] auto-commit to update version skip-checks: true --- VERSION.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION.txt b/VERSION.txt index 4c05e4ef..041792eb 100644 --- a/VERSION.txt +++ b/VERSION.txt @@ -1 +1 @@ -4.0.12 +4.0.13 From 78cfbf80c77e5775b8f29c00a306a62cd7480a20 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Sun, 1 Mar 2026 17:20:41 +0000 Subject: [PATCH 132/146] auto-commit to update debian changelog skip-checks: true --- debian/changelog | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/debian/changelog b/debian/changelog index 68ab292e..0a383d82 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,11 @@ +patchman (4.0.13-1) stable; urgency=medium + + * prefer update_fields over refresh_from_db + * better handling of kernel updates + * auto-commit to update version skip-checks: true + + -- Marcus Furlong Sun, 01 Mar 2026 17:20:41 +0000 + patchman (4.0.12-1) stable; urgency=medium * fix warnings and errors on debian packaging From 521ac64b265cafbd6a214c45923d180b9ce75c87 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Sun, 1 Mar 2026 13:17:28 -0500 Subject: [PATCH 133/146] handle priorities in kernel updates fixes: #770 --- hosts/models.py | 59 +++++++++-- hosts/tests/test_models.py | 199 +++++++++++++++++++++++++++++++++++++ 2 files changed, 250 insertions(+), 8 deletions(-) diff --git a/hosts/models.py b/hosts/models.py index 2e3ac715..10e7fb8e 100644 --- a/hosts/models.py +++ b/hosts/models.py @@ -401,18 +401,28 @@ def find_kernel_updates(self, kernel_packages, repo_packages): update_ids = [] self.reboot_required = False + # build hostrepos for priority filtering (same as find_host_repo_updates) + hostrepos = None + if self.host_repos_only: + hostrepos_q = Q(repo__mirror__enabled=True, + repo__mirror__refresh=True, + repo__mirror__repo__enabled=True, + host=self) + hostrepos = HostRepo.objects.select_related( + 'host', 'repo').filter(hostrepos_q) + deb_kernels = kernel_packages.filter(packagetype='D') rpm_kernels = kernel_packages.filter(packagetype='R') arch_kernels = kernel_packages.filter(packagetype='A') - update_ids.extend(self._find_rpm_kernel_updates(rpm_kernels, repo_packages)) - update_ids.extend(self._find_deb_kernel_updates(deb_kernels, repo_packages)) - update_ids.extend(self._find_arch_kernel_updates(arch_kernels, repo_packages)) + update_ids.extend(self._find_rpm_kernel_updates(rpm_kernels, repo_packages, hostrepos)) + update_ids.extend(self._find_deb_kernel_updates(deb_kernels, repo_packages, hostrepos)) + update_ids.extend(self._find_arch_kernel_updates(arch_kernels, repo_packages, hostrepos)) self.save(update_fields=['reboot_required']) return update_ids - def _find_rpm_kernel_updates(self, kernel_packages, repo_packages): + def _find_rpm_kernel_updates(self, kernel_packages, repo_packages, hostrepos): update_ids = [] @@ -436,9 +446,20 @@ def _find_rpm_kernel_updates(self, kernel_packages, repo_packages): pu_q = Q(name=package.name) - # find repo highest for this kernel name + # determine baseline priority from the installed package's repo + priority = None + if hostrepos is not None: + best_repo = find_best_repo(package, hostrepos) + if best_repo is not None: + priority = best_repo.priority + + # find repo highest for this kernel name, respecting priority repo_highest = None for pu in repo_packages.filter(pu_q): + if priority is not None: + pu_best_repo = find_best_repo(pu, hostrepos) + if not pu_best_repo or pu_best_repo.priority < priority: + continue if repo_highest is None or repo_highest.compare_version(pu) == -1: repo_highest = pu @@ -482,15 +503,26 @@ def _find_rpm_kernel_updates(self, kernel_packages, repo_packages): return update_ids - def _find_arch_kernel_updates(self, kernel_packages, repo_packages): + def _find_arch_kernel_updates(self, kernel_packages, repo_packages, hostrepos): update_ids = [] for package in kernel_packages: pu_q = Q(name=package.name) + # determine baseline priority from the installed package's repo + priority = None + if hostrepos is not None: + best_repo = find_best_repo(package, hostrepos) + if best_repo is not None: + priority = best_repo.priority + repo_highest = None for rp in repo_packages.filter(pu_q): + if priority is not None: + rp_best_repo = find_best_repo(rp, hostrepos) + if not rp_best_repo or rp_best_repo.priority < priority: + continue if repo_highest is None or repo_highest.compare_version(rp) == -1: repo_highest = rp @@ -522,7 +554,7 @@ def _find_arch_kernel_updates(self, kernel_packages, repo_packages): return update_ids - def _find_deb_kernel_updates(self, kernel_packages, repo_packages): + def _find_deb_kernel_updates(self, kernel_packages, repo_packages, hostrepos): update_ids = [] running_flavour = self._get_running_kernel_flavour() @@ -535,6 +567,13 @@ def _find_deb_kernel_updates(self, kernel_packages, repo_packages): running_kernel_pkg = package break + # determine baseline priority from the running kernel's repo + priority = None + if hostrepos is not None and running_kernel_pkg is not None: + best_repo = find_best_repo(running_kernel_pkg, hostrepos) + if best_repo is not None: + priority = best_repo.priority + processed_prefixes = set() for package in kernel_packages: pkg_name = package.name.name @@ -563,9 +602,13 @@ def _find_deb_kernel_updates(self, kernel_packages, repo_packages): if running_flavour: name_filter &= Q(name__name__endswith=f'-{running_flavour}') - # find repo highest for this prefix+flavour + # find repo highest for this prefix+flavour, respecting priority repo_highest = None for rp in repo_packages.filter(name_filter): + if priority is not None: + rp_best_repo = find_best_repo(rp, hostrepos) + if not rp_best_repo or rp_best_repo.priority < priority: + continue if repo_highest is None or repo_highest.compare_version(rp) == -1: repo_highest = rp diff --git a/hosts/tests/test_models.py b/hosts/tests/test_models.py index 12c52ba0..86a846bf 100644 --- a/hosts/tests/test_models.py +++ b/hosts/tests/test_models.py @@ -697,3 +697,202 @@ def test_arch_no_reboot_when_current(self): host.refresh_from_db() self.assertFalse(host.reboot_required) + + +@override_settings( + CELERY_TASK_ALWAYS_EAGER=True, + CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}} +) +class KernelPriorityTests(TestCase): + """Tests for kernel update priority filtering (backports).""" + + def setUp(self): + self.m_arch = MachineArchitecture.objects.create(name='x86_64') + self.domain = Domain.objects.create(name='example.com') + self.os_release = OSRelease.objects.create( + name='Debian 12', codename='bookworm' + ) + self.os_variant = OSVariant.objects.create( + name='Debian GNU/Linux 12', osrelease=self.os_release + ) + self.pkg_arch = PackageArchitecture.objects.create(name='amd64') + + # installed kernel (from main repo) + self.img_43_name = PackageName.objects.create( + name='linux-image-6.1.0-43-amd64' + ) + self.img_43 = Package.objects.create( + name=self.img_43_name, arch=self.pkg_arch, epoch='', + version='6.1.162-1', release='', packagetype='D' + ) + + # backports kernel (newer version) + self.img_bp_name = PackageName.objects.create( + name='linux-image-6.12.73+deb13-amd64' + ) + self.img_bp = Package.objects.create( + name=self.img_bp_name, arch=self.pkg_arch, epoch='', + version='6.12.73-1', release='', packagetype='D' + ) + + # main repo (high priority) + self.main_repo = Repository.objects.create( + name='bookworm', repotype='D', arch=self.m_arch, + ) + self.main_mirror = Mirror.objects.create( + repo=self.main_repo, + url='http://deb.debian.org/debian', + ) + self.main_mirror.packages.add(self.img_43) + + # backports repo (low priority) + self.bp_repo = Repository.objects.create( + name='bookworm-backports', repotype='D', arch=self.m_arch, + ) + self.bp_mirror = Mirror.objects.create( + repo=self.bp_repo, + url='http://deb.debian.org/debian-backports', + ) + self.bp_mirror.packages.add(self.img_bp) + + def _create_host(self, main_priority, bp_priority, host_repos_only=True): + host = Host.objects.create( + hostname='debian.example.com', + ipaddress='192.168.1.80', + osvariant=self.os_variant, + kernel='6.1.0-43-amd64', + arch=self.m_arch, + domain=self.domain, + lastreport=timezone.now(), + host_repos_only=host_repos_only, + ) + host.packages.set([self.img_43]) + HostRepo.objects.create( + host=host, repo=self.main_repo, + enabled=True, priority=main_priority, + ) + HostRepo.objects.create( + host=host, repo=self.bp_repo, + enabled=True, priority=bp_priority, + ) + return host + + def test_deb_backports_lower_priority_no_update(self): + """DEB: backports kernel with lower priority should NOT be flagged.""" + host = self._create_host(main_priority=500, bp_priority=100) + repo_packages = Package.objects.filter( + mirror__in=[self.main_mirror, self.bp_mirror] + ) + kernel_packages = host.packages.filter( + name__name__startswith='linux-image-' + ) + + host.find_kernel_updates(kernel_packages, repo_packages) + + self.assertEqual(host.updates.count(), 0) + + def test_deb_backports_equal_priority_shows_update(self): + """DEB: backports kernel with equal priority SHOULD be flagged.""" + host = self._create_host(main_priority=500, bp_priority=500) + repo_packages = Package.objects.filter( + mirror__in=[self.main_mirror, self.bp_mirror] + ) + kernel_packages = host.packages.filter( + name__name__startswith='linux-image-' + ) + + host.find_kernel_updates(kernel_packages, repo_packages) + + self.assertEqual(host.updates.count(), 1) + + def test_deb_priority_zero_no_filtering(self): + """DEB: priority 0 (unset) means no filtering — backward compat.""" + host = self._create_host(main_priority=0, bp_priority=0) + repo_packages = Package.objects.filter( + mirror__in=[self.main_mirror, self.bp_mirror] + ) + kernel_packages = host.packages.filter( + name__name__startswith='linux-image-' + ) + + host.find_kernel_updates(kernel_packages, repo_packages) + + self.assertEqual(host.updates.count(), 1) + + def test_deb_host_repos_only_false_no_filtering(self): + """DEB: host_repos_only=False skips priority filtering entirely.""" + host = self._create_host( + main_priority=500, bp_priority=100, host_repos_only=False, + ) + repo_packages = Package.objects.filter( + mirror__in=[self.main_mirror, self.bp_mirror] + ) + kernel_packages = host.packages.filter( + name__name__startswith='linux-image-' + ) + + host.find_kernel_updates(kernel_packages, repo_packages) + + self.assertEqual(host.updates.count(), 1) + + def test_rpm_backports_lower_priority_no_update(self): + """RPM: kernel from lower-priority repo should NOT be flagged.""" + rpm_arch = PackageArchitecture.objects.create(name='x86_64_rpm') + kernel_name = PackageName.objects.create(name='kernel-core') + installed = Package.objects.create( + name=kernel_name, arch=rpm_arch, epoch='0', + version='5.14.0', release='362.el9', packagetype='R' + ) + newer = Package.objects.create( + name=kernel_name, arch=rpm_arch, epoch='0', + version='5.14.0', release='503.el9', packagetype='R' + ) + + base_repo = Repository.objects.create( + name='baseos', repotype='R', arch=self.m_arch, + ) + base_mirror = Mirror.objects.create( + repo=base_repo, url='http://repo.example.com/baseos', + ) + base_mirror.packages.add(installed) + + extra_repo = Repository.objects.create( + name='extras', repotype='R', arch=self.m_arch, + ) + extra_mirror = Mirror.objects.create( + repo=extra_repo, url='http://repo.example.com/extras', + ) + extra_mirror.packages.add(newer) + + os_release = OSRelease.objects.create( + name='Rocky Linux 9', codename='rl9' + ) + os_variant = OSVariant.objects.create( + name='Rocky Linux 9.4', osrelease=os_release + ) + host = Host.objects.create( + hostname='rocky.example.com', + ipaddress='192.168.1.90', + osvariant=os_variant, + kernel='5.14.0-362.el9', + arch=self.m_arch, + domain=self.domain, + lastreport=timezone.now(), + host_repos_only=True, + ) + host.packages.set([installed]) + HostRepo.objects.create( + host=host, repo=base_repo, enabled=True, priority=500, + ) + HostRepo.objects.create( + host=host, repo=extra_repo, enabled=True, priority=100, + ) + + repo_packages = Package.objects.filter( + mirror__in=[base_mirror, extra_mirror] + ) + kernel_packages = host.packages.filter(name=kernel_name) + + host.find_kernel_updates(kernel_packages, repo_packages) + + self.assertEqual(host.updates.count(), 0) From f4bb03ea500b874fc67d5610dd1d9024c38ba3f1 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Mon, 2 Mar 2026 22:49:22 +0000 Subject: [PATCH 134/146] auto-commit to update version skip-checks: true --- VERSION.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION.txt b/VERSION.txt index 041792eb..80c4cafe 100644 --- a/VERSION.txt +++ b/VERSION.txt @@ -1 +1 @@ -4.0.13 +4.0.14 From fc4d74ae5686411eaae1c8a8b0497107b5dc9d2f Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Mon, 2 Mar 2026 22:49:24 +0000 Subject: [PATCH 135/146] auto-commit to update debian changelog skip-checks: true --- debian/changelog | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/debian/changelog b/debian/changelog index 0a383d82..126fd132 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,10 @@ +patchman (4.0.14-1) stable; urgency=medium + + * handle priorities in kernel updates + * auto-commit to update version skip-checks: true + + -- Marcus Furlong Mon, 02 Mar 2026 22:49:24 +0000 + patchman (4.0.13-1) stable; urgency=medium * prefer update_fields over refresh_from_db From 9c2fdbd524c2ee29ca7dcc84b925e4c786cb20d0 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Mon, 2 Mar 2026 20:24:31 -0500 Subject: [PATCH 136/146] address mysql utf8mb4 composite index sizes --- ...dule_context_alter_module_name_and_more.py | 33 +++++++++++++++++++ modules/models.py | 10 +++--- ...ename_alter_osrelease_cpe_name_and_more.py | 28 ++++++++++++++++ operatingsystems/models.py | 6 ++-- ...ge_epoch_alter_package_release_and_more.py | 28 ++++++++++++++++ packages/models.py | 18 +++++----- ...erity_alter_cvss_vector_string_and_more.py | 33 +++++++++++++++++++ security/models.py | 8 ++--- 8 files changed, 143 insertions(+), 21 deletions(-) create mode 100644 modules/migrations/0006_alter_module_context_alter_module_name_and_more.py create mode 100644 operatingsystems/migrations/0011_alter_osrelease_codename_alter_osrelease_cpe_name_and_more.py create mode 100644 packages/migrations/0007_alter_package_epoch_alter_package_release_and_more.py create mode 100644 security/migrations/0009_alter_cvss_severity_alter_cvss_vector_string_and_more.py diff --git a/modules/migrations/0006_alter_module_context_alter_module_name_and_more.py b/modules/migrations/0006_alter_module_context_alter_module_name_and_more.py new file mode 100644 index 00000000..389d0a41 --- /dev/null +++ b/modules/migrations/0006_alter_module_context_alter_module_name_and_more.py @@ -0,0 +1,33 @@ +# Generated by Django 4.2.28 on 2026-03-03 01:02 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('modules', '0005_alter_module_unique_together'), + ] + + operations = [ + migrations.AlterField( + model_name='module', + name='context', + field=models.CharField(max_length=128), + ), + migrations.AlterField( + model_name='module', + name='name', + field=models.CharField(max_length=128), + ), + migrations.AlterField( + model_name='module', + name='stream', + field=models.CharField(max_length=128), + ), + migrations.AlterField( + model_name='module', + name='version', + field=models.CharField(max_length=128), + ), + ] diff --git a/modules/models.py b/modules/models.py index e1f8071b..aa06a41b 100644 --- a/modules/models.py +++ b/modules/models.py @@ -24,10 +24,10 @@ class Module(models.Model): - name = models.CharField(max_length=255) - stream = models.CharField(max_length=255) - version = models.CharField(max_length=255) - context = models.CharField(max_length=255) + name = models.CharField(max_length=128) + stream = models.CharField(max_length=128) + version = models.CharField(max_length=128) + context = models.CharField(max_length=128) arch = models.ForeignKey(PackageArchitecture, on_delete=models.CASCADE) repo = models.ForeignKey(Repository, on_delete=models.CASCADE) packages = models.ManyToManyField(Package, blank=True) @@ -39,7 +39,7 @@ class Meta: ordering = ['name', 'stream'] def __str__(self): - return f'{self.name}-{self.stream}-{self.version}-{self.version}-{self.context}' + return f'{self.name}-{self.stream}-{self.version}-{self.context}' def get_absolute_url(self): return reverse('modules:module_detail', args=[str(self.id)]) diff --git a/operatingsystems/migrations/0011_alter_osrelease_codename_alter_osrelease_cpe_name_and_more.py b/operatingsystems/migrations/0011_alter_osrelease_codename_alter_osrelease_cpe_name_and_more.py new file mode 100644 index 00000000..2690d550 --- /dev/null +++ b/operatingsystems/migrations/0011_alter_osrelease_codename_alter_osrelease_cpe_name_and_more.py @@ -0,0 +1,28 @@ +# Generated by Django 4.2.28 on 2026-03-03 01:02 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('operatingsystems', '0010_backfill_osvariant_counts'), + ] + + operations = [ + migrations.AlterField( + model_name='osrelease', + name='codename', + field=models.CharField(blank=True, max_length=128), + ), + migrations.AlterField( + model_name='osrelease', + name='cpe_name', + field=models.CharField(blank=True, max_length=128, null=True, unique=True), + ), + migrations.AlterField( + model_name='osrelease', + name='name', + field=models.CharField(max_length=128, unique=True), + ), + ] diff --git a/operatingsystems/models.py b/operatingsystems/models.py index 911332a9..220cbdb3 100644 --- a/operatingsystems/models.py +++ b/operatingsystems/models.py @@ -24,10 +24,10 @@ class OSRelease(models.Model): - name = models.CharField(max_length=255, unique=True, blank=False, null=False) + name = models.CharField(max_length=128, unique=True, blank=False, null=False) repos = models.ManyToManyField(Repository, blank=True) - codename = models.CharField(max_length=255, blank=True) - cpe_name = models.CharField(max_length=255, null=True, blank=True, unique=True) + codename = models.CharField(max_length=128, blank=True) + cpe_name = models.CharField(max_length=128, null=True, blank=True, unique=True) from operatingsystems.managers import OSReleaseManager objects = OSReleaseManager() diff --git a/packages/migrations/0007_alter_package_epoch_alter_package_release_and_more.py b/packages/migrations/0007_alter_package_epoch_alter_package_release_and_more.py new file mode 100644 index 00000000..c8212c52 --- /dev/null +++ b/packages/migrations/0007_alter_package_epoch_alter_package_release_and_more.py @@ -0,0 +1,28 @@ +# Generated by Django 4.2.28 on 2026-03-03 01:02 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('packages', '0006_alter_packageupdate_options'), + ] + + operations = [ + migrations.AlterField( + model_name='package', + name='epoch', + field=models.CharField(blank=True, max_length=128, null=True), + ), + migrations.AlterField( + model_name='package', + name='release', + field=models.CharField(blank=True, max_length=128, null=True), + ), + migrations.AlterField( + model_name='package', + name='version', + field=models.CharField(max_length=128), + ), + ] diff --git a/packages/models.py b/packages/models.py index 9274a168..99ceaec3 100644 --- a/packages/models.py +++ b/packages/models.py @@ -74,9 +74,9 @@ class Package(models.Model): ) name = models.ForeignKey(PackageName, on_delete=models.CASCADE) - epoch = models.CharField(max_length=255, blank=True, null=True) - version = models.CharField(max_length=255) - release = models.CharField(max_length=255, blank=True, null=True) + epoch = models.CharField(max_length=128, blank=True, null=True) + version = models.CharField(max_length=128) + release = models.CharField(max_length=128, blank=True, null=True) arch = models.ForeignKey(PackageArchitecture, on_delete=models.CASCADE) packagetype = models.CharField(max_length=1, choices=PACKAGE_TYPES, blank=True, null=True) category = models.ForeignKey(PackageCategory, blank=True, null=True, on_delete=models.SET_NULL) @@ -173,13 +173,13 @@ def repo_count(self): class PackageString(models.Model): - name = models.CharField(max_length=255) - version = models.CharField(max_length=255) - epoch = models.CharField(max_length=255, blank=True, null=True) - release = models.CharField(max_length=255, blank=True, null=True) - arch = models.CharField(max_length=255) + name = models.CharField(max_length=128) + version = models.CharField(max_length=128) + epoch = models.CharField(max_length=128, blank=True, null=True) + release = models.CharField(max_length=128, blank=True, null=True) + arch = models.CharField(max_length=128) packagetype = models.CharField(max_length=1, blank=True, null=True) - category = models.CharField(max_length=255, blank=True, null=True) + category = models.CharField(max_length=128, blank=True, null=True) description = models.TextField(blank=True, null=True) url = models.URLField(max_length=255, blank=True, null=True) diff --git a/security/migrations/0009_alter_cvss_severity_alter_cvss_vector_string_and_more.py b/security/migrations/0009_alter_cvss_severity_alter_cvss_vector_string_and_more.py new file mode 100644 index 00000000..2ea6304b --- /dev/null +++ b/security/migrations/0009_alter_cvss_severity_alter_cvss_vector_string_and_more.py @@ -0,0 +1,33 @@ +# Generated by Django 4.2.28 on 2026-03-03 01:02 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('security', '0008_alter_cwe_options_alter_reference_options'), + ] + + operations = [ + migrations.AlterField( + model_name='cvss', + name='severity', + field=models.CharField(blank=True, max_length=128, null=True), + ), + migrations.AlterField( + model_name='cvss', + name='vector_string', + field=models.CharField(blank=True, max_length=128, null=True), + ), + migrations.AlterField( + model_name='reference', + name='ref_type', + field=models.CharField(max_length=128), + ), + migrations.AlterField( + model_name='reference', + name='url', + field=models.URLField(max_length=512), + ), + ] diff --git a/security/models.py b/security/models.py index 60921281..6fd47389 100644 --- a/security/models.py +++ b/security/models.py @@ -28,8 +28,8 @@ class Reference(models.Model): - ref_type = models.CharField(max_length=255) - url = models.URLField(max_length=765) + ref_type = models.CharField(max_length=128) + url = models.URLField(max_length=512) class Meta: unique_together = ['ref_type', 'url'] @@ -82,9 +82,9 @@ def fetch_cwe_data(self): class CVSS(models.Model): score = models.DecimalField(max_digits=3, decimal_places=1, null=True) - severity = models.CharField(max_length=255, blank=True, null=True) + severity = models.CharField(max_length=128, blank=True, null=True) version = models.DecimalField(max_digits=2, decimal_places=1) - vector_string = models.CharField(max_length=255, blank=True, null=True) + vector_string = models.CharField(max_length=128, blank=True, null=True) class Meta: unique_together = ['score', 'severity', 'version', 'vector_string'] From e8082942f643f0cde1565cdb82a3af9a42fd7980 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Tue, 3 Mar 2026 01:46:52 +0000 Subject: [PATCH 137/146] auto-commit to update version skip-checks: true --- VERSION.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION.txt b/VERSION.txt index 80c4cafe..af253c16 100644 --- a/VERSION.txt +++ b/VERSION.txt @@ -1 +1 @@ -4.0.14 +4.0.15 From b8fc93e63c242fb7206621c3338fdede815773bd Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Tue, 3 Mar 2026 01:46:53 +0000 Subject: [PATCH 138/146] auto-commit to update debian changelog skip-checks: true --- debian/changelog | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/debian/changelog b/debian/changelog index 126fd132..809007ca 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,10 @@ +patchman (4.0.15-1) stable; urgency=medium + + * address mysql utf8mb4 composite index sizes + * auto-commit to update version skip-checks: true + + -- Marcus Furlong Tue, 03 Mar 2026 01:46:53 +0000 + patchman (4.0.14-1) stable; urgency=medium * handle priorities in kernel updates From eb09bfed087cd53d15aba1768d87694301530716 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Wed, 4 Mar 2026 11:07:11 -0500 Subject: [PATCH 139/146] fix overly aggressive mysql truncation --- ...verity_alter_cvss_vector_string_and_more.py | 2 +- .../0010_fix_cvss_vector_string_length.py | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 security/migrations/0010_fix_cvss_vector_string_length.py diff --git a/security/migrations/0009_alter_cvss_severity_alter_cvss_vector_string_and_more.py b/security/migrations/0009_alter_cvss_severity_alter_cvss_vector_string_and_more.py index 2ea6304b..43305f20 100644 --- a/security/migrations/0009_alter_cvss_severity_alter_cvss_vector_string_and_more.py +++ b/security/migrations/0009_alter_cvss_severity_alter_cvss_vector_string_and_more.py @@ -18,7 +18,7 @@ class Migration(migrations.Migration): migrations.AlterField( model_name='cvss', name='vector_string', - field=models.CharField(blank=True, max_length=128, null=True), + field=models.CharField(blank=True, max_length=255, null=True), ), migrations.AlterField( model_name='reference', diff --git a/security/migrations/0010_fix_cvss_vector_string_length.py b/security/migrations/0010_fix_cvss_vector_string_length.py new file mode 100644 index 00000000..8eb5770e --- /dev/null +++ b/security/migrations/0010_fix_cvss_vector_string_length.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.28 on 2026-03-04 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('security', '0009_alter_cvss_severity_alter_cvss_vector_string_and_more'), + ] + + operations = [ + migrations.AlterField( + model_name='cvss', + name='vector_string', + field=models.CharField(blank=True, max_length=255, null=True), + ), + ] From 03f2aae07857951d2d912efa37ee86aeb6c26a05 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Wed, 4 Mar 2026 16:19:50 +0000 Subject: [PATCH 140/146] auto-commit to update version skip-checks: true --- VERSION.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION.txt b/VERSION.txt index af253c16..5188cc07 100644 --- a/VERSION.txt +++ b/VERSION.txt @@ -1 +1 @@ -4.0.15 +4.0.16 From 2002d66859ec34874a7414e4eebedd3199b12006 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Wed, 4 Mar 2026 16:19:52 +0000 Subject: [PATCH 141/146] auto-commit to update debian changelog skip-checks: true --- debian/changelog | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/debian/changelog b/debian/changelog index 809007ca..ea97e03f 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,10 @@ +patchman (4.0.16-1) stable; urgency=medium + + * fix overly aggressive mysql truncation + * auto-commit to update version skip-checks: true + + -- Marcus Furlong Wed, 04 Mar 2026 16:19:52 +0000 + patchman (4.0.15-1) stable; urgency=medium * address mysql utf8mb4 composite index sizes From d33fb34f078ac1b6b7bd67558407fc6988532ca8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 4 Mar 2026 22:51:16 +0000 Subject: [PATCH 142/146] Bump django from 4.2.28 to 4.2.29 Bumps [django](https://github.com/django/django) from 4.2.28 to 4.2.29. - [Commits](https://github.com/django/django/compare/4.2.28...4.2.29) --- updated-dependencies: - dependency-name: django dependency-version: 4.2.29 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 013bb934..3f358230 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -Django==4.2.28 +Django==4.2.29 django-taggit==4.0.0 django-extensions==3.2.3 django-bootstrap3==23.1 From 3f012102eb05fb7085322e7ec6da3b7b17e21d85 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Wed, 4 Mar 2026 21:03:58 -0500 Subject: [PATCH 143/146] fix cvss model to match released migration 0010 eb09bfe fixed the migrations but missed updating the model definition. vector_string max_length=128 in the model would reject cvss v4.0 vectors at the python level and cause makemigrations to generate a backwards migration. no new migration needed as 0010 already set the db column to 255. --- security/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/security/models.py b/security/models.py index 6fd47389..4a19f9ca 100644 --- a/security/models.py +++ b/security/models.py @@ -84,7 +84,7 @@ class CVSS(models.Model): score = models.DecimalField(max_digits=3, decimal_places=1, null=True) severity = models.CharField(max_length=128, blank=True, null=True) version = models.DecimalField(max_digits=2, decimal_places=1) - vector_string = models.CharField(max_length=128, blank=True, null=True) + vector_string = models.CharField(max_length=255, blank=True, null=True) class Meta: unique_together = ['score', 'severity', 'version', 'vector_string'] From 9363830b9b0d734f846166b93243334f5111f173 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Thu, 5 Mar 2026 05:37:57 +0000 Subject: [PATCH 144/146] auto-commit to update version skip-checks: true --- VERSION.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION.txt b/VERSION.txt index 5188cc07..9d58a584 100644 --- a/VERSION.txt +++ b/VERSION.txt @@ -1 +1 @@ -4.0.16 +4.0.17 From e5def0e0d4acf80749c13e20d68893d971e99a15 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Thu, 5 Mar 2026 05:37:58 +0000 Subject: [PATCH 145/146] auto-commit to update debian changelog skip-checks: true --- debian/changelog | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/debian/changelog b/debian/changelog index ea97e03f..4e13a9d5 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,14 @@ +patchman (4.0.17-1) stable; urgency=medium + + [ dependabot[bot] ] + * Bump django from 4.2.28 to 4.2.29 + + [ Marcus Furlong ] + * fix cvss model to match released migration 0010 + * auto-commit to update version skip-checks: true + + -- Marcus Furlong Thu, 05 Mar 2026 05:37:58 +0000 + patchman (4.0.16-1) stable; urgency=medium * fix overly aggressive mysql truncation From 51cf2cd033b832ff6f4db7b138ade5ce4918de86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Jer=C3=B3nimo?= Date: Thu, 5 Mar 2026 14:27:51 +0000 Subject: [PATCH 146/146] Added more configuration options via environment variables --- docker/README.md | 12 ++++++++- docker/docker-entrypoint.sh | 52 +++++++++++++++++++++++++++++++++++-- 2 files changed, 61 insertions(+), 3 deletions(-) diff --git a/docker/README.md b/docker/README.md index 3310fccf..b8c7afc5 100644 --- a/docker/README.md +++ b/docker/README.md @@ -23,12 +23,22 @@ This container is configured using environment variables. The following variable | Variable | Default Value | Description | | :--- | :--- | :--- | +| `DEBUG` | `False` | Debug mode | | `ADMIN_NAME` | `Your Name` | Your name | | `ADMIN_EMAIL` | `you@example.com` | Your e-mail address | | `TIMEZONE` | `America/New_York` | Your timezone | +| `LANGUAGE_CODE` | `en-us` | Your language | +| `SECRET_KEY` | | Unique string not to be shared with anyone. Leave empty to generate one randomly | +| `MAX_MIRRORS` | `2` | Maximum number of mirrors to add or refresh per repo | +| `MAX_MIRROR_FAILURES` | `14` | Maximum number of failures before disabling a mirror, set to `-1` to never disable mirrors | +| `DAYS_WITHOUT_REPORT` | `14` | Number of days to wait before raising that a host has not reported | +| `ERRATA_OS_UPDATES` | `yum, rocky, alma, arch, ubuntu, debian` | List of errata sources to update, remove unwanted ones to improve performance | +| `ALMA_RELEASES` | `8, 9, 10` | List of Alma Linux releases to update | +| `DEBIAN_CODENAMES` | `bookworm, trixie` | List of Debian Linux releases to update | +| `UBUNTU_CODENAMES` | `jammy, noble` | List of Ubuntu Linux releases to update | | `DB_ENGINE` | `SQLite` | Database engine to be used. Choose between `MySQL` or `PostgreSQL`, leave empty to use default `SQLite` | | `DB_HOST` | | Database hostname, IP or container name | -| `DB_PORT` |` | Database port | +| `DB_PORT` | | Database port | | `DB_DATABASE` | | Database name | | `DB_USER` | | Database user | | `DB_PASSWORD` | | Database password | diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index 97a75152..d22a17a9 100755 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -2,6 +2,11 @@ conf="/etc/patchman/local_settings.py" +# Configure DEBUG +if "${DEBUG}"; then + sed -i '3 {s/False/True/}' "$conf" +fi + # Configure ADMINS if [ -n "${ADMIN_NAME}" ]; then sed -i '6 {s/Your Name/'"${ADMIN_NAME}"'/}' "$conf" @@ -65,19 +70,62 @@ if [ -n "${DB_ENGINE}" ]; then fi # Configure TIME_ZONE -if [ -n "${TIMEZONE}" ]; then +if [ -n "${TIMEZONE}" ]; then sed -i '22 {s/America\/New_York/'"${TIMEZONE/\//\\/}"'/}' "$conf" fi +# Configure LANGUAGE_CODE +if [ -n "${LANGUAGE_CODE}" ]; then + sed -i '26 {s/en-us/'"${LANGUAGE_CODE}"'/}' "$conf" +fi + # Configure SECRET_KEY if [ -z "$(grep "SECRET_KEY" "$conf" | cut -d " " -f 3 | tr -d "'")" ]; then if [ -n "${SECRET_KEY}" ]; then - sed -i "s/SECRET_KEY = ''/SECRET_KEY = '"${SECRET_KEY}"'/g" "$conf" + sed -i "29 {s/SECRET_KEY = ''/SECRET_KEY = '${SECRET_KEY}'/}" "$conf" else patchman-set-secret-key fi fi +# Configure MAX_MIRRORS +if [ -n "${MAX_MIRRORS}" ]; then + sed -i '36 {s/2/'"${MAX_MIRRORS}"'/}' "$conf" +fi + +# Configure MAX_MIRROR_FAILURES +if [ -n "${MAX_MIRROR_FAILURES}" ]; then + sed -i '39 {s/14/'"${MAX_MIRROR_FAILURES}"'/}' "$conf" +fi + +# Configure DAYS_WITHOUT_REPORT +if [ -n "${DAYS_WITHOUT_REPORT}" ]; then + sed -i '42 {s/14/'"${DAYS_WITHOUT_REPORT}"'/}' "$conf" +fi + +# Configure ERRATA_OS_UPDATES +if [ -n "${ERRATA_OS_UPDATES}" ]; then + errataOSUpdates="${ERRATA_OS_UPDATES// /}" + sed -i '45 {s/\[.*\]/['"'${errataOSUpdates//,/\', \'}'"']/}' "$conf" +fi + +# Configure ALMA_RELEASES +if [ -n "${ALMA_RELEASES}" ]; then + sed -i '48 {s/\[.*\]/['"${ALMA_RELEASES}"']/}' "$conf" +fi + +# Configure DEBIAN_CODENAMES +if [ -n "${DEBIAN_CODENAMES}" ]; then + debianCodenames="${DEBIAN_CODENAMES// /}" + sed -i '51 {s/\[.*\]/['"'${debianCodenames//,/\', \'}'"']/}' "$conf" +fi + +# Configure UBUNTU_CODENAMES +if [ -n "${UBUNTU_CODENAMES}" ]; then + ubuntuCodenames="${UBUNTU_CODENAMES// /}" + sed -i '54 {s/\[.*\]/['"'${ubuntuCodenames//,/\', \'}'"']/}' "$conf" +fi + # Configure CACHES if "${USE_CACHE}"; then if [ -n "${REDIS_HOST}" ]; then