Skip to content

[feat] Git sparse checkout for sourcesdir #3629

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
victorusu wants to merge 4 commits into
base: develop
Choose a base branch
from
Open

[feat] Git sparse checkout for sourcesdir #3629

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
fbde66e
Test the sparse checkout syntax
victorusu Feb 15, 2026
e6cd68c
Change the syntax and support opts
victorusu Feb 23, 2026
44f2579
Update error message
victorusu Feb 23, 2026
ea7a888
Remove srcdir bug
victorusu Feb 23, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 24 additions & 8 deletions reframe/core/pipeline.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -413,7 +413,7 @@ def pipeline_hooks(cls):
#: .. versionchanged:: 3.0
#: Default value is now conditionally set to either ``'src'`` or
#: :class:`None`.
sourcesdir = variable(str, type(None), value='src')
sourcesdir = variable(str, typ.Dict, type(None), value='src')
Copy link
Copy Markdown
Contributor

@vkarak vkarak Mar 9, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
sourcesdir = variable(str, typ.Dict, type(None), value='src')
sourcesdir = variable(str, typ.Dict[str, object], type(None), value='src')


#: .. versionadded:: 2.14
#:
Expand Down Expand Up @@ -1949,11 +1949,12 @@ def _copy_to_stagedir(self, path):
else:
self._mark_stagedir()

def _clone_to_stagedir(self, url):
def _clone_to_stagedir(self, url, files=None):
self.logger.debug(f'Cloning URL {url} into stage directory')
osext.git_clone(
self.sourcesdir, self._stagedir,
timeout=rt.runtime().get_option('general/0/git_timeout')
url, self._stagedir,
timeout=rt.runtime().get_option('general/0/git_timeout'),
files=files
)
self._mark_stagedir()

Expand Down Expand Up @@ -1983,7 +1984,7 @@ def compile(self):
try:
commonpath = os.path.commonpath([self.sourcesdir,
self.sourcepath])
except ValueError:
except (ValueError, TypeError):
commonpath = None

if commonpath:
Expand All @@ -1994,11 +1995,26 @@ def compile(self):
)

if self._requires_stagedir_contents():
if osext.is_url(self.sourcesdir):
self._clone_to_stagedir(self.sourcesdir)
srcdir = self.sourcesdir
if isinstance(srcdir, dict):
if 'url' not in srcdir:
raise ReframeError(f'{srcdir} misses the url key')

url = srcdir['url']
if not osext.is_url(url):
raise ReframeError(f'The dictionary syntax only supports '
'git repositories')

self._clone_to_stagedir(url,
files=srcdir[url]['files'] if 'files'
in srcdir[url] else None,
opts=srcdir[url]['opts'] if 'opts'
in srcdir[url] else None)
elif osext.is_url(srcdir):
self._clone_to_stagedir(srcdir)
Comment on lines +1998 to +2014
Copy link
Copy Markdown
Contributor

@vkarak vkarak Mar 9, 2026 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is somewhat ugly. Ideally, I would see the keys of the sourcesdir correspond the arguments of the target function. You could do something along these lines:

if not isinstance(self.sourcesdir, dict):
    # Convert it to dict passing the default values
    self.sourcesdir = {'url': self.sourcesdir, ...}

try:
    self._clone_to_stagedir(**self.sourcesdir)
except TypeError as err:
    raise ReframeSyntaxError('invalid syntax for sourcesdir') from err

else:
self._copy_to_stagedir(os.path.join(self._prefix,
self.sourcesdir))
srcdir))

# Set executable (only if hasn't been provided)
if not hasattr(self, 'executable'):
Expand Down
10 changes: 8 additions & 2 deletions reframe/utility/osext.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -692,7 +692,7 @@ def is_url(s):
return parsed.scheme != '' and parsed.netloc != ''


def git_clone(url, targetdir=None, opts=None, timeout=5):
def git_clone(url, targetdir=None, opts=None, timeout=5, files=None):
Copy link
Copy Markdown
Contributor

@vkarak vkarak Mar 9, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same here, fine tune the options to match those from _clone_stagedir(). Technically, it's vice-versa: _clone_stagedir() should follow the options of this one and sourcesdir the options in _clone_stagedir().

'''Clone a git repository from a URL.

:arg url: The URL to clone from.
Expand All @@ -708,7 +708,13 @@ def git_clone(url, targetdir=None, opts=None, timeout=5):

targetdir = targetdir or ''
opts = ' '.join(opts) if opts is not None else ''
run_command(f'git clone {opts} {url} {targetdir}', check=True)
if not files:
run_command(f'git clone {opts} {url} {targetdir}', check=True)
else:
run_command(f'git clone --no-checkout --depth=1 {opts} {url} {targetdir}', check=True)
run_command(f'git sparse-checkout set --no-cone {" ".join(files)}', check=True, cwd=targetdir)
run_command('git checkout', check=True, cwd=targetdir)



def git_repo_exists(url, timeout=5):
Expand Down
Loading