Elgato_dark/unsloth
1
0
Fork 0
mirror of https://github.com/unslothai/unsloth synced 2026-04-21 13:37:39 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Add regression tests for custom llama prebuilt installer (#4772)

Browse source 
Expand test coverage for install_llama_prebuilt.py:
- Add tests for source build plan resolution with custom repos
- Add tests for branch/commit/PR ref matching and normalization
- Add tests for manifest checksum validation
- Add tests for Windows CUDA upstream asset name patterns
- Update capsys checks to capture stderr after log() redirect
This commit is contained in:
Daniel Han 2026-04-02 04:45:09 -07:00 committed by GitHub
parent e2fd946fe1
commit b20efc370a
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 445 additions and 27 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
tests/studio/install/test_install_llama_prebuilt_logic.py
View file

@ -509,7 +509,8 @@ def test_activate_install_tree_restores_existing_install_after_activation_failur
assert not staging_dir.exists()
assert not (tmp_path / ".staging").exists()
output = capsys.readouterr().out
captured = capsys.readouterr()
output = captured.out + captured.err
assert "moving existing install to rollback path" in output
assert "restored previous install from rollback path" in output
@ -571,7 +572,8 @@ def test_activate_install_tree_cleans_all_paths_when_rollback_restore_fails(
assert not staging_dir.exists()
assert not (tmp_path / ".staging").exists()
output = capsys.readouterr().out
captured = capsys.readouterr()
output = captured.out + captured.err
assert "rollback after failed activation also failed: restore failed" in output
assert (
"cleaning staging, install, and rollback paths before source build fallback"

82
tests/studio/install/test_pr4562_bugfixes.py
View file

@ -6,7 +6,7 @@ Tests cover:
- Bug 2: Source-build fallback ignores pinned tag (both .sh and .ps1)
- Bug 3: Unix fallback deletes install before checking prerequisites
- Bug 4: Linux LD_LIBRARY_PATH missing build/bin
- "latest" tag resolution fallback chain (helper -> raw)
- "latest" tag resolution fallback chain (helper only)
- Cross-platform binary_env (Linux, macOS, Windows)
- Edge cases: malformed JSON, empty responses, env overrides
@ -296,6 +296,56 @@ class TestResolveRequestedLlamaTag:
assert resolve_requested_llama_tag("latest", "unslothai/llama.cpp") == "b8999"
def test_latest_with_published_release_tag_passes_pin_through(
self, monkeypatch: pytest.MonkeyPatch
):
captured = {}
def fake_resolve(requested_tag, published_repo, published_release_tag = ""):
captured["requested_tag"] = requested_tag
captured["published_repo"] = published_repo
captured["published_release_tag"] = published_release_tag
return MOD.ResolvedPublishedRelease(
bundle = PublishedReleaseBundle(
repo = published_repo,
release_tag = published_release_tag,
upstream_tag = "b9001",
assets = {},
manifest_asset_name = "llama-prebuilt-manifest.json",
artifacts = [],
selection_log = [],
),
checksums = ApprovedReleaseChecksums(
repo = published_repo,
release_tag = published_release_tag,
upstream_tag = "b9001",
artifacts = {
source_archive_logical_name("b9001"): ApprovedArtifactHash(
asset_name = source_archive_logical_name("b9001"),
sha256 = "a" * 64,
repo = "ggml-org/llama.cpp",
kind = "upstream-source",
)
},
),
)
monkeypatch.setattr(MOD, "resolve_published_release", fake_resolve)
assert (
resolve_requested_llama_tag(
"latest",
"unslothai/llama.cpp",
"llama-prebuilt-main",
)
== "b9001"
)
assert captured == {
"requested_tag": "latest",
"published_repo": "unslothai/llama.cpp",
"published_release_tag": "llama-prebuilt-main",
}
# =========================================================================
# TEST GROUP C: setup.sh logic (bash subprocess tests)
@ -586,18 +636,29 @@ class TestSourceCodePatterns:
content = SETUP_SH.read_text()
assert "_CLONE_ARGS=(git clone --depth 1)" in content
assert (
'_CLONE_ARGS+=(--branch "$_RESOLVED_LLAMA_TAG")' in content
), "_CLONE_ARGS should be extended with --branch $_RESOLVED_LLAMA_TAG"
'_CLONE_ARGS+=(--branch "$_RESOLVED_SOURCE_REF")' in content
), "_CLONE_ARGS should be extended with --branch $_RESOLVED_SOURCE_REF"
# Verify the guard: --branch is only used when tag is not "latest"
assert (
'_RESOLVED_LLAMA_TAG" != "latest"' in content
'_RESOLVED_SOURCE_REF" != "latest"' in content
), "Should guard against literal 'latest' tag"
def test_setup_sh_source_build_uses_helper_resolution(self):
"""Shell source fallback should consult the helper for repo/ref planning."""
content = SETUP_SH.read_text()
assert "--resolve-source-build" in content
assert "--output-format json" in content
assert "_RESOLVED_SOURCE_URL" in content
assert "_RESOLVED_SOURCE_REF_KIND" in content
assert "_RESOLVED_SOURCE_REF" in content
def test_setup_sh_latest_resolution_uses_helper_only(self):
"""Shell fallback should rely on helper output, not raw GitHub API tag_name."""
content = SETUP_SH.read_text()
assert "--resolve-install-tag" in content
assert "--resolve-llama-tag" in content
assert 'tail -n 1 "$_RESOLVE_LLAMA_LOG"' not in content
assert "json.load" in content
assert "_HELPER_RELEASE_REPO}/releases/latest" not in content
assert "ggml-org/llama.cpp/releases/latest" not in content
@ -677,7 +738,7 @@ class TestSourceCodePatterns:
def test_setup_ps1_clone_uses_branch_tag(self):
"""PS1 clone should use --branch with the resolved tag."""
content = SETUP_PS1.read_text()
assert "--branch" in content and "$ResolvedLlamaTag" in content
assert "--branch" in content and "$ResolvedSourceRef" in content
# The old commented-out line should be gone
assert "# git clone --depth 1 --branch" not in content
@ -703,9 +764,20 @@ class TestSourceCodePatterns:
content = SETUP_PS1.read_text()
assert "--resolve-install-tag" in content
assert "--resolve-llama-tag" in content
assert '--output-format", "json"' in content
assert "ConvertFrom-Json" in content
assert "$HelperReleaseRepo/releases/latest" not in content
assert "ggml-org/llama.cpp/releases/latest" not in content
def test_setup_ps1_source_build_uses_helper_resolution(self):
"""PS1 source fallback should consult the helper for repo/ref planning."""
content = SETUP_PS1.read_text()
assert "--resolve-source-build" in content
assert '--output-format", "json"' in content
assert "$ResolvedSourceUrl" in content
assert "$ResolvedSourceRefKind" in content
assert "$ResolvedSourceRef" in content
def test_binary_env_linux_has_binary_parent(self):
"""The Linux branch of binary_env should include binary_path.parent."""
content = MODULE_PATH.read_text()

384
tests/studio/install/test_selection_logic.py
View file

@ -58,7 +58,21 @@ resolve_requested_install_tag = INSTALL_LLAMA_PREBUILT.resolve_requested_install
resolve_install_attempts = INSTALL_LLAMA_PREBUILT.resolve_install_attempts
resolve_install_release_plans = INSTALL_LLAMA_PREBUILT.resolve_install_release_plans
resolve_published_release = INSTALL_LLAMA_PREBUILT.resolve_published_release
resolve_source_build_plan = INSTALL_LLAMA_PREBUILT.resolve_source_build_plan
validated_checksums_for_bundle = INSTALL_LLAMA_PREBUILT.validated_checksums_for_bundle
parse_approved_release_checksums = (
INSTALL_LLAMA_PREBUILT.parse_approved_release_checksums
)
published_release_matches_request = (
INSTALL_LLAMA_PREBUILT.published_release_matches_request
)
exact_source_archive_logical_name = (
INSTALL_LLAMA_PREBUILT.exact_source_archive_logical_name
)
source_archive_logical_name = INSTALL_LLAMA_PREBUILT.source_archive_logical_name
windows_cuda_upstream_asset_names = (
INSTALL_LLAMA_PREBUILT.windows_cuda_upstream_asset_names
)
env_int = INSTALL_LLAMA_PREBUILT.env_int
@ -110,6 +124,13 @@ def make_release(artifacts, **overrides):
repo = "unslothai/llama.cpp",
release_tag = "v1.0",
upstream_tag = "b8508",
source_repo = None,
source_repo_url = None,
source_ref_kind = None,
requested_source_ref = None,
resolved_source_ref = None,
source_commit = None,
source_commit_short = None,
assets = {a.asset_name: f"https://example.com/{a.asset_name}" for a in artifacts},
manifest_asset_name = "llama-prebuilt-manifest.json",
artifacts = artifacts,
@ -124,7 +145,13 @@ def make_checksums(asset_names):
repo = "unslothai/llama.cpp",
release_tag = "v1.0",
upstream_tag = "b8508",
source_repo = None,
source_repo_url = None,
source_ref_kind = None,
requested_source_ref = None,
resolved_source_ref = None,
source_commit = None,
source_commit_short = None,
artifacts = {
name: ApprovedArtifactHash(
asset_name = name,
@ -142,29 +169,56 @@ def make_checksums_with_source(
*,
release_tag = "v1.0",
upstream_tag = "b8508",
source_repo = None,
source_repo_url = None,
source_ref_kind = None,
requested_source_ref = None,
resolved_source_ref = None,
source_commit = None,
):
artifacts = {
**{
name: ApprovedArtifactHash(
asset_name = name,
sha256 = "a" * 64,
repo = "unslothai/llama.cpp",
kind = "prebuilt",
)
for name in asset_names
},
source_archive_logical_name(upstream_tag): ApprovedArtifactHash(
asset_name = source_archive_logical_name(upstream_tag),
sha256 = "b" * 64,
repo = "ggml-org/llama.cpp",
kind = "upstream-source",
),
}
normalized_source_commit = (
source_commit.lower() if isinstance(source_commit, str) else None
)
if normalized_source_commit:
artifacts[exact_source_archive_logical_name(normalized_source_commit)] = (
ApprovedArtifactHash(
asset_name = exact_source_archive_logical_name(normalized_source_commit),
sha256 = "c" * 64,
repo = source_repo or "example/custom-llama.cpp",
kind = "exact-source",
)
)
return ApprovedReleaseChecksums(
repo = "unslothai/llama.cpp",
release_tag = release_tag,
upstream_tag = upstream_tag,
source_commit = None,
artifacts = {
**{
name: ApprovedArtifactHash(
asset_name = name,
sha256 = "a" * 64,
repo = "unslothai/llama.cpp",
kind = "prebuilt",
)
for name in asset_names
},
source_archive_logical_name(upstream_tag): ApprovedArtifactHash(
asset_name = source_archive_logical_name(upstream_tag),
sha256 = "b" * 64,
repo = "ggml-org/llama.cpp",
kind = "upstream-source",
),
},
source_repo = source_repo,
source_repo_url = source_repo_url,
source_ref_kind = source_ref_kind,
requested_source_ref = requested_source_ref,
resolved_source_ref = resolved_source_ref,
source_commit = normalized_source_commit,
source_commit_short = normalized_source_commit[:7]
if normalized_source_commit
else None,
artifacts = artifacts,
)
@ -432,6 +486,31 @@ class TestApplyApprovedHashes:
assert len(result) == 1
assert result[0].name == "a.tar.gz"
def test_upstream_asset_can_match_compatibility_tag_name(self):
choice = AssetChoice(
repo = UPSTREAM_REPO,
tag = "main",
name = "llama-main-bin-macos-arm64.tar.gz",
url = "https://x/llama-main-bin-macos-arm64.tar.gz",
source_label = "upstream",
)
checksums = ApprovedReleaseChecksums(
repo = "unslothai/llama.cpp",
release_tag = "r1",
upstream_tag = "b9000",
artifacts = {
"llama-b9000-bin-macos-arm64.tar.gz": ApprovedArtifactHash(
asset_name = "llama-b9000-bin-macos-arm64.tar.gz",
sha256 = "a" * 64,
repo = UPSTREAM_REPO,
kind = "macos-arm64-upstream",
)
},
)
result = apply_approved_hashes([choice], checksums)
assert result[0].expected_sha256 == "a" * 64
def test_none_approved(self):
c1 = self._choice("missing.tar.gz")
checksums = make_checksums(["other.tar.gz"])
@ -536,6 +615,259 @@ class TestPublishedReleaseResolution:
"unslothai/llama.cpp",
)
def test_request_matches_requested_source_ref(self, monkeypatch):
release = make_release(
[],
release_tag = "release-main",
upstream_tag = "b9000",
requested_source_ref = "main",
resolved_source_ref = "refs/heads/main",
)
monkeypatch.setattr(
INSTALL_LLAMA_PREBUILT,
"iter_published_release_bundles",
lambda repo, published_release_tag = "": iter([release]),
)
monkeypatch.setattr(
INSTALL_LLAMA_PREBUILT,
"load_approved_release_checksums",
lambda repo, release_tag: make_checksums_with_source(
[],
release_tag = release_tag,
upstream_tag = "b9000",
requested_source_ref = "main",
resolved_source_ref = "refs/heads/main",
),
)
resolved = resolve_published_release("main", "unslothai/llama.cpp")
assert resolved.bundle.release_tag == "release-main"
def test_request_matches_source_commit(self, monkeypatch):
commit = "a" * 40
release = make_release(
[],
release_tag = "release-commit",
upstream_tag = "b9000",
source_commit = commit,
)
monkeypatch.setattr(
INSTALL_LLAMA_PREBUILT,
"iter_published_release_bundles",
lambda repo, published_release_tag = "": iter([release]),
)
monkeypatch.setattr(
INSTALL_LLAMA_PREBUILT,
"load_approved_release_checksums",
lambda repo, release_tag: make_checksums_with_source(
[],
release_tag = release_tag,
upstream_tag = "b9000",
source_commit = commit,
),
)
resolved = resolve_published_release(commit, "unslothai/llama.cpp")
assert resolved.bundle.release_tag == "release-commit"
class TestSourceBuildPlanResolution:
def test_matches_request_by_non_tag_provenance(self):
bundle = make_release(
[],
requested_source_ref = "main",
resolved_source_ref = "refs/heads/main",
source_commit = "a" * 40,
)
assert published_release_matches_request(bundle, "main") is True
assert published_release_matches_request(bundle, "refs/heads/main") is True
assert published_release_matches_request(bundle, "a" * 12) is True
assert published_release_matches_request(bundle, "a" * 40) is True
def test_matches_pull_ref_aliases(self):
bundle = make_release(
[],
requested_source_ref = "refs/pull/123/head",
resolved_source_ref = "pull/123/head",
)
assert published_release_matches_request(bundle, "refs/pull/123/head") is True
assert published_release_matches_request(bundle, "pull/123/head") is True
def test_prefers_exact_source_commit_when_available(self, monkeypatch):
commit = "a" * 40
resolved = INSTALL_LLAMA_PREBUILT.ResolvedPublishedRelease(
bundle = make_release(
[],
release_tag = "release-main",
upstream_tag = "b9000",
source_repo = "example/custom-llama.cpp",
source_repo_url = "https://github.com/example/custom-llama.cpp",
source_ref_kind = "branch",
requested_source_ref = "main",
resolved_source_ref = "refs/heads/main",
source_commit = commit,
),
checksums = make_checksums_with_source(
[],
release_tag = "release-main",
upstream_tag = "b9000",
source_repo = "example/custom-llama.cpp",
source_repo_url = "https://github.com/example/custom-llama.cpp",
source_ref_kind = "branch",
requested_source_ref = "main",
resolved_source_ref = "refs/heads/main",
source_commit = commit,
),
)
monkeypatch.setattr(
INSTALL_LLAMA_PREBUILT,
"resolve_published_release",
lambda requested_tag, published_repo, published_release_tag = "": resolved,
)
plan = resolve_source_build_plan("main", "unslothai/llama.cpp")
assert plan.source_url == "https://github.com/example/custom-llama.cpp"
assert plan.source_ref_kind == "commit"
assert plan.source_ref == commit
assert plan.compatibility_upstream_tag == "b9000"
def test_uses_branch_provenance_without_exact_source_hash(self, monkeypatch):
resolved = INSTALL_LLAMA_PREBUILT.ResolvedPublishedRelease(
bundle = make_release(
[],
release_tag = "release-main",
upstream_tag = "b9000",
source_repo = "example/custom-llama.cpp",
source_repo_url = "https://github.com/example/custom-llama.cpp",
source_ref_kind = "branch",
requested_source_ref = "main",
resolved_source_ref = "main",
),
checksums = make_checksums_with_source(
[],
release_tag = "release-main",
upstream_tag = "b9000",
source_repo = "example/custom-llama.cpp",
source_repo_url = "https://github.com/example/custom-llama.cpp",
source_ref_kind = "branch",
requested_source_ref = "main",
resolved_source_ref = "main",
source_commit = None,
),
)
monkeypatch.setattr(
INSTALL_LLAMA_PREBUILT,
"resolve_published_release",
lambda requested_tag, published_repo, published_release_tag = "": resolved,
)
plan = resolve_source_build_plan("main", "unslothai/llama.cpp")
assert plan.source_url == "https://github.com/example/custom-llama.cpp"
assert plan.source_ref_kind == "branch"
assert plan.source_ref == "main"
assert plan.compatibility_upstream_tag == "b9000"
def test_direct_main_request_without_published_release_uses_branch_kind(
self, monkeypatch
):
monkeypatch.setattr(
INSTALL_LLAMA_PREBUILT,
"resolve_published_release",
lambda requested_tag, published_repo, published_release_tag = "": (
_ for _ in ()
).throw(PrebuiltFallback("missing")),
)
plan = resolve_source_build_plan("main", "unslothai/llama.cpp")
assert plan.source_url == "https://github.com/ggml-org/llama.cpp"
assert plan.source_ref_kind == "branch"
assert plan.source_ref == "main"
class TestParseApprovedReleaseChecksums:
def test_rejects_wrong_component(self):
with pytest.raises(RuntimeError, match = "did not describe llama.cpp"):
parse_approved_release_checksums(
"repo/test",
"r1",
{
"schema_version": 1,
"component": "other",
"release_tag": "r1",
"upstream_tag": "b8508",
"artifacts": {},
},
)
def test_rejects_mismatched_release_tag(self):
with pytest.raises(RuntimeError, match = "did not match pinned release tag"):
parse_approved_release_checksums(
"repo/test",
"r1",
{
"schema_version": 1,
"component": "llama.cpp",
"release_tag": "r2",
"upstream_tag": "b8508",
"artifacts": {},
},
)
def test_rejects_bad_sha256(self):
with pytest.raises(RuntimeError, match = "valid sha256"):
parse_approved_release_checksums(
"repo/test",
"r1",
{
"schema_version": 1,
"component": "llama.cpp",
"release_tag": "r1",
"upstream_tag": "b8508",
"artifacts": {
"asset.tar.gz": {
"sha256": "bad-digest",
}
},
},
)
def test_rejects_unsupported_schema_version(self):
with pytest.raises(RuntimeError, match = "schema_version=2 is unsupported"):
parse_approved_release_checksums(
"repo/test",
"r1",
{
"schema_version": 2,
"component": "llama.cpp",
"release_tag": "r1",
"upstream_tag": "b8508",
"artifacts": {},
},
)
class TestValidatedChecksumsForBundle:
def test_rejects_manifest_checksum_mismatch(self, monkeypatch):
bundle = make_release([], release_tag = "r1", upstream_tag = "b8508")
bundle.manifest_sha256 = "a" * 64
checksums = make_checksums_with_source(
[], release_tag = "r1", upstream_tag = "b8508"
)
checksums.artifacts[bundle.manifest_asset_name] = ApprovedArtifactHash(
asset_name = bundle.manifest_asset_name,
sha256 = "b" * 64,
repo = "unslothai/llama.cpp",
kind = "published-manifest",
)
monkeypatch.setattr(
INSTALL_LLAMA_PREBUILT,
"load_approved_release_checksums",
lambda repo, release_tag: checksums,
)
with pytest.raises(PrebuiltFallback, match = "manifest checksum"):
validated_checksums_for_bundle("unslothai/llama.cpp", bundle)
# ===========================================================================
# K. linux_cuda_choice_from_release -- core selection
@ -1360,10 +1692,13 @@ class TestResolveInstallReleasePlans:
class TestWindowsCudaAttempts:
TAG = "b8508"
def _upstream(self, *runtime_versions):
def _upstream(self, *runtime_versions, current_names: bool = False):
assets = {}
for rv in runtime_versions:
name = f"llama-{self.TAG}-bin-win-cuda-{rv}-x64.zip"
if current_names:
name = f"cudart-llama-bin-win-cuda-{rv}-x64.zip"
else:
name = f"llama-{self.TAG}-bin-win-cuda-{rv}-x64.zip"
assets[name] = f"https://example.com/{name}"
return assets
@ -1428,6 +1763,15 @@ class TestWindowsCudaAttempts:
result = windows_cuda_attempts(host, self.TAG, assets, None)
assert len(result) == 2
def test_current_upstream_names_are_supported(self, monkeypatch):
mock_windows_runtime(monkeypatch, ["cuda13", "cuda12"])
host = make_host(system = "Windows", machine = "AMD64", driver_cuda_version = (13, 1))
assets = self._upstream("13.1", "12.4", current_names = True)
result = windows_cuda_attempts(host, self.TAG, assets, None)
assert len(result) == 2
assert result[0].name == "cudart-llama-bin-win-cuda-13.1-x64.zip"
assert result[1].name == "cudart-llama-bin-win-cuda-12.4-x64.zip"
# ===========================================================================
# O. resolve_upstream_asset_choice -- platform routing