diff --git a/Makefile b/Makefile index 1ca7727a..ee34d055 100644 --- a/Makefile +++ b/Makefile @@ -22,6 +22,10 @@ help: @echo "๐Ÿงช Testing:" @echo " test - Run all unit tests" @echo " coverage - Run tests with coverage report" + @echo " test-e2e - Run e2e plugin tests" + @echo " test-run-tutorials - Run tutorial notebooks as e2e tests" + @echo " test-run-recipes - Run recipe scripts as e2e tests" + @echo " test-run-all-examples - Run all tutorials and recipes as e2e tests" @echo "" @echo "โœจ Code Quality:" @echo " format - Format code with ruff" @@ -91,6 +95,33 @@ test-e2e: @echo "๐Ÿงช Running e2e tests..." uv run --no-cache --refresh --directory e2e_tests pytest -s +test-run-tutorials: + @echo "๐Ÿงช Running tutorials as e2e tests..." + @TUTORIAL_WORKDIR=$$(mktemp -d); \ + trap "rm -rf $$TUTORIAL_WORKDIR" EXIT; \ + for f in docs/notebook_source/*.py; do \ + echo " ๐Ÿ““ Running $$f..."; \ + (cd "$$TUTORIAL_WORKDIR" && uv run python "$(REPO_PATH)/$$f") || exit 1; \ + done; \ + echo "๐Ÿงน Cleaning up tutorial artifacts..."; \ + rm -rf "$$TUTORIAL_WORKDIR"; \ + echo "โœ… All tutorials completed successfully!" + +test-run-recipes: + @echo "๐Ÿงช Running recipes as e2e tests..." + @RECIPE_WORKDIR=$$(mktemp -d); \ + trap "rm -rf $$RECIPE_WORKDIR" EXIT; \ + for f in docs/assets/recipes/**/*.py; do \ + echo " ๐Ÿ“œ Running $$f..."; \ + (cd "$$RECIPE_WORKDIR" && uv run python "$(REPO_PATH)/$$f" --model-alias nvidia-text --artifact-path "$$RECIPE_WORKDIR" --num-records 5) || exit 1; \ + done; \ + echo "๐Ÿงน Cleaning up recipe artifacts..."; \ + rm -rf "$$RECIPE_WORKDIR"; \ + echo "โœ… All recipes completed successfully!" + +test-run-all-examples: test-run-tutorials test-run-recipes + @echo "โœ… All examples (tutorials + recipes) completed successfully!" + convert-execute-notebooks: @echo "๐Ÿ““ Converting Python tutorials to notebooks and executing..." @mkdir -p docs/notebooks @@ -137,4 +168,4 @@ install-dev-notebooks: $(call install-pre-commit-hooks) @echo "โœ… Dev + notebooks installation complete!" -.PHONY: clean coverage format format-check lint lint-fix test check-license-headers update-license-headers check-all check-all-fix install install-dev install-dev-notebooks generate-colab-notebooks +.PHONY: clean coverage format format-check lint lint-fix test test-e2e test-run-tutorials test-run-recipes test-run-all-examples check-license-headers update-license-headers check-all check-all-fix install install-dev install-dev-notebooks generate-colab-notebooks diff --git a/docs/assets/recipes/qa_and_chat/product_info_qa.py b/docs/assets/recipes/qa_and_chat/product_info_qa.py index 086680fc..59a0110f 100644 --- a/docs/assets/recipes/qa_and_chat/product_info_qa.py +++ b/docs/assets/recipes/qa_and_chat/product_info_qa.py @@ -150,14 +150,14 @@ def build_config(model_alias: str) -> DataDesignerConfigBuilder: config_builder.add_column( ExpressionColumnConfig( name="completeness_result", - expr="{{ llm_answer_metrics.completeness.score }}", + expr="{{ llm_answer_metrics.Completeness.score }}", ) ) config_builder.add_column( ExpressionColumnConfig( name="accuracy_result", - expr="{{ llm_answer_metrics.accuracy.score }}", + expr="{{ llm_answer_metrics.Accuracy.score }}", ) )