Kheiss/cont ingest #1734

Workflow file for this run

.github/workflows/ci-pipeline.yml at ce11ebe

	name: CI Pipeline

	# Workflow control - mirroring GitLab workflow rules
	on:
	# Run on PRs (including forks) - safe jobs without secrets run for all, secret jobs only for same-repo PRs
	pull_request:
	# Run on push only for default branch (e.g. after merge); avoids duplicate runs when pushing to a PR branch
	push:
	branches:
	- develop
	- main
	- 'release-**'
	workflow_dispatch:
	schedule:
	# Run nightly at 7.30 PM UTC or 1 AM IST
	- cron: '30 19 * * *'

	env:
	# Common environment variables
	MILVUS_VERSION: v2.6.5
	NV_INGEST_MAX_UTIL: 8
	APP_VECTORSTORE_ENABLEGPUSEARCH: False
	APP_VECTORSTORE_ENABLEGPUINDEX: False

	concurrency:
	group: ${{ github.workflow }}-${{ github.event.pull_request.number \|\| github.ref }}
	cancel-in-progress: true

	jobs:
	# ============================================================================
	# TEST STAGE JOBS
	# ============================================================================

	helm-blueprint-compliance:
	name: Helm Blueprint Compliance
	runs-on: ubuntu-latest
	# Only run if push to develop OR PR from same repo (not fork) - needs secrets
	if: github.event_name == 'push' \|\| github.event_name == 'workflow_dispatch' \|\| github.event_name == 'schedule' \|\| github.event.pull_request.head.repo.full_name == github.repository
	steps:
	- name: Checkout code
	uses: actions/checkout@v4

	- name: Setup Helm
	uses: azure/setup-helm@v4
	with:
	version: 'latest'

	- name: Add Helm repositories
	env:
	NGC_API_KEY: ${{ secrets.NGC_API_KEY }}
	CI_NVSTAGING_BLUEPRINT_KEY: ${{ secrets.CI_NVSTAGING_BLUEPRINT_KEY }}
	run: \|
	helm repo add nvidia-nim https://helm.ngc.nvidia.com/nim/nvidia/ --username='$oauthtoken' --password=$NGC_API_KEY
	helm repo add nim https://helm.ngc.nvidia.com/nim/ --username='$oauthtoken' --password=$NGC_API_KEY
	helm repo add nemo-microservices https://helm.ngc.nvidia.com/nvidia/nemo-microservices --username='$oauthtoken' --password=$NGC_API_KEY
	helm repo add baidu-nim https://helm.ngc.nvidia.com/nim/baidu --username='$oauthtoken' --password=$NGC_API_KEY
	helm repo add nvstaging-nim https://helm.ngc.nvidia.com/nvstaging/blueprint --username='$oauthtoken' --password=$CI_NVSTAGING_BLUEPRINT_KEY
	helm repo update

	- name: Run Helm Blueprint Compliance
	run: \|
	# Add your helm blueprint compliance checks here
	echo "Running Helm Blueprint Compliance checks..."
	# The actual compliance command would depend on the blueprint compliance tool

	lint:
	runs-on: ubuntu-latest
	steps:
	- name: Check out repository code
	uses: actions/checkout@v4

	- uses: actions/setup-python@v3
	- uses: pre-commit/action@v3.0.1

	unit-tests:
	name: Unit Tests
	runs-on: ubuntu-latest
	container:
	image: python:3.12-slim
	steps:
	- name: Checkout code
	uses: actions/checkout@v4

	- name: Install system dependencies
	run: \|
	apt-get update && apt-get install -y gcc && rm -rf /var/lib/apt/lists/*

	- name: Install package with dependencies
	run: \|
	pip install -e .[all]
	pip install --no-cache-dir -r tests/unit/requirements-test.txt

	- name: Run unit tests with coverage
	run: \|
	python -m pytest -v -s --cov=src --cov-report=term-missing tests/unit

	frontend-unit-tests:
	name: Frontend Unit Tests
	runs-on: ubuntu-latest
	steps:
	- name: Checkout code
	uses: actions/checkout@v4

	- name: Setup Node.js
	uses: actions/setup-node@v4
	with:
	node-version: '20'

	- name: Sanitize branch name
	id: sanitize
	run: \|
	SANITIZED_REF="${GITHUB_REF_NAME//\//-}"
	echo "ref_name=$SANITIZED_REF" >> $GITHUB_OUTPUT

	- name: Setup pnpm
	uses: pnpm/action-setup@v4
	with:
	version: 10

	- name: Cache frontend dependencies
	uses: actions/cache@v4
	with:
	path: \|
	frontend/node_modules
	frontend/.pnpm-store
	key: frontend-deps-${{ github.ref_name }}
	restore-keys: \|
	frontend-deps-

	- name: Configure pnpm and install dependencies
	working-directory: frontend
	run: \|
	pnpm config set store-dir .pnpm-store
	pnpm install --frozen-lockfile \|\| pnpm install

	- name: Run frontend unit tests with coverage
	working-directory: frontend
	run: \|
	pnpm test:coverage

	- name: Upload coverage artifacts
	uses: actions/upload-artifact@v4
	if: always()
	with:
	name: frontend-coverage-${{ steps.sanitize.outputs.ref_name }}-${{ github.sha }}
	path: frontend/coverage/
	retention-days: 7

	check-markdown-links:
	name: Check Markdown Links
	runs-on: ubuntu-latest
	container:
	image: python:3.12-slim
	steps:
	- name: Checkout code
	uses: actions/checkout@v4

	- name: Install required packages
	run: \|
	pip install --no-cache-dir requests

	- name: Run markdown link checker
	run: \|
	python ci/check_markdown_links.py --root . --no-external

	# ============================================================================
	# INTEGRATION TESTS STAGE
	# ============================================================================

	integration-tests:
	name: Integration Tests
	runs-on: arc-runners-org-nvidia-ai-bp-2-gpu
	# Only run if push to develop OR PR from same repo (not fork) - needs secrets
	if: github.event_name == 'push' \|\| github.event_name == 'workflow_dispatch' \|\| github.event_name == 'schedule' \|\| github.event.pull_request.head.repo.full_name == github.repository
	steps:
	- name: Checkout code
	uses: actions/checkout@v4

	- name: Install NGC CLI
	env:
	NGC_API_KEY: ${{ secrets.CI_NVSTAGING_BLUEPRINT_KEY }}
	run: \|
	echo "Installing NGC CLI..."
	wget --content-disposition https://api.ngc.nvidia.com/v2/resources/nvidia/ngc-apps/ngc_cli/versions/4.9.10/files/ngccli_linux.zip -O ngccli_linux.zip
	unzip -o ngccli_linux.zip
	chmod u+x ngc-cli/ngc

	# Add NGC CLI to PATH for subsequent steps
	echo "$(pwd)/ngc-cli" >> $GITHUB_PATH

	echo "NGC CLI installed successfully"

	- name: Download test data files
	env:
	NGC_API_KEY: ${{ secrets.CI_NV_RAG_BLUEPRINT_KEY }}
	run: \|
	echo "Downloading test data files to tests/data..."
	# Create tests/data directory if it doesn't exist
	mkdir -p tests/data

	# Download integration test dataset from NGC
	echo "Downloading integration_test_dataset:2.4.0 from NGC..."
	ngc registry resource download-version "0648981100760671/integration_test_dataset:2.4.0" --dest ./tests/data --org 0648981100760671

	# Move files from subdirectory to tests/data root
	echo "Moving files to tests/data root directory..."
	if [ -d "tests/data/integration_test_dataset_v2.4.0" ]; then
	mv tests/data/integration_test_dataset_v2.4.0/* tests/data/
	rmdir tests/data/integration_test_dataset_v2.4.0
	echo "Files moved successfully"
	else
	echo "Warning: integration_test_dataset_v2.4.0 directory not found"
	fi

	# Verify downloads
	echo "Files in tests/data:"
	ls -lh tests/data/

	echo "Test data download completed"

	- name: Docker info
	run: docker info

	- name: Clean up existing containers
	run: \|
	echo "Cleaning up existing containers..."
	docker ps -a
	docker stop $(docker ps -aq) \|\| true
	docker rm $(docker ps -aq) \|\| true

	- name: Load common environment variables
	run: \|
	echo "Loading common environment variables..."
	export TAG=$(echo ${GITHUB_REF_NAME} \| sed 's/[^a-zA-Z0-9]/-/g')-${GITHUB_SHA::7}
	export NGC_API_KEY=${{ secrets.NGC_API_KEY }}
	export DOCKER_VOLUME_DIRECTORY=/tmp/milvus-${MILVUS_VERSION}
	export INGESTOR_SERVER_EXTERNAL_VOLUME_MOUNT=/tmp/ingestor-server-data
	echo "TAG=$TAG" >> $GITHUB_ENV
	echo "NGC_API_KEY=$NGC_API_KEY" >> $GITHUB_ENV
	echo "DOCKER_VOLUME_DIRECTORY=$DOCKER_VOLUME_DIRECTORY" >> $GITHUB_ENV
	echo "INGESTOR_SERVER_EXTERNAL_VOLUME_MOUNT=$INGESTOR_SERVER_EXTERNAL_VOLUME_MOUNT" >> $GITHUB_ENV

	# Load nvdev.env and export all variables to GITHUB_ENV
	if [ -f ./deploy/compose/nvdev.env ]; then
	set -a
	source ./deploy/compose/nvdev.env
	set +a
	# Export all variables from nvdev.env to GITHUB_ENV
	grep -E '^export ' ./deploy/compose/nvdev.env \| sed 's/export //' \| while IFS='=' read -r key value; do
	# Evaluate the value to expand any variable references
	eval "resolved_value=\"$value\""
	echo "$key=$resolved_value" >> $GITHUB_ENV
	done
	fi

	- name: Docker login
	env:
	CI_NVSTAGING_BLUEPRINT_KEY: ${{ secrets.CI_NVSTAGING_BLUEPRINT_KEY }}
	run: \|
	echo "$CI_NVSTAGING_BLUEPRINT_KEY" \| docker login nvcr.io -u '$oauthtoken' --password-stdin

	- name: Start services
	run: \|
	echo "Starting vector database services..."
	docker compose -f tests/integration/vectordb.yaml up -d
	echo "Starting RAG server..."
	docker compose -f deploy/compose/docker-compose-rag-server.yaml up -d --build
	echo "Starting ingestor server..."
	docker compose -f deploy/compose/docker-compose-ingestor-server.yaml up -d --build
	echo "Waiting for services to be ready..."
	sleep 60
	echo "Checking service status..."
	docker ps

	- name: Print logs for running containers
	run: \|
	echo "=== LOGS FOR RUNNING CONTAINERS ==="
	docker logs --tail 50 milvus-standalone \|\| echo "No logs for milvus-standalone"
	docker logs --tail 50 milvus-etcd \|\| echo "No logs for milvus-etcd"
	docker logs --tail 50 milvus-minio \|\| echo "No logs for milvus-minio"
	docker logs rag-server \|\| echo "No logs for rag-server"
	docker logs ingestor-server \|\| echo "No logs for ingestor-server"
	echo "Deploy stage completed successfully"

	# ========================================================================
	# BASIC TESTS
	# ========================================================================

	- name: Setup Python environment for tests
	run: \|
	# Ensure required Python tooling is installed before use
	if command -v apt-get >/dev/null 2>&1; then
	sudo apt-get update
	sudo apt-get install -y python3 python3-venv python3-pip fuse lsof

	fi

	python3 --version \|\| echo "Python3 not found"
	pip3 --version \|\| echo "pip3 not found"

	# Install uv (Python package and environment manager)
	curl -LsSf https://astral.sh/uv/install.sh \| sh
	export PATH="$HOME/.local/bin:$PATH"

	# Create a fresh virtual environment using uv
	rm -rf venv \|\| echo "No existing venv to clean up"
	uv venv venv
	source venv/bin/activate
	uv pip install -e .[all]
	uv pip install -r tests/integration/requirements.txt

	- name: Run basic integration tests
	id: basic-tests
	continue-on-error: true
	run: \|
	source venv/bin/activate
	echo "Running basic integration tests..."
	python -m tests.integration.main --timeout 1200
	echo "Basic integration tests completed"

	- name: Collect logs after basic tests
	if: always()
	run: \|
	echo "Collecting container logs..."
	mkdir -p logs/basic-tests
	docker logs rag-server > logs/basic-tests/rag-server.log 2>&1 \|\| true
	docker logs ingestor-server > logs/basic-tests/ingestor-server.log 2>&1 \|\| true
	docker logs compose-nv-ingest-ms-runtime-1 > logs/basic-tests/nvingest.log 2>&1 \|\| true
	cp tests/integration/integration_test.log logs/basic-tests/ 2>/dev/null \|\| true

	# ========================================================================
	# QUERY REWRITER TESTS
	# ========================================================================

	- name: Configure environment for query rewriter tests
	run: \|
	echo "ENABLE_QUERYREWRITER=True" >> $GITHUB_ENV
	echo "CONVERSATION_HISTORY=5" >> $GITHUB_ENV

	- name: Restart services for query rewriter tests
	env:
	CI_NVSTAGING_BLUEPRINT_KEY: ${{ secrets.CI_NVSTAGING_BLUEPRINT_KEY }}
	run: \|
	echo "Restarting services with query rewriter configuration..."
	docker compose -f deploy/compose/docker-compose-rag-server.yaml down \|\| true
	docker compose -f deploy/compose/docker-compose-ingestor-server.yaml down \|\| true
	sleep 5
	docker compose -f deploy/compose/docker-compose-rag-server.yaml up -d --build
	docker compose -f deploy/compose/docker-compose-ingestor-server.yaml up -d --build
	sleep 30
	docker ps

	- name: Run query rewriter integration tests
	id: query-rewriter-tests
	continue-on-error: true
	run: \|
	source venv/bin/activate
	echo "Running query rewriter integration tests..."
	python -m tests.integration.main --sequence query_rewriter
	echo "Query rewriter integration tests completed"

	- name: Collect logs after query rewriter tests
	if: always()
	run: \|
	mkdir -p logs/query-rewriter
	docker logs rag-server > logs/query-rewriter/rag-server.log 2>&1 \|\| true
	docker logs ingestor-server > logs/query-rewriter/ingestor-server.log 2>&1 \|\| true
	docker logs compose-nv-ingest-ms-runtime-1 > logs/query-rewriter/nvingest.log 2>&1 \|\| true
	cp tests/integration/integration_test.log logs/query-rewriter/ 2>/dev/null \|\| true

	# ========================================================================
	# REFLECTION TESTS
	# ========================================================================

	- name: Configure environment for reflection tests
	run: \|
	echo "ENABLE_REFLECTION=True" >> $GITHUB_ENV
	echo "RESPONSE_GROUNDEDNESS_THRESHOLD=3" >> $GITHUB_ENV
	# Unset query rewriter settings
	echo "ENABLE_QUERYREWRITER=False" >> $GITHUB_ENV
	echo "CONVERSATION_HISTORY=0" >> $GITHUB_ENV

	- name: Restart services for reflection tests
	env:
	CI_NVSTAGING_BLUEPRINT_KEY: ${{ secrets.CI_NVSTAGING_BLUEPRINT_KEY }}
	run: \|
	echo "Restarting services with reflection configuration..."
	docker compose -f deploy/compose/docker-compose-rag-server.yaml down \|\| true
	docker compose -f deploy/compose/docker-compose-ingestor-server.yaml down \|\| true
	sleep 5
	docker compose -f deploy/compose/docker-compose-rag-server.yaml up -d --build
	docker compose -f deploy/compose/docker-compose-ingestor-server.yaml up -d --build
	sleep 30
	docker ps

	- name: Run reflection integration tests
	id: reflection-tests
	continue-on-error: true
	run: \|
	source venv/bin/activate
	echo "Running reflection integration tests..."
	python -m tests.integration.main --sequence reflection
	echo "Reflection integration tests completed"

	- name: Collect logs after reflection tests
	if: always()
	run: \|
	mkdir -p logs/reflection
	docker logs rag-server > logs/reflection/rag-server.log 2>&1 \|\| true
	docker logs ingestor-server > logs/reflection/ingestor-server.log 2>&1 \|\| true
	docker logs compose-nv-ingest-ms-runtime-1 > logs/reflection/nvingest.log 2>&1 \|\| true
	cp tests/integration/integration_test.log logs/reflection/ 2>/dev/null \|\| true

	# ========================================================================
	# NEMO GUARDRAILS TESTS
	# ========================================================================

	- name: Configure environment for guardrails tests
	run: \|
	echo "ENABLE_GUARDRAILS=True" >> $GITHUB_ENV
	# Unset reflection and query rewriter settings
	echo "ENABLE_REFLECTION=False" >> $GITHUB_ENV
	echo "ENABLE_QUERYREWRITER=False" >> $GITHUB_ENV
	echo "CONVERSATION_HISTORY=0" >> $GITHUB_ENV

	- name: Restart services for guardrails tests
	env:
	CI_NVSTAGING_BLUEPRINT_KEY: ${{ secrets.CI_NVSTAGING_BLUEPRINT_KEY }}
	run: \|
	echo "Restarting services with guardrails configuration..."
	docker compose -f deploy/compose/docker-compose-rag-server.yaml down \|\| true
	docker compose -f deploy/compose/docker-compose-ingestor-server.yaml down \|\| true
	sleep 5
	docker compose -f deploy/compose/docker-compose-rag-server.yaml up -d --build
	docker compose -f deploy/compose/docker-compose-ingestor-server.yaml up -d --build
	sleep 30
	docker ps

	- name: Start NemoGuardrails microservice
	run: \|
	echo "Starting NemoGuardrails microservice..."
	docker compose -f deploy/compose/docker-compose-nemo-guardrails.yaml up -d --no-deps nemo-guardrails-microservice
	sleep 30

	- name: Run NemoGuardrails integration tests
	id: guardrails-tests
	continue-on-error: true
	run: \|
	source venv/bin/activate
	echo "Running NemoGuardrails integration tests..."
	python -m tests.integration.main --sequence nemo_guardrails
	echo "NemoGuardrails integration tests completed"

	- name: Collect logs after guardrails tests
	if: always()
	run: \|
	mkdir -p logs/nemo-guardrails
	docker logs rag-server > logs/nemo-guardrails/rag-server.log 2>&1 \|\| true
	docker logs ingestor-server > logs/nemo-guardrails/ingestor-server.log 2>&1 \|\| true
	docker logs compose-nv-ingest-ms-runtime-1 > logs/nemo-guardrails/nvingest.log 2>&1 \|\| true
	docker logs nemo-guardrails-microservice > logs/nemo-guardrails/nemo-guardrails.log 2>&1 \|\| true
	cp tests/integration/integration_test.log logs/nemo-guardrails/ 2>/dev/null \|\| true

	- name: Stop NemoGuardrails microservice
	if: always()
	run: \|
	docker compose -f deploy/compose/docker-compose-nemo-guardrails.yaml down nemo-guardrails-microservice \|\| true

	# ========================================================================
	# IMAGE CAPTIONING TESTS
	# ========================================================================

	- name: Configure environment for image captioning tests
	run: \|
	echo "APP_NVINGEST_EXTRACTIMAGES=True" >> $GITHUB_ENV
	echo "ENABLE_GUARDRAILS=False" >> $GITHUB_ENV
	echo "ENABLE_QUERYREWRITER=False" >> $GITHUB_ENV
	echo "CONVERSATION_HISTORY=0" >> $GITHUB_ENV

	- name: Restart services for image captioning tests
	env:
	CI_NVSTAGING_BLUEPRINT_KEY: ${{ secrets.CI_NVSTAGING_BLUEPRINT_KEY }}
	run: \|
	echo "Restarting services with image captioning configuration..."
	docker compose -f deploy/compose/docker-compose-rag-server.yaml down \|\| true
	docker compose -f deploy/compose/docker-compose-ingestor-server.yaml down \|\| true
	sleep 5
	docker compose -f deploy/compose/docker-compose-rag-server.yaml up -d --build
	docker compose -f deploy/compose/docker-compose-ingestor-server.yaml up -d --build
	sleep 30
	docker ps

	- name: Run image captioning integration tests
	id: image-captioning-tests
	continue-on-error: true
	run: \|
	source venv/bin/activate
	echo "Running image captioning integration tests..."
	python -m tests.integration.main --sequence image_captioning
	echo "Image captioning integration tests completed"

	- name: Collect logs after image captioning tests
	if: always()
	run: \|
	mkdir -p logs/image-captioning
	docker logs rag-server > logs/image-captioning/rag-server.log 2>&1 \|\| true
	docker logs ingestor-server > logs/image-captioning/ingestor-server.log 2>&1 \|\| true
	docker logs compose-nv-ingest-ms-runtime-1 > logs/image-captioning/nvingest.log 2>&1 \|\| true
	cp tests/integration/integration_test.log logs/image-captioning/ 2>/dev/null \|\| true

	# ========================================================================
	# VLM GENERATION TESTS
	# ========================================================================

	- name: Configure environment for VLM generation tests
	run: \|
	echo "ENABLE_VLM_INFERENCE=True" >> $GITHUB_ENV
	echo "APP_NVINGEST_EXTRACTIMAGES=False" >> $GITHUB_ENV
	echo "ENABLE_QUERYREWRITER=False" >> $GITHUB_ENV
	echo "CONVERSATION_HISTORY=0" >> $GITHUB_ENV

	- name: Restart services for VLM generation tests
	env:
	CI_NVSTAGING_BLUEPRINT_KEY: ${{ secrets.CI_NVSTAGING_BLUEPRINT_KEY }}
	run: \|
	echo "Restarting services with VLM generation configuration..."
	docker compose -f deploy/compose/docker-compose-rag-server.yaml down \|\| true
	docker compose -f deploy/compose/docker-compose-ingestor-server.yaml down \|\| true
	sleep 5
	docker compose -f deploy/compose/docker-compose-rag-server.yaml up -d --build
	docker compose -f deploy/compose/docker-compose-ingestor-server.yaml up -d --build
	sleep 30
	docker ps

	- name: Run VLM generation integration tests
	id: vlm-generation-tests
	continue-on-error: true
	run: \|
	source venv/bin/activate
	echo "Running VLM generation integration tests..."
	python -m tests.integration.main --sequence vlm_generation
	echo "VLM generation integration tests completed"

	- name: Collect logs after VLM generation tests
	if: always()
	run: \|
	mkdir -p logs/vlm-generation
	docker logs rag-server > logs/vlm-generation/rag-server.log 2>&1 \|\| true
	docker logs ingestor-server > logs/vlm-generation/ingestor-server.log 2>&1 \|\| true
	docker logs compose-nv-ingest-ms-runtime-1 > logs/vlm-generation/nvingest.log 2>&1 \|\| true
	cp tests/integration/integration_test.log logs/vlm-generation/ 2>/dev/null \|\| true

	# ========================================================================
	# MULTIMODAL QUERY TESTS
	# ========================================================================

	- name: Prepare multimodal test data
	run: \|
	mkdir -p data/multimodal/query
	[ -f tests/data/product_catalog.pdf ] && cp tests/data/product_catalog.pdf data/multimodal/ \|\| true
	[ -f tests/data/Creme_clutch_purse1-small.jpg ] && cp tests/data/Creme_clutch_purse1-small.jpg data/multimodal/query/ \|\| true
	[ -f tests/data/query/Creme_clutch_purse1-small.jpg ] && cp tests/data/query/Creme_clutch_purse1-small.jpg data/multimodal/query/ \|\| true

	- name: Configure environment for multimodal query tests
	run: \|
	# VLM embedding (required for multimodal queries)
	echo "APP_EMBEDDINGS_MODELNAME=nvidia/llama-nemotron-embed-vl-1b-v2" >> $GITHUB_ENV
	echo "APP_EMBEDDINGS_SERVERURL=https://integrate.api.nvidia.com/v1" >> $GITHUB_ENV
	# VLM model for generation
	echo "ENABLE_VLM_INFERENCE=True" >> $GITHUB_ENV
	echo "APP_VLM_MODELNAME=nvidia/nemotron-nano-12b-v2-vl" >> $GITHUB_ENV
	echo "APP_VLM_SERVERURL=https://integrate.api.nvidia.com/v1" >> $GITHUB_ENV
	# Disable reranker (not supported with multimodal)
	echo "ENABLE_RERANKER=False" >> $GITHUB_ENV
	echo "APP_RANKING_SERVERURL=" >> $GITHUB_ENV
	# Image extraction for ingestion
	echo "APP_NVINGEST_EXTRACTIMAGES=True" >> $GITHUB_ENV
	echo "APP_NVINGEST_STRUCTURED_ELEMENTS_MODALITY=" >> $GITHUB_ENV
	echo "APP_NVINGEST_IMAGE_ELEMENTS_MODALITY=image" >> $GITHUB_ENV
	# Reset other flags
	echo "ENABLE_QUERYREWRITER=False" >> $GITHUB_ENV
	echo "CONVERSATION_HISTORY=0" >> $GITHUB_ENV

	- name: Restart services for multimodal query tests
	env:
	CI_NVSTAGING_BLUEPRINT_KEY: ${{ secrets.CI_NVSTAGING_BLUEPRINT_KEY }}
	run: \|
	echo "Restarting services with multimodal query configuration..."
	docker compose -f deploy/compose/docker-compose-rag-server.yaml down \|\| true
	docker compose -f deploy/compose/docker-compose-ingestor-server.yaml down \|\| true
	sleep 5
	docker compose -f deploy/compose/docker-compose-rag-server.yaml up -d --build
	docker compose -f deploy/compose/docker-compose-ingestor-server.yaml up -d --build
	sleep 30
	docker ps

	- name: Run multimodal query integration tests
	id: multimodal-query-tests
	continue-on-error: true
	run: \|
	source venv/bin/activate
	echo "Running multimodal query integration tests..."
	python -m tests.integration.main --sequence multimodal_query
	echo "Multimodal query integration tests completed"

	- name: Collect logs after multimodal query tests
	if: always()
	run: \|
	mkdir -p logs/multimodal-query
	docker logs rag-server > logs/multimodal-query/rag-server.log 2>&1 \|\| true
	docker logs ingestor-server > logs/multimodal-query/ingestor-server.log 2>&1 \|\| true
	docker logs compose-nv-ingest-ms-runtime-1 > logs/multimodal-query/nvingest.log 2>&1 \|\| true
	cp tests/integration/integration_test.log logs/multimodal-query/ 2>/dev/null \|\| true

	# ========================================================================
	# CUSTOM PROMPT TESTS
	# ========================================================================

	- name: Configure environment for custom prompt tests
	run: \|
	echo "PROMPT_CONFIG_FILE=$(pwd)/tests/data/test_prompt.yaml" >> $GITHUB_ENV
	echo "ENABLE_VLM_INFERENCE=False" >> $GITHUB_ENV
	echo "ENABLE_QUERYREWRITER=False" >> $GITHUB_ENV
	echo "CONVERSATION_HISTORY=0" >> $GITHUB_ENV

	- name: Restart services for custom prompt tests
	env:
	CI_NVSTAGING_BLUEPRINT_KEY: ${{ secrets.CI_NVSTAGING_BLUEPRINT_KEY }}
	run: \|
	echo "Restarting services with custom prompt configuration..."
	docker compose -f deploy/compose/docker-compose-rag-server.yaml down \|\| true
	docker compose -f deploy/compose/docker-compose-ingestor-server.yaml down \|\| true
	sleep 5
	docker compose -f deploy/compose/docker-compose-rag-server.yaml up -d --build
	docker compose -f deploy/compose/docker-compose-ingestor-server.yaml up -d --build
	sleep 30
	docker ps

	- name: Run custom prompt integration tests
	id: custom-prompt-tests
	continue-on-error: true
	run: \|
	source venv/bin/activate
	echo "Running custom prompt integration tests..."
	python -m tests.integration.main --sequence custom_prompt
	echo "Custom prompt integration tests completed"

	- name: Collect logs after custom prompt tests
	if: always()
	run: \|
	mkdir -p logs/custom-prompt
	docker logs rag-server > logs/custom-prompt/rag-server.log 2>&1 \|\| true
	docker logs ingestor-server > logs/custom-prompt/ingestor-server.log 2>&1 \|\| true
	docker logs compose-nv-ingest-ms-runtime-1 > logs/custom-prompt/nvingest.log 2>&1 \|\| true
	cp tests/integration/integration_test.log logs/custom-prompt/ 2>/dev/null \|\| true

	# ========================================================================
	# LIBRARY USAGE TESTS
	# ========================================================================

	- name: Stop rag-server and ingestor-server for library tests
	run: \|
	echo "Stopping rag-server and ingestor-server containers (library mode doesn't need them)..."
	echo "Keeping nv-ingest-ms-runtime, Milvus, Redis, MinIO running for library mode..."
	docker stop rag-server \|\| true
	docker stop ingestor-server \|\| true
	echo "Services stopped. Library tests will use the nvidia_rag library directly."
	echo "Remaining services (needed for library mode):"
	docker ps

	- name: Configure environment for library usage tests
	run: \|
	echo "PROMPT_CONFIG_FILE=${PWD}/src/nvidia_rag/rag_server/prompt.yaml" >> $GITHUB_ENV
	echo "ENABLE_QUERYREWRITER=False" >> $GITHUB_ENV
	echo "CONVERSATION_HISTORY=0" >> $GITHUB_ENV

	- name: Run library usage integration tests
	id: library-usage-tests
	continue-on-error: true
	run: \|
	source venv/bin/activate
	echo "Running library usage integration tests..."
	python -m tests.integration.main --sequence library_usage
	echo "Library usage integration tests completed"

	- name: Collect logs after library usage tests
	if: always()
	run: \|
	mkdir -p logs/library-usage
	docker logs compose-nv-ingest-ms-runtime-1 > logs/library-usage/nvingest.log 2>&1 \|\| true
	cp tests/integration/integration_test.log logs/library-usage/ 2>/dev/null \|\| true

	# ========================================================================
	# LIBRARY SUMMARIZATION TESTS
	# ========================================================================

	- name: Run library summarization integration tests
	id: library-summarization-tests
	continue-on-error: true
	run: \|
	source venv/bin/activate
	echo "Running library summarization integration tests..."
	python -m tests.integration.main --sequence library_summarization
	echo "Library summarization integration tests completed"

	- name: Collect logs after library summarization tests
	if: always()
	run: \|
	mkdir -p logs/library-summarization
	docker logs compose-nv-ingest-ms-runtime-1 > logs/library-summarization/nvingest.log 2>&1 \|\| true
	cp tests/integration/integration_test.log logs/library-summarization/ 2>/dev/null \|\| true

	# ========================================================================
	# OBSERVABILITY TESTS
	# ========================================================================

	- name: Configure environment for observability tests
	run: \|
	echo "APP_TRACING_ENABLED=True" >> $GITHUB_ENV
	echo "OPENTELEMETRY_CONFIG_FILE=$(pwd)/deploy/config/otel-collector-config.yaml" >> $GITHUB_ENV
	echo "PROMPT_CONFIG_FILE=${PWD}/src/nvidia_rag/rag_server/prompt.yaml" >> $GITHUB_ENV
	echo "ENABLE_QUERYREWRITER=False" >> $GITHUB_ENV
	echo "CONVERSATION_HISTORY=0" >> $GITHUB_ENV

	- name: Restart services for observability tests
	env:
	CI_NVSTAGING_BLUEPRINT_KEY: ${{ secrets.CI_NVSTAGING_BLUEPRINT_KEY }}
	run: \|
	echo "Restarting services with observability configuration..."
	echo "(rag-server and ingestor-server were stopped for library tests, now restarting)"
	docker compose -f tests/integration/vectordb.yaml down -v \|\| true
	docker compose -f deploy/compose/docker-compose-rag-server.yaml down \|\| true
	docker compose -f deploy/compose/docker-compose-ingestor-server.yaml down \|\| true
	sleep 5
	docker compose -f tests/integration/vectordb.yaml up -d \|\| true
	docker compose -f deploy/compose/docker-compose-rag-server.yaml up -d --build
	docker compose -f deploy/compose/docker-compose-ingestor-server.yaml up -d --build
	sleep 30
	docker ps

	- name: Start observability services
	run: \|
	echo "Starting observability services..."
	docker compose -f deploy/compose/observability.yaml up -d otel-collector prometheus
	sleep 30

	- name: Run observability integration tests
	id: observability-tests
	continue-on-error: true
	run: \|
	source venv/bin/activate
	echo "Running observability integration tests..."
	python -m tests.integration.main --sequence observability
	echo "Observability integration tests completed"

	- name: Collect logs after observability tests
	if: always()
	run: \|
	mkdir -p logs/observability
	echo "=== Container Status (docker ps -a) ===" \| tee logs/observability/container-status.log
	docker ps -a --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}" \| tee -a logs/observability/container-status.log
	echo "" \| tee -a logs/observability/container-status.log
	echo "=== Milvus Container Stats ===" \| tee -a logs/observability/container-status.log
	docker stats --no-stream milvus-standalone 2>&1 \| tee -a logs/observability/container-status.log \|\| echo "Milvus container not running" \| tee -a logs/observability/container-status.log
	docker logs rag-server > logs/observability/rag-server.log 2>&1 \|\| true
	docker logs ingestor-server > logs/observability/ingestor-server.log 2>&1 \|\| true
	docker logs compose-nv-ingest-ms-runtime-1 > logs/observability/nvingest.log 2>&1 \|\| true
	docker logs milvus-standalone > logs/observability/milvus.log 2>&1 \|\| true
	docker logs otel-collector > logs/observability/otel-collector.log 2>&1 \|\| true
	docker logs zipkin > logs/observability/zipkin.log 2>&1 \|\| true
	docker logs prometheus > logs/observability/prometheus.log 2>&1 \|\| true
	docker logs grafana-service > logs/observability/grafana.log 2>&1 \|\| true
	cp tests/integration/integration_test.log logs/observability/ 2>/dev/null \|\| true

	- name: Stop observability services
	if: always()
	run: \|
	docker compose -f deploy/compose/observability.yaml down \|\| true

	# ========================================================================
	# MILVUS VDB AUTH VIA REST TESTS
	# ========================================================================

	- name: Configure Milvus VDB auth
	run: \|
	echo "Configuring Milvus authentication and override compose..."
	# Set auth credentials for the servers (running in Docker)
	echo "APP_VECTORSTORE_USERNAME=root" >> $GITHUB_ENV
	echo "APP_VECTORSTORE_PASSWORD=Milvus" >> $GITHUB_ENV
	# Set auth token for the integration tests (running on host)
	echo "VDB_AUTH_TOKEN=root:Milvus" >> $GITHUB_ENV
	echo "MILVUS_ROOT_TOKEN=root:Milvus" >> $GITHUB_ENV

	# Stop and clean up existing Milvus containers and volumes
	echo "Stopping and cleaning up existing Milvus containers..."
	docker compose -f tests/integration/vectordb.yaml down -v \|\| true

	# Remove the old data directories to ensure clean start with auth
	echo "Cleaning up old Milvus data directories..."
	sudo rm -rf /tmp/milvus-${MILVUS_VERSION}/volumes/milvus \|\| true
	sudo rm -rf /tmp/milvus-${MILVUS_VERSION}/volumes/etcd \|\| true

	# Ensure milvus.yaml exists in tests/integration by copying from running container
	# Since we stopped the container, we need to start a temporary one to extract config
	mkdir -p tests/integration
	if [ ! -f tests/integration/milvus.yaml ]; then
	echo "Creating temporary Milvus container to extract config..."
	docker run --rm -d --name milvus-temp milvusdb/milvus:${MILVUS_VERSION:-v2.6.5-gpu} sleep 60
	docker cp milvus-temp:/milvus/configs/milvus.yaml tests/integration/milvus.yaml
	docker stop milvus-temp \|\| true
	fi

	# Update authentication settings in milvus.yaml
	sed -i 's/authorizationEnabled:.*/authorizationEnabled: true/' tests/integration/milvus.yaml
	sed -i 's/defaultRootPassword:.*/defaultRootPassword: Milvus/' tests/integration/milvus.yaml
	echo "MILVUS_CONFIG_FILE=$(pwd)/tests/integration/milvus.yaml" >> $GITHUB_ENV

	# Verify the changes
	echo "Verifying milvus.yaml authentication settings:"
	grep -A2 "security:" tests/integration/milvus.yaml \| head -5

	# Update vectordb.yaml to comment out data volume and enable config volume
	sed -i 's\|- \${DOCKER_VOLUME_DIRECTORY:-.}/volumes/milvus:/var/lib/milvus\|# - ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/milvus:/var/lib/milvus\|' tests/integration/vectordb.yaml
	sed -i 's\|# - \${MILVUS_CONFIG_FILE:-./milvus.yaml}:/milvus/configs/milvus.yaml\|- ${MILVUS_CONFIG_FILE:-./milvus.yaml}:/milvus/configs/milvus.yaml\|' tests/integration/vectordb.yaml

	- name: Restart vector database with auth
	run: \|
	echo "Starting vector database with authentication enabled..."
	docker compose -f tests/integration/vectordb.yaml up -d
	echo "Waiting for Milvus services to be ready..."
	sleep 60
	docker ps
	echo "Checking Milvus logs..."
	docker logs --tail 100 milvus-standalone \|\| true
	docker logs --tail 50 milvus-etcd \|\| true
	docker logs --tail 50 milvus-minio \|\| true

	- name: Restart services for Milvus VDB auth tests
	env:
	CI_NVSTAGING_BLUEPRINT_KEY: ${{ secrets.CI_NVSTAGING_BLUEPRINT_KEY }}
	run: \|
	echo "Restarting rag/ingestor services to pick up Milvus auth configuration..."
	docker compose -f deploy/compose/docker-compose-rag-server.yaml down \|\| true
	docker compose -f deploy/compose/docker-compose-ingestor-server.yaml down \|\| true
	sleep 5
	docker compose -f deploy/compose/docker-compose-rag-server.yaml up -d --build
	docker compose -f deploy/compose/docker-compose-ingestor-server.yaml up -d --build
	sleep 30
	docker ps

	- name: Run Milvus VDB Auth tests
	id: milvus-vdb-auth-tests
	continue-on-error: true
	env:
	APP_VECTORSTORE_URL: http://localhost:19530
	run: \|
	source venv/bin/activate
	echo "Running Milvus VDB Auth tests..."
	echo "APP_VECTORSTORE_URL set to: $APP_VECTORSTORE_URL"
	python -m tests.integration.main --sequence milvus_vdb_auth_through_rest_api
	echo "Milvus VDB Auth tests completed"

	- name: Collect logs after Milvus VDB auth tests
	if: always()
	run: \|
	mkdir -p logs/milvus-vdb-auth
	docker logs rag-server > logs/milvus-vdb-auth/rag-server.log 2>&1 \|\| true
	docker logs ingestor-server > logs/milvus-vdb-auth/ingestor-server.log 2>&1 \|\| true
	docker logs milvus-standalone > logs/milvus-vdb-auth/milvus.log 2>&1 \|\| true
	docker logs milvus-etcd > logs/milvus-vdb-auth/etcd.log 2>&1 \|\| true
	docker logs milvus-minio > logs/milvus-vdb-auth/minio.log 2>&1 \|\| true
	cp tests/integration/integration_test.log logs/milvus-vdb-auth/ 2>/dev/null \|\| true

	- name: Revert Milvus VDB auth configurations
	if: always()
	run: \|
	echo "Reverting Milvus VDB auth configurations..."
	# Stop Milvus with auth enabled
	docker compose -f tests/integration/vectordb.yaml down -v \|\| true

	# Clean up auth-specific data to ensure fresh start
	sudo rm -rf /tmp/milvus-${MILVUS_VERSION}/volumes/milvus \|\| true
	sudo rm -rf /tmp/milvus-${MILVUS_VERSION}/volumes/etcd \|\| true

	# Remove the auth config file
	rm -f tests/integration/milvus.yaml \|\| true

	# Revert vectordb.yaml to original state
	sed -i 's\|- \${MILVUS_CONFIG_FILE:-./milvus.yaml}:/milvus/configs/milvus.yaml\|# - ${MILVUS_CONFIG_FILE:-./milvus.yaml}:/milvus/configs/milvus.yaml\|' tests/integration/vectordb.yaml
	sed -i 's\|# - \${DOCKER_VOLUME_DIRECTORY:-.}/volumes/milvus:/var/lib/milvus\|- \${DOCKER_VOLUME_DIRECTORY:-.}/volumes/milvus:/var/lib/milvus\|' tests/integration/vectordb.yaml

	# Restart Milvus without auth for subsequent tests
	echo "Restarting Milvus without authentication for subsequent tests..."
	docker compose -f tests/integration/vectordb.yaml up -d
	sleep 30

	# Unset auth environment variables
	echo "APP_VECTORSTORE_USERNAME=" >> $GITHUB_ENV
	echo "APP_VECTORSTORE_PASSWORD=" >> $GITHUB_ENV
	echo "VDB_AUTH_TOKEN=" >> $GITHUB_ENV
	echo "MILVUS_ROOT_TOKEN=" >> $GITHUB_ENV

	# ========================================================================
	# FAIL JOB IF ANY INTEGRATION TEST FAILED
	# ========================================================================
	# All test steps use continue-on-error so every suite runs; this step
	# marks the job as failed if any of them failed.
	- name: Fail job if any integration test failed
	if: always() && (steps.basic-tests.outcome == 'failure' \|\| steps.query-rewriter-tests.outcome == 'failure' \|\| steps.reflection-tests.outcome == 'failure' \|\| steps.guardrails-tests.outcome == 'failure' \|\| steps.image-captioning-tests.outcome == 'failure' \|\| steps.vlm-generation-tests.outcome == 'failure' \|\| steps.multimodal-query-tests.outcome == 'failure' \|\| steps.custom-prompt-tests.outcome == 'failure' \|\| steps.library-usage-tests.outcome == 'failure' \|\| steps.library-summarization-tests.outcome == 'failure' \|\| steps.observability-tests.outcome == 'failure' \|\| steps.milvus-vdb-auth-tests.outcome == 'failure')
	run: \|
	echo "=== Failed integration test suites ==="
	[ "${{ steps.basic-tests.outcome }}" = "failure" ] && echo " - basic-tests"
	[ "${{ steps.query-rewriter-tests.outcome }}" = "failure" ] && echo " - query-rewriter-tests"
	[ "${{ steps.reflection-tests.outcome }}" = "failure" ] && echo " - reflection-tests"
	[ "${{ steps.guardrails-tests.outcome }}" = "failure" ] && echo " - guardrails-tests"
	[ "${{ steps.image-captioning-tests.outcome }}" = "failure" ] && echo " - image-captioning-tests"
	[ "${{ steps.vlm-generation-tests.outcome }}" = "failure" ] && echo " - vlm-generation-tests"
	[ "${{ steps.multimodal-query-tests.outcome }}" = "failure" ] && echo " - multimodal-query-tests"
	[ "${{ steps.custom-prompt-tests.outcome }}" = "failure" ] && echo " - custom-prompt-tests"
	[ "${{ steps.library-usage-tests.outcome }}" = "failure" ] && echo " - library-usage-tests"
	[ "${{ steps.library-summarization-tests.outcome }}" = "failure" ] && echo " - library-summarization-tests"
	[ "${{ steps.observability-tests.outcome }}" = "failure" ] && echo " - observability-tests"
	[ "${{ steps.milvus-vdb-auth-tests.outcome }}" = "failure" ] && echo " - milvus-vdb-auth-tests"
	echo "One or more integration test suites failed. Failing job."
	exit 1

	# ========================================================================
	# UPLOAD ALL LOGS
	# ========================================================================

	- name: Sanitize branch name for artifacts
	if: always()
	id: sanitize
	run: \|
	SANITIZED_REF="${GITHUB_REF_NAME//\//-}"
	echo "ref_name=$SANITIZED_REF" >> $GITHUB_OUTPUT

	- name: Upload all integration test logs
	uses: actions/upload-artifact@v4
	if: always()
	with:
	name: integration-tests-logs-${{ steps.sanitize.outputs.ref_name }}-${{ github.sha }}
	path: logs/
	retention-days: 7

	# ========================================================================
	# CLEANUP
	# ========================================================================

	- name: Cleanup virtual environment
	if: always()
	run: \|
	echo "Cleaning up virtual environment..."
	rm -rf venv
	echo "Virtual environment cleanup completed"

	- name: Cleanup Docker containers and volumes
	if: always()
	run: \|
	echo "Cleaning up integration test environment..."
	# Bring down only the specific compose stacks used in this workflow,
	# and clean up their containers, networks, and locally built images.
	docker compose -f deploy/compose/docker-compose-ingestor-server.yaml down -v --remove-orphans --rmi local \|\| true
	docker compose -f deploy/compose/docker-compose-rag-server.yaml down -v --remove-orphans --rmi local \|\| true
	docker compose -f tests/integration/vectordb.yaml down -v --remove-orphans --rmi local \|\| true
	echo "Cleanup completed for integration test Docker resources"

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Kheiss/cont ingest #1734

Workflow file

Kheiss/cont ingest #1734

Uh oh!

Workflow file for this run