Issue #23 - add test suite for CleverBRAG
Unit test coverage / pytest (push) Successful in 1m6s
Unit test coverage / pytest (pull_request) Successful in 1m11s

This commit is contained in:
Stanislav Hejny
2025-06-02 12:18:14 +01:00
parent 7e04106ab4
commit 748af5a3d7
10 changed files with 1600 additions and 0 deletions
+149
View File
@@ -0,0 +1,149 @@
echo "#!/bin/bash"
. ./test_shared_functions.sh
extract_document_id() {
R=$1
C=$(echo $R | sed 's/[0-9][0-9][0-9]$//')
#echo ">>${C}<<"
if [[ ! -z "$C" ]] && [[ "$C" == *results* ]]; then
COLLECTION_IDS=($(echo $C | jq -r '.results[] | select(.document_count >= 1) | .id' | tr '\n' ' '))
echo "CIDs = ${COLLECTION_IDS[*]}"
else
echo "No IDs detected in the response."
fi
HTTP_RETURN_CODE=$(echo echo $1 | tail -1 | sed -e 's/^.*\([0-9][0-9][0-9]\)$/\1/')
}
echo "# 1. Create Collection Tests"
echo "=== Testing Collection Creation ==="
echo "# Test 1.1: Create collection with name only"
make_request "POST" "/collections" \
"-d '{\"name\":\"Test Collection\"}'"
echo "# Test 1.2: Create collection with name and description"
make_request "POST" "/collections" \
"-d '{\"name\":\"Test Collection\",\"description\":\"A test collection\"}'"
echo "# Test 1.3: Create collection with empty name (should fail)"
make_request "POST" "/collections" \
"-d '{\"name\":\"\"}'"
echo "# Test 1.4: Create collection with very long name (edge case)"
make_request "POST" "/collections" \
"-d '{\"name\":\"$(printf 'a%.0s' {1..1000})\"}'"
echo "# 2. List Collections Tests"
echo "=== Testing Collection Listing ==="
echo "# Test 2.1: List all collections"
make_request "GET" "/collections" ""
ALL_COLLECTION_IDS=("${COLLECTION_IDS[@]}")
echo "# Test 2.2: List collections with pagination"
make_request "GET" "/collections?offset=0&limit=10" ""
echo "# Test 2.3: List collections with invalid pagination (edge cases)"
make_request "GET" "/collections?offset=-1&limit=10" ""
make_request "GET" "/collections?offset=0&limit=0" ""
make_request "GET" "/collections?offset=0&limit=1001" ""
echo "# Test 2.4: List specific collections by IDs"
# Join array elements with commas
COMMA_SEPARATED_IDS=$(echo "${ALL_COLLECTION_IDS[*]}" | sed -e "s/ /,/")
make_request "GET" "/collections?ids=${COMMA_SEPARATED_IDS}" ""
echo "# 3. Get Collection Tests"
echo "=== Testing Collection Retrieval ==="
echo "# Test 3.1: Get collection by ID"
make_request "GET" "/collections/${ALL_COLLECTION_IDS[0]}" ""
echo "# Test 3.2: Get non-existent collection"
make_request "GET" "/collections/00000000-0000-0000-0000-000000000000" ""
echo "# Test 3.3: Get collection with invalid UUID format"
make_request "GET" "/collections/invalid-uuid" ""
echo "# 4. Update Collection Tests"
echo "=== Testing Collection Updates ==="
echo "# Test 4.1: Update collection name"
make_request "POST" "/collections/${ALL_COLLECTION_IDS[0]}" \
"-d '{\"name\":\"Updated Collection Name\"}'"
echo "# Test 4.2: Update collection description"
make_request "POST" "/collections/${ALL_COLLECTION_IDS[0]}" \
"-d '{\"description\":\"Updated description\"}'"
echo "# Test 4.3: Update both name and description"
make_request "POST" "/collections/${ALL_COLLECTION_IDS[0]}" \
"-d '{\"name\":\"Updated Name\",\"description\":\"Updated description\"}'"
echo "# Test 4.4: Generate description"
make_request "POST" "/collections/${ALL_COLLECTION_IDS[0]}" \
"-d '{\"generate_description\":true}'"
echo "# Test 4.5: Invalid update (both description and generate_description)"
make_request "POST" "/collections/${ALL_COLLECTION_IDS[0]}" \
"-d '{\"description\":\"New description\",\"generate_description\":true}'"
echo "# 5. Delete Collection Tests"
echo "=== Testing Collection Deletion ==="
echo "# Test 5.1: Delete existing collection"
make_request "DELETE" "/collections/${ALL_COLLECTION_IDS[0]}" ""
echo "# Test 5.2: Delete non-existent collection"
make_request "DELETE" "/collections/00000000-0000-0000-0000-000000000000" ""
echo "# 6. Document Management Tests"
echo "=== Testing Document Management ==="
echo "# Test 6.1: Add document to collection"
make_request "POST" "/collections/${ALL_COLLECTION_IDS[0]}/documents/456e789a-b12c-34d5-e678-901234567890" ""
echo "# Test 6.2: List documents in collection"
make_request "GET" "/collections/${ALL_COLLECTION_IDS[0]}/documents?offset=0&limit=10" ""
echo "# Test 6.3: List documents with invalid pagination"
make_request "GET" "/collections/${ALL_COLLECTION_IDS[0]}/documents?offset=-1&limit=10" ""
make_request "GET" "/collections/${ALL_COLLECTION_IDS[0]}/documents?offset=0&limit=1001" ""
echo "# Test 6.4: Remove document from collection"
make_request "DELETE" "/collections/${ALL_COLLECTION_IDS[0]}/documents/456e789a-b12c-34d5-e678-901234567890" ""
echo "# 7. User Management Tests"
echo "=== Testing User Management ==="
echo "# Test 7.1: List users in collection"
make_request "GET" "/collections/${ALL_COLLECTION_IDS[0]}/users?offset=0&limit=10" ""
echo "# Test 7.2: List users with invalid pagination"
make_request "GET" "/collections/${ALL_COLLECTION_IDS[0]}/users?offset=-1&limit=10" ""
make_request "GET" "/collections/${ALL_COLLECTION_IDS[0]}/users?offset=0&limit=1001" ""
echo "# Test 7.3: Add user to collection"
make_request "POST" "/collections/${ALL_COLLECTION_IDS[0]}/users/789a012b-c34d-5e6f-a789-012345678901" ""
echo "# Test 7.4: Remove user from collection"
make_request "DELETE" "/collections/${ALL_COLLECTION_IDS[0]}/users/789a012b-c34d-5e6f-a789-012345678901" ""
echo "# 8. Entity Extraction Tests"
echo "=== Testing Entity Extraction ==="
echo "# Test 8.1: Extract entities with default settings"
make_request "POST" "/collections/${ALL_COLLECTION_IDS[0]}/extract" \
"-d '{\"run_type\":\"RUN\"}'"
echo "# Test 8.2: Extract entities with custom settings"
make_request "POST" "/collections/${ALL_COLLECTION_IDS[0]}/extract" \
"-d '{\"run_type\":\"RUN\",\"settings\":{\"model\":\"gpt-4\"}}'"
echo "# Test 8.3: Extract entities without orchestration"
make_request "POST" "/collections/${ALL_COLLECTION_IDS[0]}/extract" \
"-d '{\"run_type\":\"RUN\",\"run_with_orchestration\":false}'"
echo "# Test 8.4: Get estimate only"
make_request "POST" "/collections/${ALL_COLLECTION_IDS[0]}/extract" \
"-d '{\"run_type\":\"ESTIMATE\"}'"
File diff suppressed because one or more lines are too long
@@ -0,0 +1,42 @@
On the first part of the journey
I was looking at all the life
There were plants and birds and rocks and things
There was sand and hills and rings
The first thing I met was a fly with a buzz
And the sky with no clouds
The heat was hot and the ground was dry
But the air was full of sound
I've been through the desert on a horse with no name
It felt good to be out of the rain
In the desert you can remember your name
'Cause there ain't no one for to give you no pain
La, la ...
After two days in the desert sun
My skin began to turn red
After three days in the desert fun
I was looking at a river bed
And the story it told of a river that flowed
Made me sad to think it was dead
You see I've been through the desert on a horse with no name
It felt good to be out of the rain
In the desert you can remember your name
'Cause there ain't no one for to give you no pain
La, la ...
After nine days I let the horse run free
'Cause the desert had turned to sea
There were plants and birds and rocks and things
there was sand and hills and rings
The ocean is a desert with it's life underground
And a perfect disguise above
Under the cities lies a heart made of ground
But the humans will give no love
You see I've been through the desert on a horse with no name
It felt good to be out of the rain
In the desert you can remember your name
'Cause there ain't no one for to give you no pain
La, la ...
Binary file not shown.

After

Width:  |  Height:  |  Size: 201 KiB

Binary file not shown.
@@ -0,0 +1 @@
[{"@id": "data:text/plain;charset=US-ASCII,instituto%20superior%20tecnico", "@type": "http://www.cleverthis.com/university#Organization"}, {"@id": "data:text/plain;charset=US-ASCII,lisbon", "@type": "http://www.cleverthis.com/university#City"}, {"@id": "data:text/plain;charset=US-ASCII,lisbon", "@type": "http://www.cleverthis.com/university#Campus"}, {"@id": "data:text/plain;charset=US-ASCII,loures", "@type": "http://www.cleverthis.com/university#Campus"}, {"@id": "data:text/plain;charset=US-ASCII,Rogerio%20colaco", "@type": "http://www.cleverthis.com/university#Person"}, {"@id": "data:text/plain;charset=US-ASCII,1049-001%20lisboa", "@type": "data:,"}, {"@id": "data:text/plain;charset=US-ASCII,Instituto%20superior%20tecnico", "@type": "http://www.cleverthis.com/university#University", "http://www.cleverthis.com/university#campus": {"@id": "data:text/plain;charset=US-ASCII,three%20campuses", "@type": "http://www.cleverthis.com/university#Campus"}, "data:,": {"@id": "data:text/plain;charset=US-ASCII,tecnico", "@type": "http://www.cleverthis.com/university#Name"}, "http://www.cleverthis.com/university#country": {"@id": "data:text/plain;charset=US-ASCII,portugal", "@type": "http://www.cleverthis.com/university#Country"}}, {"@id": "data:text/plain;charset=US-ASCII,taguspark", "@type": "http://www.cleverthis.com/university#Campus"}, {"@id": "data:text/plain;charset=US-ASCII,oeiras", "@type": "http://www.cleverthis.com/university#City"}, {"@id": "data:text/plain;charset=US-ASCII,alameda", "@type": "http://www.cleverthis.com/university#Campus"}]
+180
View File
@@ -0,0 +1,180 @@
#!/bin/bash
. ./test_shared_functions.sh
JSON_FILE=./test_data/Instituto_Superior_Tecnico.md.json
HTML_FILE=./test_data/AMERICA-VENTURE_HIGHWAY.html
MP3_FILE=./test_data/flight_of_the_bumblebee_2.mp3
JPG_FILE=./test_data/China_Beijing_Summer_Palace2.jpg
EXCEL_FILE=./test_data/Worldbank-Datasheet-02082022.xlsx
TEXT_FILE=./test_data/America-Horse_with_no_name.txt
OPEN_OFFICE_FILE=./test_data/GAR_ARchitecture.odg
echo "# 1. Create Document Tests"
echo "=== Testing Document Creation ==="
echo "# Test 1.1: Create document with file, use different formats"
make_request "POST" "/documents" \
"-F file=@${TEXT_FILE}" \
"-H 'Content-Type: multipart/form-data'"
DOCUMENT_ID_FROM_FILE=${DOCUMENT_ID}
make_request "POST" "/documents" \
"-F file=@${JSON_FILE}" \
"-H 'Content-Type: multipart/form-data'"
DOCUMENT_ID_FROM_JSON_FILE=${DOCUMENT_ID}
make_request "POST" "/documents" \
"-F file=@${HTML_FILE}" \
"-H 'Content-Type: multipart/form-data'"
DOCUMENT_ID_FROM_HTML_FILE=${DOCUMENT_ID}
make_request "POST" "/documents" \
"-F file=@${MP3_FILE}" \
"-H 'Content-Type: multipart/form-data'"
DOCUMENT_ID_FROM_MP3_FILE=${DOCUMENT_ID}
make_request "POST" "/documents" \
"-F file=@${JPG_FILE}" \
"-H 'Content-Type: multipart/form-data'"
DOCUMENT_ID_FROM_JPG_FILE=${DOCUMENT_ID}
make_request "POST" "/documents" \
"-F file=@${EXCEL_FILE}" \
"-H 'Content-Type: multipart/form-data'"
DOCUMENT_ID_FROM_EXCEL_FILE=${DOCUMENT_ID}
make_request "POST" "/documents" \
"-F file=@${OPEN_OFFICE_FILE}" \
"-H 'Content-Type: multipart/form-data'"
DOCUMENT_ID_FROM_OPEN_OFFICE_FILE=${DOCUMENT_ID}
echo "# Test 1.2: Create document with raw text"
make_request "POST" "/documents" \
'-F raw_text="On the first part of the journey
I was looking at all the life
There were plants and birds and rocks and things
There was sand and hills and rings
The first thing I met was a fly with a buzz
And the sky with no clouds
The heat was hot and the ground was dry
But the air was full of sound"' \
"-H 'Content-Type: multipart/form-data'"
DOCUMENT_ID_FROM_RAW_TEXT=${DOCUMENT_ID}
echo "# Test 1.3: Create document with chunks"
make_request "POST" "/documents" \
"-F chunks=\"[\\\"I've been through the desert on a horse with no name\\\",\\\"It felt good to be out of the rain\\\"]\"" \
"-H 'Content-Type: multipart/form-data'"
DOCUMENT_ID_FROM_CHUNKS=${DOCUMENT_ID}
echo "# Test 1.4: Create document with metadata"
make_request "POST" "/documents" \
"-F file=@${TEXT_FILE} -F metadata='{\"title\":\"Horse with No Name\",\"author\":\"America\"}'" \
"-H 'Content-Type: multipart/form-data'"
DOCUMENT_ID_FROM_FILE_WITH_METADATA=${DOCUMENT_ID}
echo "# Test 1.5: Create document with custom ingestion config"
make_request "POST" "/documents" \
"-F file=@${TEXT_FILE} -F ingestion_mode=hi-res -F ingestion_config='{\"chunk_size\":1000}'" \
"-H 'Content-Type: multipart/form-data'"
DOCUMENT_ID_FROM_FILE_WITH_CUSTOM_INGEST=${DOCUMENT_ID}
echo "# 2. List Documents Tests"
echo "=== Testing Document Listing ==="
echo "# Test 2.1: List all documents"
make_request "GET" "/documents" ""
echo "# Test 2.2: List documents with pagination"
make_request "GET" "/documents?offset=0&limit=10" ""
echo "# Test 2.3: List specific documents by IDs"
make_request "GET" "/documents?ids=${DOCUMENT_ID_FROM_CHUNKS},${DOCUMENT_ID_FROM_FILE_WITH_METADATA}" ""
echo "# Test 2.4: List documents with summary embeddings"
make_request "GET" "/documents?include_summary_embeddings=1" ""
echo "# 3. Get Document Tests"
echo "=== Testing Document Retrieval ==="
echo "# Test 3.1: Get document by ID"
make_request "GET" "/documents/${DOCUMENT_ID_FROM_FILE_WITH_METADATA}" ""
echo "# Test 3.2: Get non-existent document"
make_request "GET" "/documents/00000000-0000-0000-0000-000000000000" ""
echo "# 4. List Document Chunks Tests"
echo "=== Testing Document Chunks ==="
echo "# Test 4.1: List chunks with pagination"
make_request "GET" "/documents/${DOCUMENT_ID_FROM_CHUNKS}/chunks?offset=0&limit=10" ""
echo "# Test 4.2: List chunks with vectors"
make_request "GET" "/documents/${DOCUMENT_ID_FROM_CHUNKS}/chunks?include_vectors=true" ""
echo "# 5. Download Document Tests"
echo "=== Testing Document Download ==="
echo "# Test 5.1: Download document"
make_request "GET" "/documents/${DOCUMENT_ID_FROM_FILE}/download" ""
echo "# 6. Delete Document Tests"
echo "=== Testing Document Deletion ==="
echo "# Test 6.1: Delete document by ID"
make_request "DELETE" "/documents/${DOCUMENT_ID_FROM_RAW_TEXT}" ""
echo "# Test 6.2: Delete documents by filter"
make_request "DELETE" "/documents/by-filter" \
"-d '{\"document_type\":{\"\$eq\":\"txt\"}}'"
echo "# 7. Document Collections Tests"
echo "=== Testing Document Collections ==="
echo "# Test 7.1: List document collections"
make_request "GET" "/documents/${DOCUMENT_ID_FROM_CHUNKS}/collections?offset=0&limit=10" ""
echo "# 8. Entity Extraction Tests"
echo "=== Testing Entity Extraction ==="
echo "# Test 8.1: Extract entities with default settings"
make_request "POST" "/documents/${DOCUMENT_ID_FROM_FILE_WITH_METADATA}/extract" \
"-d '{\"run_type\":\"RUN\"}'"
echo "# Test 8.2: Extract entities with custom settings"
make_request "POST" "/documents/${DOCUMENT_ID_FROM_FILE_WITH_METADATA}/extract" \
"-d '{\"run_type\":\"RUN\",\"settings\":{\"model\":\"gpt-4\"}}'"
echo "# 9. Entity Retrieval Tests"
echo "=== Testing Entity Retrieval ==="
echo "# Test 9.1: Get entities with pagination"
make_request "GET" "/documents/${DOCUMENT_ID_FROM_FILE_WITH_METADATA}/entities?offset=0&limit=10" ""
echo "# Test 9.2: Get entities with embeddings"
make_request "GET" "/documents/${DOCUMENT_ID_FROM_FILE_WITH_METADATA}/entities?include_embeddings=true" ""
echo "# 10. Relationship Tests"
echo "=== Testing Relationships ==="
echo "# Test 10.1: Get relationships with pagination"
make_request "GET" "/documents/${DOCUMENT_ID_FROM_FILE_WITH_METADATA}/relationships?offset=0&limit=10" ""
echo "# Test 10.2: Get relationships with filters"
make_request "GET" "/documents/${DOCUMENT_ID_FROM_FILE_WITH_METADATA}/relationships?entity_names=person,company&relationship_types=works_for" ""
echo "# 11. Document Search Tests"
echo "=== Testing Document Search ==="
echo "# Test 11.1: Basic search"
make_request "POST" "/documents/search" \
"-d '{\"query\":\"test query\",\"search_mode\":\"basic\"}'"
echo "# Test 11.2: Advanced search with custom settings"
make_request "POST" "/documents/search" \
"-d '{\"query\":\"test query\",\"search_mode\":\"advanced\",\"search_settings\":{\"filters\":{\"document_type\":\"pdf\"}}}'"
echo "# Test 11.3: Custom search with complex filters"
make_request "POST" "/documents/search" \
"-d '{\"query\":\"test query\",\"search_mode\":\"custom\",\"search_settings\":{\"filters\":{\"created_at\":{\"\$gt\":\"2023-01-01\"}}}}'"
+98
View File
@@ -0,0 +1,98 @@
#!/bin/bash
#
# Generates random valid UUID
#
generate_uuid() {
local N B C='89ab'
local RES=''
for (( N=0; N < 16; ++N )) do
B=$(( $RANDOM % 256 ))
case $N in
6)
RES="${RES}$(printf '4%x' $(( B % 16 )))"
;;
8)
RES="${RES}$(printf '%c%x' ${C:$RANDOM%${#C}:1} $(( B % 16 )))"
;;
3 | 5 | 7 | 9)
RES="${RES}$(printf '%02x-' $B)"
;;
*)
RES="${RES}$(printf '%02x' $B)"
;;
esac
done
echo -n $RES
}
#
# Extract document_id (if present) and HTTP response code from curl response passed as argument
#
extract_document_id() {
DOCUMENT_ID=$(echo $1 | grep document_id | tail -1 | sed -e 's/^.*"document_id": *"\([^"]*\)".*$/\1/')
HTTP_RETURN_CODE=$(echo echo $1 | tail -1 | sed -e 's/^.*\([0-9][0-9][0-9]\)$/\1/')
}
# Helper function to make curl requests
make_request() {
local method=$1
local endpoint=$2
local data=$3
local headers=$4
if [ "xx${headers}" == "xx" ]; then
headers='-H "Content-Type: application/json"'
fi
# echo "Testing $method $endpoint"
# echo "Request data: $data"
# echo "Request headers: $headers"
CMD="curl -X $method \"$API_BASE$endpoint\" -s -w \"\n%{http_code}\" -H \"Authorization: Bearer $AUTH_TOKEN\" $headers $data"
echo $CMD
RES=$(eval $CMD 2>/dev/null)
echo "Response: [${RES}]"
extract_document_id "$RES"
# echo "document_id --> ${DOCUMENT_ID}"
if [[ ${HTTP_RETURN_CODE} -lt 300 ]]; then
echo "SUCCESS. code = ${HTTP_RETURN_CODE}"
else
echo "ERROR. code = ${HTTP_RETURN_CODE}"
fi
echo -e "\n\n"
if [ "${EXECUTE_ALL}" != "-a" ]; then
echo "Press any key to continue..."
read -n 1 -s -r
echo -e "\n\n"
fi
}
#
# Simple scanner of cmd line arguments, inferring meaning of the value
# It recognized:
# a) host URL (starts with 'http' prefix)
# b) JWT token (starts with 'ey' prefix)
# c) flag -a to execute all tests w/o waiting for user pressing any key
#
while [ $# -gt 0 ]; do
echo "Argument: [${1}]"
if [[ "$1" == ey* ]]; then
AUTH_TOKEN=$1
fi
if [[ "$1" == http* ]]; then
HOST=$1
fi
if [[ "$1" == "-a" ]]; then
EXECUTE_ALL=$1
fi
shift # Removes $1 and shifts remaining args left
done
if [ "xx${HOST}" == "xx" ]; then
HOST="http://localhost:7272"
fi
API_BASE="$HOST/v3"
# Once integrated with CleverMicro, token can be obtained from CleverMicro
# if [ "xx${AUTH_TOKEN}" == "xx" ]; then
# AUTH_TOKEN=$(curl $HOST/api/v0/login -s -d '{"username":"hugo@cleverthis.com","password":"tellmeall"}' -H "Content-Type: application/json" | sed -e "s/.*access_token...\(.*\)..$/\1/")
# fi