216 lines
8.6 KiB
Python
216 lines
8.6 KiB
Python
 
|
"""Integration test for AI-powered feedback analysis flow"""
|
||
|
|
import pytest
|
||
|
|
import os
|
||
|
|
import yaml
|
||
|
|
from unittest.mock import Mock, patch
|
||
|
|
|
||
|
|
|
||
|
|
@pytest.fixture
|
||
|
|
def test_product_for_analysis(app):
|
||
|
|
"""Create a test product for analysis testing"""
|
||
|
|
with app.app_context():
|
||
|
|
# Create test product directory and config
|
||
|
|
product_dir = os.path.join(app.config['DATA_DIR'], 'products', 'analysis-test-product')
|
||
|
|
os.makedirs(product_dir, exist_ok=True)
|
||
|
|
|
||
|
|
# Create product config
|
||
|
|
config_file = os.path.join(product_dir, 'config.yaml')
|
||
|
|
config_data = {
|
||
|
|
'product_id': 'analysis-test-product',
|
||
|
|
'name': 'Analysis Test Product',
|
||
|
|
'submission_url_slug': 'analysis-test',
|
||
|
|
'owner_language': 'en',
|
||
|
|
'assigned_owner_ids': ['usr_0001'],
|
||
|
|
'status': 'active'
|
||
|
|
}
|
||
|
|
|
||
|
|
with open(config_file, 'w') as f:
|
||
|
|
yaml.dump(config_data, f)
|
||
|
|
|
||
|
|
yield 'analysis-test-product'
|
||
|
|
|
||
|
|
|
||
|
|
@pytest.mark.integration
|
||
|
|
def test_full_ai_analysis_flow(client, app, test_product_for_analysis):
|
||
|
|
"""T070: Integration test for full AI analysis flow
|
||
|
|
|
||
|
|
Test the complete AI analysis workflow:
|
||
|
|
1. User submits feedback in German
|
||
|
|
2. System saves feedback to filesystem
|
||
|
|
3. Background analysis task is triggered
|
||
|
|
4. AI analyzes feedback (category, summary, translation)
|
||
|
|
5. Analysis.md is created with results
|
||
|
|
6. Metadata is updated with status and language
|
||
|
|
7. Original content.txt is preserved
|
||
|
|
"""
|
||
|
|
# Mock the Claude API response
|
||
|
|
mock_api_response = Mock()
|
||
|
|
mock_api_response.content = [Mock(text="""
|
||
|
|
# Feedback Analysis
|
||
|
|
|
||
|
|
**Category**: bug
|
||
|
|
|
||
|
|
**Original Language**: de
|
||
|
|
|
||
|
|
**Summary**: User reports that the login button is not working on mobile devices
|
||
|
|
|
||
|
|
**Translation**: The login button on mobile devices does not respond when I click it. I tried multiple times but nothing happens.
|
||
|
|
""")]
|
||
|
|
|
||
|
|
# Patch the AI analyzer to use mock response
|
||
|
|
with patch('app.services.ai_analyzer.anthropic.Anthropic'):
|
||
|
|
with patch('app.services.ai_analyzer.ClaudeAnalyzer._call_claude_api', return_value=mock_api_response):
|
||
|
|
with patch('os.getenv', return_value='test-api-key'):
|
||
|
|
# Step 1-2: Submit feedback in German
|
||
|
|
feedback_text = "Der Login-Button auf mobilen Geräten reagiert nicht, wenn ich darauf klicke. Ich habe es mehrmals versucht, aber es passiert nichts."
|
||
|
|
|
||
|
|
data = {
|
||
|
|
'feedback_text': feedback_text
|
||
|
|
}
|
||
|
|
|
||
|
|
# Also mock the background threading to run synchronously in tests
|
||
|
|
with patch('app.routes.submission.threading.Thread') as mock_thread:
|
||
|
|
# Make the thread run immediately in the test with proper args
|
||
|
|
def run_sync():
|
||
|
|
target = mock_thread.call_args[1]['target']
|
||
|
|
args = mock_thread.call_args[1]['args']
|
||
|
|
# Call with app context - first arg is app instance
|
||
|
|
with args[0].app_context():
|
||
|
|
target(*args)
|
||
|
|
|
||
|
|
mock_thread.return_value.start.side_effect = run_sync
|
||
|
|
|
||
|
|
response = client.post('/submit/analysis-test',
|
||
|
|
data=data,
|
||
|
|
follow_redirects=True)
|
||
|
|
|
||
|
|
assert response.status_code == 200
|
||
|
|
|
||
|
|
# Step 6-7: Verify feedback was saved and analyzed
|
||
|
|
with app.app_context():
|
||
|
|
data_dir = app.config['DATA_DIR']
|
||
|
|
products_dir = os.path.join(data_dir, 'products', 'analysis-test-product', 'feedback')
|
||
|
|
|
||
|
|
# Find the created feedback directory
|
||
|
|
feedback_dirs = [d for d in os.listdir(products_dir)
|
||
|
|
if os.path.isdir(os.path.join(products_dir, d))]
|
||
|
|
|
||
|
|
assert len(feedback_dirs) > 0, "No feedback directory was created"
|
||
|
|
|
||
|
|
feedback_dir = os.path.join(products_dir, feedback_dirs[0])
|
||
|
|
|
||
|
|
# Verify original content.txt is preserved (FR-016)
|
||
|
|
content_file = os.path.join(feedback_dir, 'content.txt')
|
||
|
|
assert os.path.exists(content_file)
|
||
|
|
|
||
|
|
with open(content_file, 'r', encoding='utf-8') as f:
|
||
|
|
saved_content = f.read()
|
||
|
|
|
||
|
|
assert feedback_text in saved_content, "Original content not preserved"
|
||
|
|
|
||
|
|
# Verify analysis.md was created
|
||
|
|
analysis_file = os.path.join(feedback_dir, 'analysis.md')
|
||
|
|
assert os.path.exists(analysis_file), "Analysis file not created"
|
||
|
|
|
||
|
|
with open(analysis_file, 'r', encoding='utf-8') as f:
|
||
|
|
analysis_content = f.read()
|
||
|
|
|
||
|
|
# Verify analysis contains expected sections
|
||
|
|
assert '# Feedback Analysis' in analysis_content
|
||
|
|
assert 'Category' in analysis_content
|
||
|
|
assert 'bug' in analysis_content
|
||
|
|
assert 'Original Language' in analysis_content
|
||
|
|
assert 'de' in analysis_content
|
||
|
|
assert 'Summary' in analysis_content
|
||
|
|
assert 'Translation' in analysis_content
|
||
|
|
|
||
|
|
# Verify metadata was updated
|
||
|
|
metadata_file = os.path.join(feedback_dir, 'metadata.yaml')
|
||
|
|
assert os.path.exists(metadata_file)
|
||
|
|
|
||
|
|
with open(metadata_file, 'r') as f:
|
||
|
|
metadata = yaml.safe_load(f)
|
||
|
|
|
||
|
|
# Status should be 'analyzed' after successful analysis
|
||
|
|
assert metadata['status'] in ['analyzed', 'analyzing']
|
||
|
|
# Original language should be detected and stored
|
||
|
|
assert metadata.get('original_language') == 'de'
|
||
|
|
# Category should be stored
|
||
|
|
assert metadata.get('category') == 'bug'
|
||
|
|
|
||
|
|
|
||
|
|
@pytest.mark.integration
|
||
|
|
def test_analysis_preserves_images(client, app, test_product_for_analysis):
|
||
|
|
"""Integration test: Verify images are stored but not analyzed via OCR (FR-021)
|
||
|
|
|
||
|
|
Per FR-021, images should be stored as attachments but not processed for OCR.
|
||
|
|
Only text content should be analyzed.
|
||
|
|
"""
|
||
|
|
import io
|
||
|
|
|
||
|
|
feedback_text = "Screenshot of the error"
|
||
|
|
|
||
|
|
data = {
|
||
|
|
'feedback_text': feedback_text,
|
||
|
|
'files': [
|
||
|
|
(io.BytesIO(b'PNG fake image data'), 'screenshot.png')
|
||
|
|
]
|
||
|
|
}
|
||
|
|
|
||
|
|
# Mock AI to ensure it only receives text, not image data
|
||
|
|
mock_api_response = Mock()
|
||
|
|
mock_api_response.content = [Mock(text="""
|
||
|
|
# Feedback Analysis
|
||
|
|
|
||
|
|
**Category**: bug
|
||
|
|
|
||
|
|
**Original Language**: en
|
||
|
|
|
||
|
|
**Summary**: User provided screenshot of error
|
||
|
|
|
||
|
|
**Translation**: (same as original)
|
||
|
|
""")]
|
||
|
|
|
||
|
|
with patch('app.services.ai_analyzer.anthropic.Anthropic'):
|
||
|
|
with patch('app.services.ai_analyzer.ClaudeAnalyzer._call_claude_api', return_value=mock_api_response) as mock_call:
|
||
|
|
with patch('os.getenv', return_value='test-api-key'):
|
||
|
|
with patch('app.routes.submission.threading.Thread') as mock_thread:
|
||
|
|
# Make the thread run immediately in the test with proper args
|
||
|
|
def run_sync():
|
||
|
|
target = mock_thread.call_args[1]['target']
|
||
|
|
args = mock_thread.call_args[1]['args']
|
||
|
|
# Call with app context - first arg is app instance
|
||
|
|
with args[0].app_context():
|
||
|
|
target(*args)
|
||
|
|
|
||
|
|
mock_thread.return_value.start.side_effect = run_sync
|
||
|
|
|
||
|
|
response = client.post('/submit/analysis-test',
|
||
|
|
data=data,
|
||
|
|
content_type='multipart/form-data',
|
||
|
|
follow_redirects=True)
|
||
|
|
|
||
|
|
assert response.status_code == 200
|
||
|
|
|
||
|
|
# Verify AI was called with text only, not image data
|
||
|
|
if mock_call.called:
|
||
|
|
call_args = str(mock_call.call_args)
|
||
|
|
# Should contain text feedback
|
||
|
|
assert 'Screenshot of the error' in call_args or 'screenshot' in call_args.lower()
|
||
|
|
# Should NOT contain image binary data
|
||
|
|
assert b'PNG' not in call_args.encode() if isinstance(call_args, str) else b'PNG' not in call_args
|
||
|
|
|
||
|
|
# Verify image was stored as attachment
|
||
|
|
with app.app_context():
|
||
|
|
data_dir = app.config['DATA_DIR']
|
||
|
|
products_dir = os.path.join(data_dir, 'products', 'analysis-test-product', 'feedback')
|
||
|
|
|
||
|
|
feedback_dirs = [d for d in os.listdir(products_dir)
|
||
|
|
if os.path.isdir(os.path.join(products_dir, d))]
|
||
|
|
|
||
|
|
feedback_dir = os.path.join(products_dir, feedback_dirs[0])
|
||
|
|
attachments_dir = os.path.join(feedback_dir, 'attachments')
|
||
|
|
|
||
|
|
assert os.path.exists(attachments_dir)
|
||
|
|
assert 'screenshot.png' in os.listdir(attachments_dir)
|