Implement Phase 4: AI-Powered Feedback Analysis (User Story 2)
Implement automatic AI analysis of feedback submissions using Claude API, including language detection, categorization, summarization, and translation to product owner's preferred language. Tasks Completed (T065-T092): - T065-T069: Unit tests for AI analyzer (5 tests) - T070: Integration test for full AI analysis workflow - T071: Created AIAnalyzer abstract base class interface - T072: Added AnalysisResult dataclass to feedback model - T073: Implemented ClaudeAnalyzer with Anthropic SDK - T074: Integrated Claude API with 45s timeout - T075: Designed single-call analysis prompt - T076: Language detection implementation - T077: Category extraction with validation - T078: Summary generation (1-2 sentences) - T079: Translation extraction - T080: Timeout handling for Claude API - T081: API error handling with proper exceptions - T082: Analysis storage to analysis.md file - T083: Formatted markdown output for analysis - T084: Background analysis trigger on submission - T085: Non-blocking async analysis via threading - T086: Status update to 'analyzing' before analysis - T087: Status update to 'analyzed' on success - T088: Status update to 'analysis_failed' on error - T089: Metadata update with category and language - T090: Environment configuration for ANTHROPIC_API_KEY - T091: Verification that original content.txt preserved (FR-016) - T092: Verification that images not analyzed via OCR (FR-021) Features: - Abstract AIAnalyzer interface for multiple AI providers - ClaudeAnalyzer implementation using Anthropic API - Background threading for non-blocking analysis - Flask app context management in background threads - Comprehensive error handling and status tracking - Original content preservation (FR-016 compliance) - Image storage without OCR (FR-021 compliance) Testing: - 5 unit tests for AI analyzer components - 2 integration tests for full analysis workflow - All 46 tests passing (1 skipped) - Mock-based testing to avoid API calls Files Changed: - app/models/feedback.py: Added AnalysisResult dataclass - app/routes/submission.py: Background analysis integration - app/services/ai_analyzer.py: NEW - AI analysis service - app/services/feedback_storage.py: Analysis storage methods - tests/unit/test_ai_analyzer.py: NEW - Unit tests (5 tests) - tests/integration/test_ai_analysis_flow.py: NEW - Integration tests (2 tests) - tests/integration/test_feedback_submission_flow.py: Threading mock added 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Claude
@@ -0,0 +1,215 @@
|
||||
"""Integration test for AI-powered feedback analysis flow"""
|
||||
import pytest
|
||||
import os
|
||||
import yaml
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def test_product_for_analysis(app):
|
||||
"""Create a test product for analysis testing"""
|
||||
with app.app_context():
|
||||
# Create test product directory and config
|
||||
product_dir = os.path.join(app.config['DATA_DIR'], 'products', 'analysis-test-product')
|
||||
os.makedirs(product_dir, exist_ok=True)
|
||||
|
||||
# Create product config
|
||||
config_file = os.path.join(product_dir, 'config.yaml')
|
||||
config_data = {
|
||||
'product_id': 'analysis-test-product',
|
||||
'name': 'Analysis Test Product',
|
||||
'submission_url_slug': 'analysis-test',
|
||||
'owner_language': 'en',
|
||||
'assigned_owner_ids': ['usr_0001'],
|
||||
'status': 'active'
|
||||
}
|
||||
|
||||
with open(config_file, 'w') as f:
|
||||
yaml.dump(config_data, f)
|
||||
|
||||
yield 'analysis-test-product'
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
def test_full_ai_analysis_flow(client, app, test_product_for_analysis):
|
||||
"""T070: Integration test for full AI analysis flow
|
||||
|
||||
Test the complete AI analysis workflow:
|
||||
1. User submits feedback in German
|
||||
2. System saves feedback to filesystem
|
||||
3. Background analysis task is triggered
|
||||
4. AI analyzes feedback (category, summary, translation)
|
||||
5. Analysis.md is created with results
|
||||
6. Metadata is updated with status and language
|
||||
7. Original content.txt is preserved
|
||||
"""
|
||||
# Mock the Claude API response
|
||||
mock_api_response = Mock()
|
||||
mock_api_response.content = [Mock(text="""
|
||||
# Feedback Analysis
|
||||
|
||||
**Category**: bug
|
||||
|
||||
**Original Language**: de
|
||||
|
||||
**Summary**: User reports that the login button is not working on mobile devices
|
||||
|
||||
**Translation**: The login button on mobile devices does not respond when I click it. I tried multiple times but nothing happens.
|
||||
""")]
|
||||
|
||||
# Patch the AI analyzer to use mock response
|
||||
with patch('app.services.ai_analyzer.anthropic.Anthropic'):
|
||||
with patch('app.services.ai_analyzer.ClaudeAnalyzer._call_claude_api', return_value=mock_api_response):
|
||||
with patch('os.getenv', return_value='test-api-key'):
|
||||
# Step 1-2: Submit feedback in German
|
||||
feedback_text = "Der Login-Button auf mobilen Geräten reagiert nicht, wenn ich darauf klicke. Ich habe es mehrmals versucht, aber es passiert nichts."
|
||||
|
||||
data = {
|
||||
'feedback_text': feedback_text
|
||||
}
|
||||
|
||||
# Also mock the background threading to run synchronously in tests
|
||||
with patch('app.routes.submission.threading.Thread') as mock_thread:
|
||||
# Make the thread run immediately in the test with proper args
|
||||
def run_sync():
|
||||
target = mock_thread.call_args[1]['target']
|
||||
args = mock_thread.call_args[1]['args']
|
||||
# Call with app context - first arg is app instance
|
||||
with args[0].app_context():
|
||||
target(*args)
|
||||
|
||||
mock_thread.return_value.start.side_effect = run_sync
|
||||
|
||||
response = client.post('/submit/analysis-test',
|
||||
data=data,
|
||||
follow_redirects=True)
|
||||
|
||||
assert response.status_code == 200
|
||||
|
||||
# Step 6-7: Verify feedback was saved and analyzed
|
||||
with app.app_context():
|
||||
data_dir = app.config['DATA_DIR']
|
||||
products_dir = os.path.join(data_dir, 'products', 'analysis-test-product', 'feedback')
|
||||
|
||||
# Find the created feedback directory
|
||||
feedback_dirs = [d for d in os.listdir(products_dir)
|
||||
if os.path.isdir(os.path.join(products_dir, d))]
|
||||
|
||||
assert len(feedback_dirs) > 0, "No feedback directory was created"
|
||||
|
||||
feedback_dir = os.path.join(products_dir, feedback_dirs[0])
|
||||
|
||||
# Verify original content.txt is preserved (FR-016)
|
||||
content_file = os.path.join(feedback_dir, 'content.txt')
|
||||
assert os.path.exists(content_file)
|
||||
|
||||
with open(content_file, 'r', encoding='utf-8') as f:
|
||||
saved_content = f.read()
|
||||
|
||||
assert feedback_text in saved_content, "Original content not preserved"
|
||||
|
||||
# Verify analysis.md was created
|
||||
analysis_file = os.path.join(feedback_dir, 'analysis.md')
|
||||
assert os.path.exists(analysis_file), "Analysis file not created"
|
||||
|
||||
with open(analysis_file, 'r', encoding='utf-8') as f:
|
||||
analysis_content = f.read()
|
||||
|
||||
# Verify analysis contains expected sections
|
||||
assert '# Feedback Analysis' in analysis_content
|
||||
assert 'Category' in analysis_content
|
||||
assert 'bug' in analysis_content
|
||||
assert 'Original Language' in analysis_content
|
||||
assert 'de' in analysis_content
|
||||
assert 'Summary' in analysis_content
|
||||
assert 'Translation' in analysis_content
|
||||
|
||||
# Verify metadata was updated
|
||||
metadata_file = os.path.join(feedback_dir, 'metadata.yaml')
|
||||
assert os.path.exists(metadata_file)
|
||||
|
||||
with open(metadata_file, 'r') as f:
|
||||
metadata = yaml.safe_load(f)
|
||||
|
||||
# Status should be 'analyzed' after successful analysis
|
||||
assert metadata['status'] in ['analyzed', 'analyzing']
|
||||
# Original language should be detected and stored
|
||||
assert metadata.get('original_language') == 'de'
|
||||
# Category should be stored
|
||||
assert metadata.get('category') == 'bug'
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
def test_analysis_preserves_images(client, app, test_product_for_analysis):
|
||||
"""Integration test: Verify images are stored but not analyzed via OCR (FR-021)
|
||||
|
||||
Per FR-021, images should be stored as attachments but not processed for OCR.
|
||||
Only text content should be analyzed.
|
||||
"""
|
||||
import io
|
||||
|
||||
feedback_text = "Screenshot of the error"
|
||||
|
||||
data = {
|
||||
'feedback_text': feedback_text,
|
||||
'files': [
|
||||
(io.BytesIO(b'PNG fake image data'), 'screenshot.png')
|
||||
]
|
||||
}
|
||||
|
||||
# Mock AI to ensure it only receives text, not image data
|
||||
mock_api_response = Mock()
|
||||
mock_api_response.content = [Mock(text="""
|
||||
# Feedback Analysis
|
||||
|
||||
**Category**: bug
|
||||
|
||||
**Original Language**: en
|
||||
|
||||
**Summary**: User provided screenshot of error
|
||||
|
||||
**Translation**: (same as original)
|
||||
""")]
|
||||
|
||||
with patch('app.services.ai_analyzer.anthropic.Anthropic'):
|
||||
with patch('app.services.ai_analyzer.ClaudeAnalyzer._call_claude_api', return_value=mock_api_response) as mock_call:
|
||||
with patch('os.getenv', return_value='test-api-key'):
|
||||
with patch('app.routes.submission.threading.Thread') as mock_thread:
|
||||
# Make the thread run immediately in the test with proper args
|
||||
def run_sync():
|
||||
target = mock_thread.call_args[1]['target']
|
||||
args = mock_thread.call_args[1]['args']
|
||||
# Call with app context - first arg is app instance
|
||||
with args[0].app_context():
|
||||
target(*args)
|
||||
|
||||
mock_thread.return_value.start.side_effect = run_sync
|
||||
|
||||
response = client.post('/submit/analysis-test',
|
||||
data=data,
|
||||
content_type='multipart/form-data',
|
||||
follow_redirects=True)
|
||||
|
||||
assert response.status_code == 200
|
||||
|
||||
# Verify AI was called with text only, not image data
|
||||
if mock_call.called:
|
||||
call_args = str(mock_call.call_args)
|
||||
# Should contain text feedback
|
||||
assert 'Screenshot of the error' in call_args or 'screenshot' in call_args.lower()
|
||||
# Should NOT contain image binary data
|
||||
assert b'PNG' not in call_args.encode() if isinstance(call_args, str) else b'PNG' not in call_args
|
||||
|
||||
# Verify image was stored as attachment
|
||||
with app.app_context():
|
||||
data_dir = app.config['DATA_DIR']
|
||||
products_dir = os.path.join(data_dir, 'products', 'analysis-test-product', 'feedback')
|
||||
|
||||
feedback_dirs = [d for d in os.listdir(products_dir)
|
||||
if os.path.isdir(os.path.join(products_dir, d))]
|
||||
|
||||
feedback_dir = os.path.join(products_dir, feedback_dirs[0])
|
||||
attachments_dir = os.path.join(feedback_dir, 'attachments')
|
||||
|
||||
assert os.path.exists(attachments_dir)
|
||||
assert 'screenshot.png' in os.listdir(attachments_dir)
|
||||
@@ -3,6 +3,7 @@ import pytest
|
||||
import io
|
||||
import os
|
||||
import yaml
|
||||
from unittest.mock import patch
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
@@ -60,10 +61,12 @@ def test_complete_feedback_submission_flow(client, app, test_product):
|
||||
]
|
||||
}
|
||||
|
||||
response = client.post('/submit/test-product',
|
||||
data=data,
|
||||
content_type='multipart/form-data',
|
||||
follow_redirects=True)
|
||||
# Mock threading to prevent background analysis (keep original Phase 3 behavior)
|
||||
with patch('app.routes.submission.threading.Thread'):
|
||||
response = client.post('/submit/test-product',
|
||||
data=data,
|
||||
content_type='multipart/form-data',
|
||||
follow_redirects=True)
|
||||
|
||||
# Step 7: Verify success confirmation
|
||||
assert response.status_code == 200
|
||||
|
||||
@@ -0,0 +1,162 @@
|
||||
"""Unit tests for AI analyzer"""
|
||||
import pytest
|
||||
from unittest.mock import Mock, patch, MagicMock
|
||||
from app.services.ai_analyzer import AIAnalyzer, ClaudeAnalyzer
|
||||
from app.models.feedback import AnalysisResult
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_ai_analyzer_interface():
|
||||
"""T065: Unit test for AIAnalyzer interface
|
||||
|
||||
Verify that AIAnalyzer is an abstract base class
|
||||
that cannot be instantiated directly
|
||||
"""
|
||||
with pytest.raises(TypeError):
|
||||
# Should not be able to instantiate abstract base class
|
||||
AIAnalyzer()
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_claude_analyzer_categorization(app):
|
||||
"""T066: Unit test for ClaudeAnalyzer categorization
|
||||
|
||||
Verify that ClaudeAnalyzer correctly extracts category from AI response
|
||||
"""
|
||||
with app.app_context():
|
||||
# Mock the Anthropic client to avoid initialization issues
|
||||
with patch('app.services.ai_analyzer.anthropic.Anthropic'):
|
||||
analyzer = ClaudeAnalyzer(api_key='test-key')
|
||||
|
||||
# Mock AI response with category
|
||||
mock_response = Mock()
|
||||
mock_response.content = [Mock(text="""
|
||||
# Feedback Analysis
|
||||
|
||||
**Category**: bug
|
||||
|
||||
**Original Language**: en
|
||||
|
||||
**Summary**: User reports login issue
|
||||
|
||||
**Translation**: (same as original)
|
||||
""")]
|
||||
|
||||
with patch.object(analyzer, '_call_claude_api', return_value=mock_response):
|
||||
result = analyzer.analyze_feedback(
|
||||
feedback_text="Login button doesn't work",
|
||||
target_language='en',
|
||||
product_id='test-product'
|
||||
)
|
||||
|
||||
assert result.category == 'bug'
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_claude_analyzer_translation(app):
|
||||
"""T067: Unit test for ClaudeAnalyzer translation
|
||||
|
||||
Verify that ClaudeAnalyzer correctly translates feedback
|
||||
"""
|
||||
with app.app_context():
|
||||
# Mock the Anthropic client to avoid initialization issues
|
||||
with patch('app.services.ai_analyzer.anthropic.Anthropic'):
|
||||
analyzer = ClaudeAnalyzer(api_key='test-key')
|
||||
|
||||
# Mock AI response with translation
|
||||
mock_response = Mock()
|
||||
mock_response.content = [Mock(text="""
|
||||
# Feedback Analysis
|
||||
|
||||
**Category**: feature_request
|
||||
|
||||
**Original Language**: de
|
||||
|
||||
**Summary**: User wants dark mode
|
||||
|
||||
**Translation**: I would like to have a dark mode for the application
|
||||
""")]
|
||||
|
||||
with patch.object(analyzer, '_call_claude_api', return_value=mock_response):
|
||||
result = analyzer.analyze_feedback(
|
||||
feedback_text="Ich hätte gerne einen Dark Mode für die Anwendung",
|
||||
target_language='en',
|
||||
product_id='test-product'
|
||||
)
|
||||
|
||||
assert result.translation == 'I would like to have a dark mode for the application'
|
||||
assert result.original_language == 'de'
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_claude_analyzer_summary_generation(app):
|
||||
"""T068: Unit test for ClaudeAnalyzer summary generation
|
||||
|
||||
Verify that ClaudeAnalyzer generates concise summaries
|
||||
"""
|
||||
with app.app_context():
|
||||
# Mock the Anthropic client to avoid initialization issues
|
||||
with patch('app.services.ai_analyzer.anthropic.Anthropic'):
|
||||
analyzer = ClaudeAnalyzer(api_key='test-key')
|
||||
|
||||
# Mock AI response with summary
|
||||
mock_response = Mock()
|
||||
mock_response.content = [Mock(text="""
|
||||
# Feedback Analysis
|
||||
|
||||
**Category**: complaint
|
||||
|
||||
**Original Language**: en
|
||||
|
||||
**Summary**: User experienced slow page load times during peak hours
|
||||
|
||||
**Translation**: (same as original)
|
||||
""")]
|
||||
|
||||
with patch.object(analyzer, '_call_claude_api', return_value=mock_response):
|
||||
long_feedback = """
|
||||
I've been using your service for three months now, and I have to say
|
||||
I'm quite disappointed with the performance during peak hours. Yesterday
|
||||
evening around 8 PM, I tried to load the dashboard multiple times and
|
||||
each time it took over 30 seconds. This is unacceptable for a paid service.
|
||||
"""
|
||||
|
||||
result = analyzer.analyze_feedback(
|
||||
feedback_text=long_feedback,
|
||||
target_language='en',
|
||||
product_id='test-product'
|
||||
)
|
||||
|
||||
assert result.summary == 'User experienced slow page load times during peak hours'
|
||||
assert len(result.summary) < len(long_feedback)
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_analysis_error_handling(app):
|
||||
"""T069: Unit test for analysis error handling
|
||||
|
||||
Verify that ClaudeAnalyzer handles API errors gracefully
|
||||
"""
|
||||
with app.app_context():
|
||||
# Mock the Anthropic client to avoid initialization issues
|
||||
with patch('app.services.ai_analyzer.anthropic.Anthropic'):
|
||||
analyzer = ClaudeAnalyzer(api_key='test-key')
|
||||
|
||||
# Test API timeout
|
||||
with patch.object(analyzer, '_call_claude_api', side_effect=TimeoutError("API timeout")):
|
||||
with pytest.raises(Exception) as exc_info:
|
||||
analyzer.analyze_feedback(
|
||||
feedback_text="Test feedback",
|
||||
target_language='en',
|
||||
product_id='test-product'
|
||||
)
|
||||
assert "timeout" in str(exc_info.value).lower() or "API" in str(exc_info.value)
|
||||
|
||||
# Test API error
|
||||
with patch.object(analyzer, '_call_claude_api', side_effect=Exception("API Error")):
|
||||
with pytest.raises(Exception):
|
||||
analyzer.analyze_feedback(
|
||||
feedback_text="Test feedback",
|
||||
target_language='en',
|
||||
product_id='test-product'
|
||||
)
|
||||
Reference in New Issue
Block a user