python / test_evaluation.py
python / test_evaluation.py
import copy
import json
from pathlib import Path
import pytest
import httpx
import server
from evaluate_report import evaluate
from review import build_review
def sample():
url = 'https://example.com/'
response = httpx.Response(200, headers={'content-type': 'text/html',
'content-security-policy': "frame-ancestors 'self' https://admin.thebase.com"},
text='<script src="https://cdn.example/a.js"></script>' * 2,
request=httpx.Request('GET', url))
return server.analyze_response(response, url, [url])
def report_text(report):
review, _ = build_review(report, render_markdown=False)
text = review['fixed_tables_markdown']
for row in review['finding_rows']:
text += f"\n### {row['rank']}. {row['finding']}\n"
text += row['protected_markdown']
text += '\n' + review['mandatory_caveats_markdown']
return text
def test_tables_preserve_explicit_domains_counts_and_full_codes():
report = sample()
tables = report['review']['fixed_tables_markdown']
assert 'https://admin.thebase.com' in tables
assert '*.thebase.com' not in tables
assert '| https://cdn.example/a.js | 2 |' in tables
assert '全script要素数ではない' in tables
for row in report['review']['finding_rows']:
assert f"| {row['rank']} | {row['finding']} |" in tables
def test_evaluation_detects_modified_tables_and_missing_conditions():
report = sample()
text = report_text(report)
assert evaluate(report, text)['mechanical_checks_passed']
altered = text.replace('https://admin.thebase.com', '*.thebase.com')
assert 'fixed_tables_changed_or_missing' in evaluate(report, altered)['errors']
condition = report['review']['finding_rows'][0]['protected_markdown'].splitlines()[3]
altered = text.replace(condition + '\n', '', 1)
assert any('condition_list_count' in e for e in evaluate(report, altered)['errors'])
def test_evaluation_detects_missing_or_reordered_sections_and_disclaims_semantics():
report = sample()
text = report_text(report)
row = report['review']['finding_rows'][0]
altered = text.replace(f"### 1. {row['finding']}", f"### 2. {row['finding']}")
assert 'finding_sections_missing_reordered_or_renamed' in evaluate(report, altered)['errors']
result = evaluate(report, text)
assert result['semantic_review_required']
assert 'translation fidelity' in result['not_checked']
def test_full_width_heading_suffix_is_not_a_missing_finding():
report = sample()
text = report_text(report)
text = text.replace('### 1. csp_script_restriction_missing', '### 1. csp_script_restriction_missing(説明)')
assert evaluate(report, text)['mechanical_checks_passed']
def test_protected_text_detects_semantic_negation_and_impact_omission():
report = server.analyze_response(httpx.Response(200, headers={'content-type': 'text/html'}, text='<script src="https://cdn.jsdelivr.net/npm/axios/a.js"></script>',
request=httpx.Request('GET', 'https://example.com/')), 'https://example.com/', ['https://example.com/'])
text = report_text(report)
assert evaluate(report, text)['mechanical_checks_passed']
changed = text.replace('侵害の有無は未確認です。', '侵害は成立していません。')
assert any(e.startswith('protected_text_changed_or_missing:') for e in evaluate(report, changed)['errors'])
changed = text.replace('変更されたスクリプトが、そのページの権限で実行される可能性があります。', '')
assert any(e.startswith('protected_text_changed_or_missing:') for e in evaluate(report, changed)['errors'])
changed = text.replace(report['review']['mandatory_caveats_markdown'], '')
assert 'mandatory_caveats_changed_or_missing' in evaluate(report, changed)['errors']
def test_fixed_render_is_repeatable_and_escapes_untrusted_observations():
report = sample()
report['headers']['server'] = '<script>|[link](https://evil.test)'
original = copy.deepcopy(report)
first, _ = build_review(report, render_markdown=False)
second, _ = build_review(report, render_markdown=False)
assert report == original and first == second
assert '<script>' not in first['fixed_tables_markdown']
assert '[link](https://evil.test)' not in first['fixed_tables_markdown']
@pytest.mark.parametrize('name,count,score', [('hoju',18,46), ('base',17,63)])
def test_captured_baselines_replay_without_network(name, count, score):
directory = Path(__file__).parent / 'evaluation'
report = json.loads((directory / f'{name}.json').read_text(encoding='utf-8'))
assert len(report['findings']) == count and report['security_score'] == score
review, _ = build_review(report, render_markdown=False)
assert review['fixed_tables_markdown'] == (directory / f'{name}-tables.md').read_text(encoding='utf-8')
assert evaluate(report, report_text(report))['mechanical_checks_passed']
if name == 'base':
assert '*.thebase.com' not in review['fixed_tables_markdown']
assert '| https://ajax.googleapis.com/ajax/libs/jquery/3.7.1/jquery.min.js | 2 |' in review['fixed_tables_markdown']
else:
assert '| https://hoju.jp/cart/proxy/basket/items/add | post | 未観測 | 6 |' in review['fixed_tables_markdown']
import copy
import json
from pathlib import Path
import pytest
import httpx
import server
from evaluate_report import evaluate
from review import build_review
def sample():
url = 'https://example.com/'
response = httpx.Response(200, headers={'content-type': 'text/html',
'content-security-policy': "frame-ancestors 'self' https://admin.thebase.com"},
text='<script src="https://cdn.example/a.js"></script>' * 2,
request=httpx.Request('GET', url))
return server.analyze_response(response, url, [url])
def report_text(report):
review, _ = build_review(report, render_markdown=False)
text = review['fixed_tables_markdown']
for row in review['finding_rows']:
text += f"\n### {row['rank']}. {row['finding']}\n"
text += row['protected_markdown']
text += '\n' + review['mandatory_caveats_markdown']
return text
def test_tables_preserve_explicit_domains_counts_and_full_codes():
report = sample()
tables = report['review']['fixed_tables_markdown']
assert 'https://admin.thebase.com' in tables
assert '*.thebase.com' not in tables
assert '| https://cdn.example/a.js | 2 |' in tables
assert '全script要素数ではない' in tables
for row in report['review']['finding_rows']:
assert f"| {row['rank']} | {row['finding']} |" in tables
def test_evaluation_detects_modified_tables_and_missing_conditions():
report = sample()
text = report_text(report)
assert evaluate(report, text)['mechanical_checks_passed']
altered = text.replace('https://admin.thebase.com', '*.thebase.com')
assert 'fixed_tables_changed_or_missing' in evaluate(report, altered)['errors']
condition = report['review']['finding_rows'][0]['protected_markdown'].splitlines()[3]
altered = text.replace(condition + '\n', '', 1)
assert any('condition_list_count' in e for e in evaluate(report, altered)['errors'])
def test_evaluation_detects_missing_or_reordered_sections_and_disclaims_semantics():
report = sample()
text = report_text(report)
row = report['review']['finding_rows'][0]
altered = text.replace(f"### 1. {row['finding']}", f"### 2. {row['finding']}")
assert 'finding_sections_missing_reordered_or_renamed' in evaluate(report, altered)['errors']
result = evaluate(report, text)
assert result['semantic_review_required']
assert 'translation fidelity' in result['not_checked']
def test_full_width_heading_suffix_is_not_a_missing_finding():
report = sample()
text = report_text(report)
text = text.replace('### 1. csp_script_restriction_missing', '### 1. csp_script_restriction_missing(説明)')
assert evaluate(report, text)['mechanical_checks_passed']
def test_protected_text_detects_semantic_negation_and_impact_omission():
report = server.analyze_response(httpx.Response(200, headers={'content-type': 'text/html'}, text='<script src="https://cdn.jsdelivr.net/npm/axios/a.js"></script>',
request=httpx.Request('GET', 'https://example.com/')), 'https://example.com/', ['https://example.com/'])
text = report_text(report)
assert evaluate(report, text)['mechanical_checks_passed']
changed = text.replace('侵害の有無は未確認です。', '侵害は成立していません。')
assert any(e.startswith('protected_text_changed_or_missing:') for e in evaluate(report, changed)['errors'])
changed = text.replace('変更されたスクリプトが、そのページの権限で実行される可能性があります。', '')
assert any(e.startswith('protected_text_changed_or_missing:') for e in evaluate(report, changed)['errors'])
changed = text.replace(report['review']['mandatory_caveats_markdown'], '')
assert 'mandatory_caveats_changed_or_missing' in evaluate(report, changed)['errors']
def test_fixed_render_is_repeatable_and_escapes_untrusted_observations():
report = sample()
report['headers']['server'] = '<script>|[link](https://evil.test)'
original = copy.deepcopy(report)
first, _ = build_review(report, render_markdown=False)
second, _ = build_review(report, render_markdown=False)
assert report == original and first == second
assert '<script>' not in first['fixed_tables_markdown']
assert '[link](https://evil.test)' not in first['fixed_tables_markdown']
@pytest.mark.parametrize('name,count,score', [('hoju',18,46), ('base',17,63)])
def test_captured_baselines_replay_without_network(name, count, score):
directory = Path(__file__).parent / 'evaluation'
report = json.loads((directory / f'{name}.json').read_text(encoding='utf-8'))
assert len(report['findings']) == count and report['security_score'] == score
review, _ = build_review(report, render_markdown=False)
assert review['fixed_tables_markdown'] == (directory / f'{name}-tables.md').read_text(encoding='utf-8')
assert evaluate(report, report_text(report))['mechanical_checks_passed']
if name == 'base':
assert '*.thebase.com' not in review['fixed_tables_markdown']
assert '| https://ajax.googleapis.com/ajax/libs/jquery/3.7.1/jquery.min.js | 2 |' in review['fixed_tables_markdown']
else:
assert '| https://hoju.jp/cart/proxy/basket/items/add | post | 未観測 | 6 |' in review['fixed_tables_markdown']