Source code for nltk.test.unit.test_disagreement

# -*- coding: utf-8 -*-
from __future__ import absolute_import, unicode_literals
import unittest

from nltk.metrics.agreement import AnnotationTask

[docs]class TestDisagreement(unittest.TestCase): ''' Class containing unit tests for nltk.metrics.agreement.Disagreement. '''
[docs] def test_easy(self): ''' Simple test, based on https://github.com/foolswood/krippendorffs_alpha/raw/master/krippendorff.pdf. ''' data = [('coder1', 'dress1', 'YES'), ('coder2', 'dress1', 'NO'), ('coder3', 'dress1', 'NO'), ('coder1', 'dress2', 'YES'), ('coder2', 'dress2', 'NO'), ('coder3', 'dress3', 'NO'), ] annotation_task = AnnotationTask(data) self.assertAlmostEqual(annotation_task.alpha(), -0.3333333)
[docs] def test_easy2(self): ''' Same simple test with 1 rating removed. Removal of that rating should not matter: K-Apha ignores items with only 1 rating. ''' data = [('coder1', 'dress1', 'YES'), ('coder2', 'dress1', 'NO'), ('coder3', 'dress1', 'NO'), ('coder1', 'dress2', 'YES'), ('coder2', 'dress2', 'NO'), ] annotation_task = AnnotationTask(data) self.assertAlmostEqual(annotation_task.alpha(), -0.3333333)
[docs] def test_advanced(self): ''' More advanced test, based on http://www.agreestat.com/research_papers/onkrippendorffalpha.pdf ''' data = [('A', '1', '1'), ('B', '1', '1'), ('D', '1', '1'), ('A', '2', '2'), ('B', '2', '2'), ('C', '2', '3'), ('D', '2', '2'), ('A', '3', '3'), ('B', '3', '3'), ('C', '3', '3'), ('D', '3', '3'), ('A', '4', '3'), ('B', '4', '3'), ('C', '4', '3'), ('D', '4', '3'), ('A', '5', '2'), ('B', '5', '2'), ('C', '5', '2'), ('D', '5', '2'), ('A', '6', '1'), ('B', '6', '2'), ('C', '6', '3'), ('D', '6', '4'), ('A', '7', '4'), ('B', '7', '4'), ('C', '7', '4'), ('D', '7', '4'), ('A', '8', '1'), ('B', '8', '1'), ('C', '8', '2'), ('D', '8', '1'), ('A', '9', '2'), ('B', '9', '2'), ('C', '9', '2'), ('D', '9', '2'), ('B', '10', '5'), ('C', '10', '5'), ('D', '10', '5'), ('C', '11', '1'), ('D', '11', '1'), ('C', '12', '3'), ] annotation_task = AnnotationTask(data) self.assertAlmostEqual(annotation_task.alpha(), 0.743421052632)
[docs] def test_advanced2(self): ''' Same more advanced example, but with 1 rating removed. Again, removal of that 1 rating shoudl not matter. ''' data = [('A', '1', '1'), ('B', '1', '1'), ('D', '1', '1'), ('A', '2', '2'), ('B', '2', '2'), ('C', '2', '3'), ('D', '2', '2'), ('A', '3', '3'), ('B', '3', '3'), ('C', '3', '3'), ('D', '3', '3'), ('A', '4', '3'), ('B', '4', '3'), ('C', '4', '3'), ('D', '4', '3'), ('A', '5', '2'), ('B', '5', '2'), ('C', '5', '2'), ('D', '5', '2'), ('A', '6', '1'), ('B', '6', '2'), ('C', '6', '3'), ('D', '6', '4'), ('A', '7', '4'), ('B', '7', '4'), ('C', '7', '4'), ('D', '7', '4'), ('A', '8', '1'), ('B', '8', '1'), ('C', '8', '2'), ('D', '8', '1'), ('A', '9', '2'), ('B', '9', '2'), ('C', '9', '2'), ('D', '9', '2'), ('B', '10', '5'), ('C', '10', '5'), ('D', '10', '5'), ('C', '11', '1'), ('D', '11', '1'), ('C', '12', '3'), ] annotation_task = AnnotationTask(data) self.assertAlmostEqual(annotation_task.alpha(), 0.743421052632)