Source code for nltk.test.unit.test_corpus_views

# -*- coding: utf-8 -*-
"""
Corpus View Regression Tests
"""
from __future__ import absolute_import, unicode_literals
import unittest
import nltk.data
from nltk.corpus.reader.util import (
    StreamBackedCorpusView,
    read_whitespace_block,
    read_line_block,
)


[docs]class TestCorpusViews(unittest.TestCase): linetok = nltk.LineTokenizer(blanklines='keep') names = [ 'corpora/inaugural/README', # A very short file (160 chars) 'corpora/inaugural/1793-Washington.txt', # A relatively short file (791 chars) 'corpora/inaugural/1909-Taft.txt', # A longer file (32k chars) ]
[docs] def data(self): for name in self.names: f = nltk.data.find(name) with f.open() as fp: file_data = fp.read().decode('utf8') yield f, file_data
[docs] def test_correct_values(self): # Check that corpus views produce the correct sequence of values. for f, file_data in self.data(): v = StreamBackedCorpusView(f, read_whitespace_block) self.assertEqual(list(v), file_data.split()) v = StreamBackedCorpusView(f, read_line_block) self.assertEqual(list(v), self.linetok.tokenize(file_data))
[docs] def test_correct_length(self): # Check that the corpus views report the correct lengths: for f, file_data in self.data(): v = StreamBackedCorpusView(f, read_whitespace_block) self.assertEqual(len(v), len(file_data.split())) v = StreamBackedCorpusView(f, read_line_block) self.assertEqual(len(v), len(self.linetok.tokenize(file_data)))