Source code for nltk.corpus.reader.panlex_swadesh

# Natural Language Toolkit: Word List Corpus Reader
# Copyright (C) 2001-2021 NLTK Project
# Author: Steven Bird <>
#         Edward Loper <>
# URL: <>
# For license information, see LICENSE.TXT

import re
from collections import defaultdict, namedtuple

from nltk.corpus.reader.api import *
from nltk.corpus.reader.util import *
from nltk.corpus.reader.wordlist import WordListCorpusReader
from nltk.tokenize import line_tokenize

PanlexLanguage = namedtuple(
        "panlex_uid",  # (1) PanLex UID
        "iso639",  # (2) ISO 639 language code
        "iso639_type",  # (3) ISO 639 language type, see README
        "script",  # (4) normal scripts of expressions
        "name",  # (5) PanLex default name
        "langvar_uid",  # (6) UID of the language variety in which the default name is an expression

[docs]class PanlexSwadeshCorpusReader(WordListCorpusReader): """ This is a class to read the PanLex Swadesh list from David Kamholz, Jonathan Pool, and Susan M. Colowick (2014). PanLex: Building a Resource for Panlingual Lexical Translation. In LREC. License: CC0 1.0 Universal """
[docs] def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) # Find the swadesh size using the fileids' path. self.swadesh_size = re.match(r"swadesh([0-9].*)\/", self.fileids()[0]).group(1) self._languages = {lang.panlex_uid: lang for lang in self.get_languages()} self._macro_langauges = self.get_macrolanguages()
[docs] def license(self): return "CC0 1.0 Universal"
[docs] def language_codes(self): return self._languages.keys()
[docs] def get_languages(self): for line in self.raw(f"langs{self.swadesh_size}.txt").split("\n"): if not line.strip(): # Skip empty lines. continue yield PanlexLanguage(*line.strip().split("\t"))
[docs] def get_macrolanguages(self): macro_langauges = defaultdict(list) for lang in self._languages.values(): macro_langauges[lang.iso639].append(lang.panlex_uid) return macro_langauges
[docs] def words_by_lang(self, lang_code): """ :return: a list of list(str) """ fileid = f"swadesh{self.swadesh_size}/{lang_code}.txt" return [concept.split("\t") for concept in self.words(fileid)]
[docs] def words_by_iso639(self, iso63_code): """ :return: a list of list(str) """ fileids = [ f"swadesh{self.swadesh_size}/{lang_code}.txt" for lang_code in self._macro_langauges[iso63_code] ] return [ concept.split("\t") for fileid in fileids for concept in self.words(fileid) ]
[docs] def entries(self, fileids=None): """ :return: a tuple of words for the specified fileids. """ if not fileids: fileids = self.fileids() wordlists = [self.words(f) for f in fileids] return list(zip(*wordlists))