Source code for nltk.stem.arlstem2

# -*- coding: utf-8 -*-
#
# Natural Language Toolkit: ARLSTem Stemmer v2
#
# Copyright (C) 2001-2021 NLTK Project
#
# Author: Kheireddine Abainia (x-programer) <k.abainia@gmail.com>
# Algorithms: Kheireddine Abainia <k.abainia@gmail.com>
#                         Hamza Rebbani <hamrebbani@gmail.com>
# URL: <http://nltk.org/>
# For license information, see LICENSE.TXT


"""
ARLSTem2 Arabic Light Stemmer
The details about the implementation of this algorithm are described in:
K. Abainia and H. Rebbani, Comparing the Effectiveness of the Improved ARLSTem
Algorithm with Existing Arabic Light Stemmers, International Conference on
Theoretical and Applicative Aspects of Computer Science (ICTAACS'19), Skikda,
Algeria, December 15-16, 2019.
ARLSTem2 is an Arabic light stemmer based on removing the affixes from
the words (i.e. prefixes, suffixes and infixes). It is an improvement
of the previous Arabic light stemmer (ARLSTem). The new version was compared to
the original algorithm and several existing Arabic light stemmers, where the
results showed that the new version considerably improves the under-stemming
errors that are common to light stemmers. Both ARLSTem and ARLSTem2 can be run
online and do not use any dictionary.
"""
from __future__ import unicode_literals
import re

from nltk.stem.api import StemmerI


[docs]class ARLSTem2(StemmerI): """ Return a stemmed Arabic word after removing affixes. This an improved version of the previous algorithm, which reduces under-stemming errors. Typically used in Arabic search engine, information retrieval and NLP. >>> from nltk.stem import arlstem2 >>> stemmer = ARLSTem2() >>> word = stemmer.stem('يعمل') >>> print(word) :param token: The input Arabic word (unicode) to be stemmed :type token: unicode :return: A unicode Arabic word """ def __init__(self): # different Alif with hamza self.re_hamzated_alif = re.compile(r'[\u0622\u0623\u0625]') self.re_alifMaqsura = re.compile(r'[\u0649]') self.re_diacritics = re.compile(r'[\u064B-\u065F]') # Alif Laam, Laam Laam, Fa Laam, Fa Ba self.pr2 = ['\u0627\u0644', '\u0644\u0644', '\u0641\u0644', '\u0641\u0628'] # Ba Alif Laam, Kaaf Alif Laam, Waaw Alif Laam self.pr3 = ['\u0628\u0627\u0644', '\u0643\u0627\u0644', '\u0648\u0627\u0644'] # Fa Laam Laam, Waaw Laam Laam self.pr32 = ['\u0641\u0644\u0644', '\u0648\u0644\u0644'] # Fa Ba Alif Laam, Waaw Ba Alif Laam, Fa Kaaf Alif Laam self.pr4 = [ '\u0641\u0628\u0627\u0644', '\u0648\u0628\u0627\u0644', '\u0641\u0643\u0627\u0644', ] # Kaf Yaa, Kaf Miim self.su2 = ['\u0643\u064A', '\u0643\u0645'] # Ha Alif, Ha Miim self.su22 = ['\u0647\u0627', '\u0647\u0645'] # Kaf Miim Alif, Kaf Noon Shadda self.su3 = ['\u0643\u0645\u0627', '\u0643\u0646\u0651'] # Ha Miim Alif, Ha Noon Shadda self.su32 = ['\u0647\u0645\u0627', '\u0647\u0646\u0651'] # Alif Noon, Ya Noon, Waaw Noon self.pl_si2 = ['\u0627\u0646', '\u064A\u0646', '\u0648\u0646'] # Taa Alif Noon, Taa Ya Noon self.pl_si3 = ['\u062A\u0627\u0646', '\u062A\u064A\u0646'] # Alif Noon, Waaw Noon self.verb_su2 = ['\u0627\u0646', '\u0648\u0646'] # Siin Taa, Siin Yaa self.verb_pr2 = ['\u0633\u062A', '\u0633\u064A'] # Siin Alif, Siin Noon self.verb_pr22 = ['\u0633\u0627', '\u0633\u0646'] # Lam Noon, Lam Taa, Lam Yaa, Lam Hamza self.verb_pr33 = [ '\u0644\u0646', '\u0644\u062A', '\u0644\u064A', '\u0644\u0623', ] # Taa Miim Alif, Taa Noon Shadda self.verb_suf3 = ['\u062A\u0645\u0627', '\u062A\u0646\u0651'] # Noon Alif, Taa Miim, Taa Alif, Waaw Alif self.verb_suf2 = [ '\u0646\u0627', '\u062A\u0645', '\u062A\u0627', '\u0648\u0627', ] # Taa, Alif, Noon self.verb_suf1 = ['\u062A', '\u0627', '\u0646']
[docs] def stem1(self, token): """ call this function to get the first stem """ try: if token is None: raise ValueError( "The word could not be stemmed, because \ it is empty !" ) self.is_verb = False # remove Arabic diacritics and replace some letters with others token = self.norm(token) # strip the common noun prefixes pre = self.pref(token) if pre is not None: token = pre # transform the feminine form to masculine form fm = self.fem2masc(token) if fm is not None: return fm # strip the adjective affixes adj = self.adject(token) if adj is not None: return adj # strip the suffixes that are common to nouns and verbs token = self.suff(token) # transform a plural noun to a singular noun ps = self.plur2sing(token) if ps is None: if pre is None: # if the noun prefixes are not stripped # strip the verb prefixes and suffixes verb = self.verb(token) if verb is not None: self.is_verb = True return verb else: return ps return token except ValueError as e: print(e)
[docs] def stem(self, token): # stem the input word try: if token is None: raise ValueError( "The word could not be stemmed, because \ it is empty !" ) # run the first round of stemming token = self.stem1(token) # check if there is some additional noun affixes if len(token) > 4: # ^Taa, $Yaa + char if token.startswith('\u062A') and token[-2] == '\u064A': token = token[1:-2] + token[-1] return token # ^Miim, $Waaw + char if token.startswith('\u0645') and token[-2] == '\u0648': token = token[1:-2] + token[-1] return token if len(token) > 3: # !^Alif, $Yaa if not token.startswith('\u0627') and token.endswith('\u064A'): token = token[:-1] return token # $Laam if token.startswith('\u0644'): return token[1:] return token except ValueError as e: print(e)
[docs] def norm(self, token): """ normalize the word by removing diacritics, replace hamzated Alif with Alif bare, replace AlifMaqsura with Yaa and remove Waaw at the beginning. """ # strip Arabic diacritics token = self.re_diacritics.sub('', token) # replace Hamzated Alif with Alif bare token = self.re_hamzated_alif.sub('\u0627', token) # replace alifMaqsura with Yaa token = self.re_alifMaqsura.sub('\u064A', token) # strip the Waaw from the word beginning if the remaining is # tri-literal at least if token.startswith('\u0648') and len(token) > 3: token = token[1:] return token
[docs] def pref(self, token): """ remove prefixes from the words' beginning. """ if len(token) > 5: for p3 in self.pr3: if token.startswith(p3): return token[3:] if len(token) > 6: for p4 in self.pr4: if token.startswith(p4): return token[4:] if len(token) > 5: for p3 in self.pr32: if token.startswith(p3): return token[3:] if len(token) > 4: for p2 in self.pr2: if token.startswith(p2): return token[2:]
[docs] def adjective(self, token): """ remove the infixes from adjectives """ # ^Alif, Alif, $Yaa if len(token) > 5: if ( token.startswith('\u0627') and token[-3] == '\u0627' and token.endswith('\u064A') ): return token[:-3] + token[-2]
[docs] def suff(self, token): """ remove the suffixes from the word's ending. """ if token.endswith('\u0643') and len(token) > 3: return token[:-1] if len(token) > 4: for s2 in self.su2: if token.endswith(s2): return token[:-2] if len(token) > 5: for s3 in self.su3: if token.endswith(s3): return token[:-3] if token.endswith('\u0647') and len(token) > 3: token = token[:-1] return token if len(token) > 4: for s2 in self.su22: if token.endswith(s2): return token[:-2] if len(token) > 5: for s3 in self.su32: if token.endswith(s3): return token[:-3] # $Noon and Alif if token.endswith('\u0646\u0627') and len(token) > 4: return token[:-2] return token
[docs] def fem2masc(self, token): """ transform the word from the feminine form to the masculine form. """ if len(token) > 6: # ^Taa, Yaa, $Yaa and Taa Marbuta if ( token.startswith('\u062A') and token[-4] == '\u064A' and token.endswith('\u064A\u0629') ): return token[1:-4] + token[-3] # ^Alif, Yaa, $Yaa and Taa Marbuta if ( token.startswith('\u0627') and token[-4] == '\u0627' and token.endswith('\u064A\u0629') ): return token[:-4] + token[-3] # $Alif, Yaa and Taa Marbuta if token.endswith('\u0627\u064A\u0629') and len(token) > 5: return token[:-2] if len(token) > 4: # Alif, $Taa Marbuta if token[1] == '\u0627' and token.endswith('\u0629'): return token[0] + token[2:-1] # $Yaa and Taa Marbuta if token.endswith('\u064A\u0629'): return token[:-2] # $Taa Marbuta if token.endswith('\u0629') and len(token) > 3: return token[:-1]
[docs] def plur2sing(self, token): """ transform the word from the plural form to the singular form. """ # ^Haa, $Noon, Waaw if len(token) > 5: if token.startswith('\u0645') and token.endswith('\u0648\u0646'): return token[1:-2] if len(token) > 4: for ps2 in self.pl_si2: if token.endswith(ps2): return token[:-2] if len(token) > 5: for ps3 in self.pl_si3: if token.endswith(ps3): return token[:-3] if len(token) > 4: # $Alif, Taa if token.endswith('\u0627\u062A'): return token[:-2] # ^Alif Alif if token.startswith('\u0627') and token[2] == '\u0627': return token[:2] + token[3:] # ^Alif Alif if token.startswith('\u0627') and token[-2] == '\u0627': return token[1:-2] + token[-1]
[docs] def verb(self, token): """ stem the verb prefixes and suffixes or both """ vb = self.verb_t1(token) if vb is not None: return vb vb = self.verb_t2(token) if vb is not None: return vb vb = self.verb_t3(token) if vb is not None: return vb vb = self.verb_t4(token) if vb is not None: return vb vb = self.verb_t5(token) if vb is not None: return vb vb = self.verb_t6(token) return vb
[docs] def verb_t1(self, token): """ stem the present tense co-occurred prefixes and suffixes """ if len(token) > 5 and token.startswith('\u062A'): # Taa for s2 in self.pl_si2: if token.endswith(s2): return token[1:-2] if len(token) > 5 and token.startswith('\u064A'): # Yaa for s2 in self.verb_su2: if token.endswith(s2): return token[1:-2] if len(token) > 4 and token.startswith('\u0627'): # Alif # Waaw Alif if len(token) > 5 and token.endswith('\u0648\u0627'): return token[1:-2] # Yaa if token.endswith('\u064A'): return token[1:-1] # Alif if token.endswith('\u0627'): return token[1:-1] # Noon if token.endswith('\u0646'): return token[1:-1] # ^Yaa, Noon$ if ( len(token) > 4 and token.startswith('\u064A') and token.endswith('\u0646') ): return token[1:-1] # ^Taa, Noon$ if ( len(token) > 4 and token.startswith('\u062A') and token.endswith('\u0646') ): return token[1:-1]
[docs] def verb_t2(self, token): """ stem the future tense co-occurred prefixes and suffixes """ if len(token) > 6: for s2 in self.pl_si2: # ^Siin Taa if ( token.startswith(self.verb_pr2[0]) and token.endswith(s2) ): return token[2:-2] # ^Siin Yaa, Alif Noon$ if ( token.startswith(self.verb_pr2[1]) and token.endswith(self.pl_si2[0]) ): return token[2:-2] # ^Siin Yaa, Waaw Noon$ if ( token.startswith(self.verb_pr2[1]) and token.endswith(self.pl_si2[2]) ): return token[2:-2] # ^Siin Taa, Noon$ if ( len(token) > 5 and token.startswith(self.verb_pr2[0]) and token.endswith('\u0646') ): return token[2:-1] # ^Siin Yaa, Noon$ if ( len(token) > 5 and token.startswith(self.verb_pr2[1]) and token.endswith('\u0646') ): return token[2:-1]
[docs] def verb_t3(self, token): """ stem the present tense suffixes """ if len(token) > 5: for su3 in self.verb_suf3: if token.endswith(su3): return token[:-3] if len(token) > 4: for su2 in self.verb_suf2: if token.endswith(su2): return token[:-2] if len(token) > 3: for su1 in self.verb_suf1: if token.endswith(su1): return token[:-1]
[docs] def verb_t4(self, token): """ stem the present tense prefixes """ if len(token) > 3: for pr1 in self.verb_suf1: if token.startswith(pr1): return token[1:] if token.startswith('\u064A'): return token[1:]
[docs] def verb_t5(self, token): """ stem the future tense prefixes """ if len(token) > 4: for pr2 in self.verb_pr22: if token.startswith(pr2): return token[2:] for pr2 in self.verb_pr2: if token.startswith(pr2): return token[2:]
[docs] def verb_t6(self, token): """ stem the imperative tense prefixes """ if len(token) > 4: for pr3 in self.verb_pr33: if token.startswith(pr3): return token[2:] return token