Source code for nltk.tokenize.stanford

# -*- coding: utf-8 -*-
# Natural Language Toolkit: Interface to the Stanford Tokenizer
# Copyright (C) 2001-2021 NLTK Project
# Author: Steven Xu <>
# URL: <>
# For license information, see LICENSE.TXT

import tempfile
import os
import json
from subprocess import PIPE
import warnings

from nltk.internals import find_jar, config_java, java, _java_options
from nltk.tokenize.api import TokenizerI
from nltk.parse.corenlp import CoreNLPParser

_stanford_url = ""

[docs]class StanfordTokenizer(TokenizerI): r""" Interface to the Stanford Tokenizer >>> from nltk.tokenize.stanford import StanfordTokenizer >>> s = "Good muffins cost $3.88\nin New York. Please buy me\ntwo of them.\nThanks." >>> StanfordTokenizer().tokenize(s) ['Good', 'muffins', 'cost', '$', '3.88', 'in', 'New', 'York', '.', 'Please', 'buy', 'me', 'two', 'of', 'them', '.', 'Thanks', '.'] >>> s = "The colour of the wall is blue." >>> StanfordTokenizer(options={"americanize": True}).tokenize(s) ['The', 'color', 'of', 'the', 'wall', 'is', 'blue', '.'] """ _JAR = "stanford-postagger.jar" def __init__( self, path_to_jar=None, encoding="utf8", options=None, verbose=False, java_options="-mx1000m", ): # Raise deprecation warning. warnings.warn( str( "\nThe StanfordTokenizer will " "be deprecated in version 3.2.5.\n" "Please use \033[91mnltk.parse.corenlp.CoreNLPParser\033[0m instead.'" ), DeprecationWarning, stacklevel=2, ) self._stanford_jar = find_jar( self._JAR, path_to_jar, env_vars=("STANFORD_POSTAGGER",), searchpath=(), url=_stanford_url, verbose=verbose, ) self._encoding = encoding self.java_options = java_options options = {} if options is None else options self._options_cmd = ",".join( "{0}={1}".format(key, val) for key, val in options.items() ) @staticmethod def _parse_tokenized_output(s): return s.splitlines()
[docs] def tokenize(self, s): """ Use stanford tokenizer's PTBTokenizer to tokenize multiple sentences. """ cmd = ["edu.stanford.nlp.process.PTBTokenizer"] return self._parse_tokenized_output(self._execute(cmd, s))
def _execute(self, cmd, input_, verbose=False): encoding = self._encoding cmd.extend(["-charset", encoding]) _options_cmd = self._options_cmd if _options_cmd: cmd.extend(["-options", self._options_cmd]) default_options = " ".join(_java_options) # Configure java. config_java(options=self.java_options, verbose=verbose) # Windows is incompatible with NamedTemporaryFile() without passing in delete=False. with tempfile.NamedTemporaryFile(mode="wb", delete=False) as input_file: # Write the actual sentences to the temporary input file if isinstance(input_, str) and encoding: input_ = input_.encode(encoding) input_file.write(input_) input_file.flush() cmd.append( # Run the tagger and get the output. stdout, stderr = java( cmd, classpath=self._stanford_jar, stdout=PIPE, stderr=PIPE ) stdout = stdout.decode(encoding) os.unlink( # Return java configurations to their default values. config_java(options=default_options, verbose=False) return stdout