# Natural Language Toolkit: Maximum Entropy Classifiers
#
# Copyright (C) 2001-2014 NLTK Project
# Author: Edward Loper <edloper@gmail.com>
#         Dmitry Chichkov <dchichkov@gmail.com> (TypedMaxentFeatureEncoding)
# URL: <http://nltk.org/>
# For license information, see LICENSE.TXT
 
"""
A classifier model based on maximum entropy modeling framework.  This
framework considers all of the probability distributions that are
empirically consistent with the training data; and chooses the
distribution with the highest entropy.  A probability distribution is
"empirically consistent" with a set of training data if its estimated
frequency with which a class and a feature vector value co-occur is
equal to the actual frequency in the data.
 
Terminology: 'feature'
======================
The term *feature* is usually used to refer to some property of an
unlabeled token.  For example, when performing word sense
disambiguation, we might define a ``'prevword'`` feature whose value is
the word preceding the target word.  However, in the context of
maxent modeling, the term *feature* is typically used to refer to a
property of a "labeled" token.  In order to prevent confusion, we
will introduce two distinct terms to disambiguate these two different
concepts:
 
  - An "input-feature" is a property of an unlabeled token.
  - A "joint-feature" is a property of a labeled token.
 
In the rest of the ``nltk.classify`` module, the term "features" is
used to refer to what we will call "input-features" in this module.
 
In literature that describes and discusses maximum entropy models,
input-features are typically called "contexts", and joint-features
are simply referred to as "features".
 
Converting Input-Features to Joint-Features
-------------------------------------------
In maximum entropy models, joint-features are required to have numeric
values.  Typically, each input-feature ``input_feat`` is mapped to a
set of joint-features of the form:
 
|   joint_feat(token, label) = { 1 if input_feat(token) == feat_val
|                              {      and label == some_label
|                              {
|                              { 0 otherwise
 
For all values of ``feat_val`` and ``some_label``.  This mapping is
performed by classes that implement the ``MaxentFeatureEncodingI``
interface.
"""
from __future__ import print_function, unicode_literals
__docformat__ = 'epytext en'
 
try:
    import numpy
except ImportError:
    pass
 
import time
import tempfile
import os
import gzip
from collections import defaultdict
 
from nltk import compat
from nltk.data import gzip_open_unicode
from nltk.util import OrderedDict
from nltk.probability import DictionaryProbDist
 
from nltk.classify.api import ClassifierI
from nltk.classify.util import CutoffChecker, accuracy, log_likelihood
from nltk.classify.megam import call_megam, write_megam_file, parse_megam_weights
from nltk.classify.tadm import call_tadm, write_tadm_file, parse_tadm_weights
...

_megam_bin = None
def config_megam(bin=None):
    """
    Configure NLTK's interface to the ``megam`` maxent optimization
    package.
 
    :param bin: The full path to the ``megam`` binary.  If not specified,
        then nltk will search the system for a ``megam`` binary; and if
        one is not found, it will raise a ``LookupError`` exception.
    :type bin: str
    """
    global _megam_bin
    _megam_bin = find_binary(
        'megam', bin,
        env_vars=['MEGAM'],
        binary_names=['megam.opt', 'megam', 'megam_686', 'megam_i686.opt'],
        url='http://www.umiacs.umd.edu/~hal/megam/index.html')

 
In [1]: import random
 
In [2]: from nltk.corpus import names
 
In [3]: from nltk import MaxentClassifier
 
In [5]: from nltk import classify
 
In [7]: names = ([(name, 'male') for name in names.words('male.txt')] + [(name, 'female') for name in names.words('female.txt')])
 
In [8]: random.shuffle(names)
 
In [10]: def gender_features3(name):
        features = {}
        features["fl"] = name[0].lower()
        features["ll"] = name[-1].lower()
        features["fw"] = name[:2].lower()
        features["lw"] = name[-2:].lower()
        return features
   ....: 
 
In [11]: featuresets = [(gender_features3(n), g) for (n, g) in names]
 
In [12]: train_set, test_set = featuresets[500:], featuresets[:500]
 
In [17]: me3_megam_classifier = MaxentClassifier.train(train_set, "megam")
[Found megam: megam]
Scanning file...7444 train, 0 dev, 0 test, reading...done
Warning: there only appear to be two classes, but we're
         optimizing with BFGS...using binary optimization
         with CG would be much faster
optimizing with lambda = 0
it 1   dw 4.640e-01 pp 6.38216e-01 er 0.37413
it 2   dw 2.065e-01 pp 5.74892e-01 er 0.37413
it 3   dw 3.503e-01 pp 5.43226e-01 er 0.24328
it 4   dw 1.209e-01 pp 5.29406e-01 er 0.22394
it 5   dw 4.864e-01 pp 5.27097e-01 er 0.26115
it 6   dw 5.765e-01 pp 4.92409e-01 er 0.23415
it 7   dw 0.000e+00 pp 4.92409e-01 er 0.23415
-------------------------
it 1 dw 1.802e-01 pp 4.74930e-01 er 0.21588
it 2   dw 3.478e-02 pp 4.70876e-01 er 0.21548
it 3   dw 1.963e-01 pp 4.61761e-01 er 0.21709
it 4   dw 9.624e-02 pp 4.56257e-01 er 0.21574
it 5   dw 3.442e-01 pp 4.54401e-01
......
it 10  dw 2.399e-03 pp 3.71020e-01 er 0.16967
it 11  dw 2.202e-02 pp 3.71017e-01 er 0.16980
it 12  dw 0.000e+00 pp 3.71017e-01 er 0.16980
-------------------------
it 1 dw 2.620e-02 pp 3.70816e-01 er 0.17020
it 2   dw 2.285e-02 pp 3.70721e-01 er 0.16953
it 3   dw 1.074e-02 pp 3.70631e-01 er 0.16980
it 4   dw 3.152e-02 pp 3.70580e-01 er 0.16994
it 5   dw 2.263e-02 pp 3.70504e-01 er 0.16940
it 6   dw 1.115e-01 pp 3.70370e-01 er 0.16886
it 7   dw 1.938e-01 pp 3.70318e-01 er 0.16913
it 8   dw 1.365e-01 pp 3.69815e-01 er 0.16940
it 9   dw 2.634e-01 pp 3.69366e-01 er 0.16873
it 10  dw 2.498e-01 pp 3.69290e-01 er 0.17007
it 11  dw 2.515e-01 pp 3.69240e-01 er 0.16994
it 12  dw 3.027e-01 pp 3.69234e-01 er 0.16994
it 13  dw 9.850e-03 pp 3.69233e-01 er 0.16994
it 14  dw 1.152e-01 pp 3.69214e-01 er 0.16994
it 15  dw 0.000e+00 pp 3.69214e-01 er 0.16994
 
In [18]: classify.accuracy(me3_megam_classifier, test_set)
Out[18]: 0.812