Figure 8: Extracts tokens containing only alphanumeric characters

import java.io.*;
import java.util.*;

class SortTokens2
{
    public static void main(String[] args)
        throws IOException
    {
        BufferedReader in =
            new BufferedReader(
               new FileReader(args[0]));
        final int MAXTOKS = 256;
        String[] tokens = new String[MAXTOKS];
        int nTokens = 0;
        String delims =
            " `~!@#$%^&*()-_=+]}[{\\|'\";:/?.>,<";

        // Read lines:
        String line;
        while ((line = in.readLine()) != null)
        {
            // Tokenize line:
            StringTokenizer strtok =
                new StringTokenizer(line, delims);
            while (nTokens < MAXTOKS &&
                   strtok.hasMoreTokens())
                tokens[nTokens++] =
                    strtok.nextToken().toLowerCase();
        }

        // Sort/print:
        Arrays.sort(tokens, 0, nTokens);
        for (int i = 0; i < nTokens; ++i)
            System.out.println(tokens[i]);
    }
}

/* Output:
1900
1925
an
associated
barber
bathtub
became
began
connected
decline
form
from
gin
harmony
its
its
of
offkey
overindulgence
peak
popularity
prohibition
raucous
reached
shop
singing
the
the
then
to
to
with
with
with
*/