/* ****************************************************************
 * Copyright (C) 2004 by SAS Institute Inc., Cary, NC 27513
 *
 * Name:             textsyn.sas
 * Support:          cox  James A. Cox
 * Product:          SAS Text Miner
 * Language:         Sas Macro
 * Script:
 *
 * Usage: %textsyn( termds=<key-data-set>
 *                , synds=<output-syn-dsname>
 *                ,<docds=&_MAC_1
 *                , outds=&_MAC_2
 *                ,...
 *                ) ;  (see additional parameters in Notes below).
 *
 * Purpose: To automatically create a synonym list to pull together misspellings,
 *    undetected stems and shorthand, to use as an input synonym data set to
 *    SAS Text Miner.
 *
 * Output: It creates a data set appropriate for use as a input
 *    synonym data set for SAS Text Miner.  This data set has variables
 *    for term, parent, and category (required for a synonym data set),
 *    but it also adds additional variables that can be useful for user
 *    browsing: minsped (indicating the minimum spelling edit distance
 *    there is between the terms, where smaller is closer), childndocs
 *    (number of documents the child term was in), and numdocs (the
 *    number of documents the parent term was in).  If you specify certain
 *    parameters, it will augment these with example1 and example2,
 *    extracting portions of up to two documents that the child term
 *    occurs in.
 *
 * Algorithm: It creates candidate child terms by including all terms
 *    that occurred in mxchddoc (default=6) or fewer documents, and
 *    candidate parent terms by including all terms that occurred in
 *    mnpardoc (default=3) or more documents.  It then finds all
 *    combinations of the parent and child data sets where the first
 *    alphabetic character agrees between them.  It then calculates a
 *    minsped parameter for each combination, based on using the spedis
 *    function available in Base SAS.  However, the default spedis
 *    function is not symmetric, and divides the result by the length of
 *    the first argument.  Through extensive validation, we found that
 *    better results were obtained by taking the minimum of the spedis
 *    results in both directions divided by the length of the shorter
 *    term. We also found that it was necessary to penalize multi-word
 *    terms (or "adjust to" would always map to "adjust for") and that
 *    better results were obtained by penalizing the result if the
 *    candidate child term (or misspelling, or abbreviation, or whatever)
 *    was not found in a target dictionary.  So the minsped is multiplied
 *    by the multipen= (default=2) parameter if the child or parent is a
 *    multi-word term, and is multiplied by the dictpen= (default=2)
 *    parameter if a dict= dictionary is applied and the child is not
 *    found in it.  The defaults for these were also generated by testing
 *    with different data sets.
 *
 *    After creating the minsped measure for all valid combinations,
 *    the candidate parent with the lowest minsped for each child is
 *    output as a synonym IF it's minsped is lower than the maxsped=
 *    parameter (default=15).  If the docds=, outds=, and textvar=
 *    parameters have been specified, up to two examples are then created
 *    for each resulting synonyms by merging the synonym data set with
 *    the out data set to find the document ids for that child, then
 *    merging with the document table to find at most two documents
 *    containing that child.  The document is searched for the presence
 *    of the child term, and then the example is created by putting !!
 *    before and after the term, and taking up to context= (default=4)
 *    words before and after that term in the document.
 *
 *
 * History:
 * 03Aug04 Initial Coding [cox]
 *
 * Notes:  First run a SAS Text Miner node, but set the option to include
 *    singly occuring terms. You may process with or without parts of
 *    speech, depending on preference. The best way to use the macro is
 *    to then put a code node after the Text Miner tool and call
 *    the macro from the code node. After running the code node, open
 *    the Text Miner tool (or use a new Text Miner tool), set the synonym
 *    data set to the new data set you created in the tool, and turn off
 *    singly occuring terms.  You may want to manually edit the
 *    synonym list that is created before using it, to eliminate
 *    "spurious" items.  Using the "contextual" parameters below will
 *    help you see how that term is being used.  However, the default
 *    settings do a pretty good job of eliminating false positives
 *    (less than 5% false positives in the test data sets),
 *    particularly if you use an appropriate dictionary with the dict=
 *    command.
 *
 *    All parameters are named, not positional.  Reasonable defaults have
 *    been created for many of the optional parameters, based on
 *    validation across three different data sets.
 *
 *    Required Parameters:
 *      termds= the term data set output from the Text Miner tool.
 *        If using a code node, this should be set to the macro corresponding
 *        to the KEY data set (&_MAC_3 in EM4.3).
 *      synds= the synonym data set that you want to create.  Make
 *        sure that this is saved permanently, you may want to use it in
 *        other projects.
 *
 *    Required Parameters for examples:  If these parameters are
 * specified, two additional variables will be created, example1 and
 * example2, that will contain example usages of the terms from the
 * document table.
 *
 *      docds= Name of document table.  This could be set to &_MAC_1 if using
 *        the SAS code node after the Text Mining Tool in EM4.3.
 *      outds= Name of the "OUT" table from Text Miner.  If using a
 *        SAS code node after a TM node, this is &_MAC_2.  If using EM5.1,
 *        it is exported at the out table, from the Text Miner node.
 *      textvar= Name of variable parsed in TM node.
 *
 *    If you want it to check the children against a dictionary, which
 * will reduce the false positives significantly, specify the
 * dictionary data set using the dict= option.  This data set should
 * have a variable named TERM that contains the text of the term in
 * all lower case.
 *
 *    Optional parameters:  These will be set to intelligent defaults
 * if not specified by the user.
 *
 *      mnpardoc= Minimum number of documents for a term to be
 *        considered a possible parent.  Default=3.
 *      mxchddoc= Maximum number of documents for a term to be
 *        considered a possible child.  Default=6.
 *      maxsped= Maximum spedis measure that will accept a parent-child
 *        combination as valid. Default=15.
 *      multipen= Number to multiply the spedis measure by when either
 *        the parent or child is a multi-word term.  Default=2.
 *      dictpen= Number to multiply the spedis measure by when the child
 *        term is found in the dictionary table.  Default=2.
 *      context= Number of words to include in the context in the
 *        examples, before and after the target term.  Default=4.
 *
 *
 *
 * Last Modified By:
 * Last Modified On: Fri Feb 10 16:23:34 2006
 *
 * End
 *
 *
 * ************************************************************** */

%macro textsyn( context=4

              , dict=
              , dictpen=2

              , maxsped=15
              , mnpardoc=3
              , multipen=2
              , mxchddoc=6

              , docds=      /* TRNnnnnn &_MAC1 */
              , outds=      /* OUT_nnnn &_MAC2 */
              , synds=
              , termds=      /* KEY_nnnn &_MAC3 */
              , textvar=
              ) ;

   data
      _parent( keep=firstchar numdocs term role )
      _child( keep=firstchar childterm childndocs childlen dict key role )
      ;
      length firstchar $1 ;
      set &TERMDS ;

      /* exclude all terms that are parents */

      if substr( term, 1, 1 ) ne '+' ;

      pos = indexc( term, '0123456789abcdefghijklmnopqrstuvwxyz' ) ;

      if pos > 0 ;

      firstchar = substr( term, pos, 1 ) ;

      if numdocs >= &MNPARDOC then output _parent ;

      if numdocs <= &MXCHDDOC & parent = .
      then do ;
         label childterm = 'child term' ;
         childterm = term ;
         childndocs = numdocs ;

         childlen = length( trim( term )) ;
         dict = 'N' ;
         output _child ;
      end ;
   run ;

   %if &DICT ne
   %then %do ;
      proc sort data=_child ; by childterm ; run ;

      data _child ;
         merge
            &DICT( rename=( term=childterm ) in=a )
            _child( in=b )
            ;
         by childterm ;

         if a & b then dict = 'Y' ;
         if b then output _child ;
      run ;
   %end ;

   proc sql ;
      create table _syncands as
         select
            a.firstchar
         ,   a.numdocs
         ,   a.term
         ,   a.role

         ,   b.childterm
         ,   b.childndocs
         ,   b.childlen
         ,   b.key
         ,   b.dict
         from
            _parent a
         ,    _child b
         where
            a.firstchar = b.firstchar & a.numdocs > b.childndocs & a.role = b.role ;
   quit ;

   data _syncands( keep=parent term numdocs childndocs minsped key category dict ) ;
      set _syncands ;

      parent   = term ;
      cterm    = childterm ;
      category = role ;

      if substr( term, length( term ), 1 ) = '.' then
         term = substr( term, 1, length( term ) - 1 ) ;
      if substr( childterm,length( childterm ), 1 ) = '.' then
         childterm = substr( childterm, 1, length( childterm ) - 1 ) ;

      minlen = min( length( term ), length( childterm )) ;

      /* Delete candidates where child is parent with other info appended */

      if length( childterm ) > length( parent )+1 then
         if substr( childterm,1, length( parent )) = parent then delete ;

      /* lendiff = abs( length( childterm ) - length( term )) */

      spedis1 = int((spedis( term, childterm ) * length( term ))/ minlen );
      spedis2 = int((spedis( childterm, term ) * length( childterm ))/minlen );
      minsped = min( spedis1, spedis2 );

      if dict = 'Y' then minsped = minsped * &DICTPEN ;

      /* For multi-word terms, need to increase minsped */

      if scan( term, 2, ' ') ne ' ' or scan( childterm, 2, ' ') ne ' ' then
         minsped = minsped * &MULTIPEN ;

      if minsped <= &MAXSPED ;

      term = cterm ;
   run ;

   proc sort data=_syncands ; by category term minsped descending numdocs ; run ;

   data _syncands ;
      set _syncands ;
      by category term ;

      if first.term ;
   run ;

   /* show examples of word in context */

   %if ( &DOCDS ne & &OUTDS ne & &TEXTVAR ne )
   %then %do ;
      proc sql ;
      create table &SYNDS as
         select
            term
         ,  parent
         ,  category
         ,  childndocs
         ,  numdocs
         ,  &TEXTVAR
         ,  minsped
         ,  dict
         from
             _syncands as a
         ,  &DOCDS as b
         ,  &OUTDS as c
         where ( c._document_ = b._document_) & ( a.key = c._termid_) ;
      quit ;

      proc sort data=&SYNDS ; by category term ; run ;

      data &SYNDS( drop=&TEXTVAR tmpstr tmpterm pos i pos2 lastfirst) ;
         length example $300 tmpstr $32000 tmpterm $200 ;
         retain lastfirst ;
         set &synds ; by category term ;

         if first.term or lastfirst
         then do ;
            pos = index( lowcase( &TEXTVAR), trim( term )) ;

            if pos
            then do ;
               example = '!!' ||substr( &TEXTVAR, pos, length( term )) || '!!' ;

               if pos > 1
               then do ;
                  tmpstr = reverse( substr( &TEXTVAR, 1, pos - 1 )) ;

                  do i=1 to &CONTEXT + 1 ;
                     tmpterm = scan( tmpstr, i, ' ') ;
                  end ;

                  pos2 = indexw( tmpstr, trim( tmpterm )) ;

                  if tmpterm ne ' ' and pos2 > 0
                      then example = '...' ||
                           trim( substr( &textvar, pos - pos2 + 1, pos2 - 1 )) ||
                           ' ' ||
                            trim( example)
                           ;
                  else
                     example = substr( &TEXTVAR, 1, pos - 1 ) || ' ' || trim( example ) ;
               end ;

               if pos + length( term) < length( &TEXTVAR )
               then do ;
                  tmpstr=substr( &TEXTVAR, pos + length( term )) ;
                  do i=1 to &CONTEXT + 1 ;
                     tmpterm = scan( tmpstr, i, ' ') ;
                     end ;

                  pos2 = indexw( tmpstr, trim( tmpterm )) ;
                  if tmpterm ne ' ' and pos2 < length( tmpstr)
                  then example = trim( example) || trim( substr( tmpstr,1,pos2 - 1 )) || '...' ;
                  else example = trim( example) || tmpstr ;
                  end ;
               end ;
            else do ;
               example = &TEXTVAR ;
               put term 'not found in ' example ;
               end ;

            lastfirst = first.term ;
            output ;
         end ;
         else
            lastfirst = 0 ;
      run ;

      data &SYNDS( drop=example ) ;
         length example1 example2 $300 ;
         retain example1 ;
         set &SYNDS ;
         by category term ;

         if first.term then example1 = example ; else example2 = example ;

         if last.term then output ;
      run ;
   %end ;
   %else %do ;
      data &SYNDS( drop=key ) ; set _syncands ; run ;
   %end ;

%mend TEXTSYN ;

/***
%textsyn( termds=&_MAC_3
        , docds=&_MAC_1
        , outds=&_MAC_2
        , textvar=cnt_desc
        , mnpardoc=3
        , mxchddoc=6
        , synds=sampsio.autosyns
        , dict=sampsio.engdict
        , maxsped=15
        ) ;
***/