/*

HyPhy - Hypothesis Testing Using Phylogenies.

Copyright (C) 1997-now
Core Developers:
   Sergei L Kosakovsky Pond (sergeilkp@icloud.com)
   Art FY Poon    (apoon42@uwo.ca)
   Steven Weaver (sweaver@temple.edu)

Module Developers:
        Lance Hepler (nlhepler@gmail.com)
        Martin Smith (martin.audacis@gmail.com)

Significant contributions from:
  Spencer V Muse (muse@stat.ncsu.edu)
  Simon DW Frost (sdf22@cam.ac.uk)

Permission is hereby granted, free of charge, to any person obtaining a
copy of this software and associated documentation files (the
"Software"), to deal in the Software without restriction, including
without limitation the rights to use, copy, modify, merge, publish,
distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so, subject to
the following conditions:

The above copyright notice and this permission notice shall be included
in all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

*/

#include "global_things.h"
#include "translation_table.h"
#include "function_templates.h"

#define HYPHY_SITE_DEFAULT_BUFFER_SIZE 256

_TranslationTable hy_default_translation_table;

using namespace hy_global;

_List _TranslationTable::_list_of_default_tables(_List() < "ACGT" < "ACGU" <
                                                 "ACDEFGHIKLMNPQRSTVWY" < "01");

const _String &_TranslationTable::GetDefaultTable(long tableType) {

  switch (tableType) {
  case HY_TRANSLATION_TABLE_BINARY:
    return *(_String *)_list_of_default_tables(3);
  case HY_TRANSLATION_TABLE_RNA:
    return *(_String *)_list_of_default_tables(1);
  case HY_TRANSLATION_TABLE_PROTEIN:
    return *(_String *)_list_of_default_tables(2);
  case HY_TRANSLATION_TABLE_DNA:
    return *(_String *)_list_of_default_tables(0);
  }

  return kEmptyString;
}

_TranslationTable::_TranslationTable(void) {
  baseLength = 4;
  checkTable = NULL;
}

//_________________________________________________________
_TranslationTable::_TranslationTable(unsigned char baseL) {
  baseLength = (baseL == 20) ? 20 : 4;
  checkTable = NULL;
}

//_________________________________________________________
_TranslationTable::_TranslationTable(_TranslationTable const &t) {
   *this = t;
}
                                                 
//_________________________________________________________
_TranslationTable const & _TranslationTable::operator = (_TranslationTable const &t) {
   if (this != &t) {
       tokensAdded = t.tokensAdded;
       baseLength = t.baseLength;
       baseSet = t.baseSet;
       translationsAdded << t.translationsAdded;
       checkTable = NULL;
    }
    return *this;
 }

//_________________________________________________________
_TranslationTable::_TranslationTable(_String &alphabet) {
  baseLength = alphabet.length();
  checkTable = NULL;
  if (_list_of_default_tables.FindObject(&alphabet) < 0L) {
    AddBaseSet(alphabet);
  }
}

//_________________________________________________________
BaseRef _TranslationTable::makeDynamic(void) const {
  _TranslationTable *r = new _TranslationTable;
  r->baseLength = baseLength;
  r->tokensAdded.Duplicate(&tokensAdded);
  r->baseSet.Duplicate(&baseSet);
  r->translationsAdded.Duplicate(&translationsAdded);
  r->checkTable = NULL;
  return r;
}

//_________________________________________________________
void _TranslationTable::Duplicate(BaseRefConst source) {
  _TranslationTable const *s = (_TranslationTable const *)source;
  tokensAdded.Duplicate(&s->tokensAdded);
  baseSet.Duplicate(&s->baseSet);
  translationsAdded.Duplicate(&s->translationsAdded);
  if (checkTable) {
    free(checkTable);
  };
  checkTable = NULL;
}

//_________________________________________________________
long _TranslationTable::TokenCode(char token) const {
  // standard translations
  long receptacle[256], resolution_count = TokenResolutions(token, receptacle);

  long theCode = 0L;

  for (unsigned long i = 0; i < resolution_count; i++) {
    theCode |= (1L << receptacle[i]); // set the right bit
  }

  return theCode;
}

//_________________________________________________________
char _TranslationTable::AmbigToLetter(long *split,
                                      unsigned long resolutions) const
// assumes a non-unique translation of split
// for unique - use ConvertCodeToLetters
{
  long encoding = 0L;

  for (unsigned long k = 0UL; k < resolutions; k++) {
    encoding |= (1L << split[k]);
  }

  if (baseSet.length() == 0)
  // one of the standard alphabers
  {
    if (baseLength == 4)
    // nucleotides
    {
      switch (encoding) {
      case 3:
        return 'M';
      case 5:
        return 'S';
      case 6:
        return 'R';
      case 7:
        return 'V';
      case 9:
        return 'W';
      case 10:
        return 'Y';
      case 11:
        return 'H';
      case 12:
        return 'K';
      case 14:
        return 'B';
      }
    } else if (baseLength == 20)
    // amino acids
    {
      switch (encoding) {
      case 2052:
        return 'B';
      case 8200:
        return 'Z';
      }
    }
  } else if (tokensAdded.length()) {
    long lookup = translationsAdded.Find(encoding);
    // linear search for (binary) translations
    if (lookup >= 0L) {
      return tokensAdded.char_at(lookup);
    }
  }
  return '?';
}

//_________________________________________________________
void _TranslationTable::SplitTokenCode(long code, long *receptacle) const {
  unsigned long shifter = 1L;
  for (unsigned int i = 0; i < baseLength; i++) {
    receptacle[i] = ((code & shifter) != 0) ? 1L : 0L;
    shifter >>= 1;
  }
}

//_________________________________________________________
long _TranslationTable::LengthOfAlphabet(void) const {
  return baseSet.length() ? baseSet.length() : baseLength;
}

//_________________________________________________________

const _String _TranslationTable::ExpandToken(char token) const {
  long buf[256];

  long resolution_count = TokenResolutions(token, buf);
  _String const *base_set = &GetAlphabetString();
  _StringBuffer expansion(base_set->length());

  for (long tc = 0; tc < resolution_count; tc++) {
    expansion << base_set->char_at(buf[tc]);
  }

  return expansion;
}

//_________________________________________________________

long _TranslationTable::MultiTokenResolutions(_String const &tokens,
                                              long *receptacle,
                                              bool gapToOnes) const {

  if (tokens.length() == 1UL) {
    return TokenResolutions(tokens.char_at (0UL), receptacle, gapToOnes);
  } else {

    long *large_store, large_store_static[HYPHY_SITE_DEFAULT_BUFFER_SIZE];

    if ((baseLength + 1)* tokens.length()  >=
        HYPHY_SITE_DEFAULT_BUFFER_SIZE) {
      large_store = new long[baseLength * tokens.length() + tokens.length()];
    } else {
      large_store = large_store_static;
    }

    /*
     large_store is a linear array which stores the following data

     [0,unitLength) -- the number of resolutions for the i-th character

     [unitLength,unitLength + baseLength] -- the actual resolutions for the 1st
     char [unitLength + baseLength, unitLength + 2*baseLength] -- the actual
     resolutions for the 2nd char
     ...
     */

    long resolution_count = 1L;

    for (unsigned long char_index = 0; char_index < tokens.length();
         char_index++) {
      large_store[char_index] = TokenResolutions(
          tokens.char_at(char_index),
          large_store + tokens.length() + baseLength * char_index, gapToOnes);
      if (gapToOnes && large_store[char_index] == 0) {
        large_store[char_index] = baseLength;
        InitializeArray(large_store + tokens.length() + baseLength * char_index,
                        baseLength, 1L);
      }
      resolution_count *=
          large_store[char_index] > 0 ? large_store[char_index] : 0;
    }

    if (resolution_count == 1L) {
      for (unsigned long char_index = 0; char_index < tokens.length();
           char_index++) {
        large_store[char_index] =
            large_store[tokens.length() + baseLength * char_index];
      }

      if (receptacle) {
        receptacle[0] = CombineDigits(large_store, tokens.length(), baseLength);
      } else {
        resolution_count =
            CombineDigits(large_store, tokens.length(), baseLength);
      }
    } else {
      if (receptacle) {
        // handle cases of 2 and 3 characters separately since they are the most
        // common

        if (resolution_count > HYPHY_SITE_DEFAULT_BUFFER_SIZE) {
          HandleApplicationError(
              (_String("Too many ambiguous states in call to ") &
               _String(__PRETTY_FUNCTION__).Enquote()));
          return -1L;
        }

        if (tokens.length() == 3) {
          long digits[3],
              *resolution_arrays[3] = {large_store + tokens.length(),
                                       large_store + tokens.length() +
                                           baseLength,
                                       large_store + tokens.length() +
                                           2 * baseLength},
              resolutions_index = 0L;

          for ( long digit1 = 0L; digit1 < large_store[0]; digit1++) {
            for ( long digit2 = 0L; digit2 < large_store[1]; digit2++) {
              for ( long digit3 = 0L; digit3 < large_store[2];
                   digit3++) {
                receptacle[resolutions_index++] =
                    resolution_arrays[0][digit1] * baseLength * baseLength +
                    resolution_arrays[1][digit2] * baseLength +
                    resolution_arrays[2][digit3];
              }
            }
          }

        } else {
          if (tokens.length() == 2) {
            long *resolution_arrays[2] = {large_store + tokens.length(),
                                         large_store + tokens.length() +
                                             baseLength},
                resolutions_index = 0L;

            for ( long digit1 = 0L; digit1 < large_store[0]; digit1++) {
                for ( long digit2 = 0L; digit2 < large_store[1];
                   digit2++) {
                receptacle[resolutions_index++] =
                    resolution_arrays[0][digit1] * baseLength +
                    resolution_arrays[1][digit2];
              }
            }
          } else { // more than 3 tokens [rare!]

            if (tokens.length() >= 32) {
              HandleApplicationError(
                  _String("The token string is too long in call to ") &
                  _String(__PRETTY_FUNCTION__).Enquote());
              return -1L;
            }

            long digits[32]{}, resolutions_index = 0L;

            do {
              // assemble the current token, backwards
              long this_resolution = 0L, weight = 1L;
              for (long digit = tokens.length() - 1; digit >= 0; digit--) {
                this_resolution +=
                    weight * *(large_store + tokens.length() +
                               baseLength * digit + digits[digit]);
                weight *= tokens.length();
              }

              receptacle[resolutions_index++] = this_resolution;

              for (long digit = tokens.length() - 1; digit >= 0; digit--) {
                if (++digits[digit] < large_store[digit]) {
                  break;
                }
                if (digit > 0) {
                  digits[digit] = 0L;
                }
              }

            } while (digits[0] < large_store[0]);
          }
        }
      } else {
        resolution_count = -1L;
      }
    }

    if (large_store != large_store_static) {
      delete[] large_store;
    }

    return resolution_count;
  }
}

//_________________________________________________________

long _TranslationTable::TokenResolutions(char token, long *receptacle,
                                         bool gapToOnes) const {

  long custom_code = tokensAdded.length() ? tokensAdded.Find(token) : -1;
  long resolution_counter = -1L;

  if (custom_code != -1) {
    resolution_counter = 0L;
    unsigned long shifter = 1L;
    for (unsigned long i = 0UL; i < baseLength; i++) {
      if ((custom_code & shifter) != 0) {
        receptacle[resolution_counter++] = i;
      }
      shifter >>= 1;
    }
  } else {

    if (baseSet.length()) {

      long base_char = baseSet.Find(token);
      // OPTIMIZE FLAG linear search:
      // SLKP 20071002 should really be a 256 char lookup table

      if (base_char != -1) {
        resolution_counter = 1;
        receptacle[0] = base_char;
      }
    } else {

      if (baseLength == 4) {

        switch (token) {
        case 'A':
          resolution_counter = 1L;
          receptacle[0] = 0;
          break;

        case 'C':
          resolution_counter = 1L;
          receptacle[0] = 1;
          break;

        case 'G':
          resolution_counter = 1L;
          receptacle[0] = 2;
          break;

        case 'T':
        case 'U':
          resolution_counter = 1L;
          receptacle[0] = 3;
          break;

        case 'Y':
          resolution_counter = 2L;
          receptacle[0] = 1;
          receptacle[1] = 3;
          break;

        case 'R':
          resolution_counter = 2L;
          receptacle[0] = 0;
          receptacle[1] = 2;
          break;

        case 'W':
          resolution_counter = 2L;
          receptacle[0] = 0;
          receptacle[1] = 3;
          break;

        case 'S':
          resolution_counter = 2L;
          receptacle[0] = 1;
          receptacle[1] = 2;
          break;

        case 'K':
          resolution_counter = 2L;
          receptacle[0] = 2;
          receptacle[1] = 3;
          break;

        case 'M':
          resolution_counter = 2L;
          receptacle[0] = 0;
          receptacle[1] = 1;
          break;

        case 'B':
          resolution_counter = 3L;
          receptacle[0] = 1;
          receptacle[1] = 2;
          receptacle[2] = 3;
          break;

        case 'D':
          resolution_counter = 3L;
          receptacle[0] = 0;
          receptacle[1] = 2;
          receptacle[2] = 3;
          break;

        case 'H':
          resolution_counter = 3L;
          receptacle[0] = 0;
          receptacle[1] = 1;
          receptacle[2] = 3;
          break;

        case 'V':
          resolution_counter = 3L;
          receptacle[0] = 0;
          receptacle[1] = 1;
          receptacle[2] = 2;
          break;

        case 'X':
        case 'N':
        case '?':
        case '.':
        case '*':
          resolution_counter = 4L;
          receptacle[0] = 0;
          receptacle[1] = 1;
          receptacle[2] = 2;
          receptacle[3] = 3;
          break;

        case '-':
          resolution_counter = 0L;
          break;
        }
      } else {
        if (baseLength == 20) {

          switch (token) {
          case 'A':
            resolution_counter = 1L;
            receptacle[0] = 0;
            break;

          case 'B':
            resolution_counter = 2L;
            receptacle[0] = 2;
            receptacle[1] = 11;
            break;

          case 'C':
            resolution_counter = 1L;
            receptacle[0] = 1;
            break;

          case 'D':
            resolution_counter = 1L;
            receptacle[0] = 2;
            break;

          case 'E':
            resolution_counter = 1L;
            receptacle[0] = 3;
            break;

          case 'F':
            resolution_counter = 1L;
            receptacle[0] = 4;
            break;

          case 'G':
            resolution_counter = 1L;
            receptacle[0] = 5;
            break;

          case 'H':
            resolution_counter = 1L;
            receptacle[0] = 6;
            break;

          case 'I':
            resolution_counter = 1L;
            receptacle[0] = 7;
            break;

          case 'K':
            resolution_counter = 1L;
            receptacle[0] = 8;
            break;

          case 'L':
            resolution_counter = 1L;
            receptacle[0] = 9;
            break;

          case 'M':
            resolution_counter = 1L;
            receptacle[0] = 10;
            break;

          case 'N':
            resolution_counter = 1L;
            receptacle[0] = 11;
            break;

          case 'P':
            resolution_counter = 1L;
            receptacle[0] = 12;
            break;

          case 'Q':
            resolution_counter = 1L;
            receptacle[0] = 13;
            break;

          case 'R':
            resolution_counter = 1L;
            receptacle[0] = 14;
            break;

          case 'S':
            resolution_counter = 1L;
            receptacle[0] = 15;
            break;

          case 'T':
            resolution_counter = 1L;
            receptacle[0] = 16;
            break;

          case 'V':
            resolution_counter = 1L;
            receptacle[0] = 17;
            break;

          case 'W':
            resolution_counter = 1L;
            receptacle[0] = 18;
            break;

          case 'Y':
            resolution_counter = 1L;
            receptacle[0] = 19;
            break;

          case 'Z':
            resolution_counter = 2L;
            receptacle[0] = 3;
            receptacle[1] = 13;
            break;

          case 'X':
          case '?':
          case '.':
          case '*': {
            resolution_counter = 20L;
            for (unsigned long j = 0UL; j < 20UL; j++) {
              receptacle[j] = j;
            }
          } break;
          case '-': {
            resolution_counter = 0L;
          } break;
          }
        } else
        // binary
        {

          switch (token) {
          case '0':
            resolution_counter = 1L;
            receptacle[0] = 0;
            break;

          case '1':
            resolution_counter = 1L;
            receptacle[0] = 1;
            break;

          case 'X':
          case '?':
          case '.':
          case '*': {
            resolution_counter = 2L;
            receptacle[0] = 0;
            receptacle[1] = 1;
          } break;
          case '-': {
            resolution_counter = 0L;
          } break;
          }
        }
      }
    }
  }

  if (resolution_counter == 0L && gapToOnes) {
    for (unsigned long i = 0UL; i < baseLength; i++) {
      receptacle[i] = i;
    }
    return baseLength;
  }

  return resolution_counter;
}

//_________________________________________________________
void _TranslationTable::PrepareForChecks(void) {
  if (checkTable == NULL) {
    checkTable = (char *)MemAllocate(256);
  }

  InitializeArray(checkTable, 256, (char)0);

  _String checkSymbols;
  //  if (baseLength == 4)
  //      checkSymbols = _String("ACGTUYRWSKMBDHVXN?O-.")&tokensAdded;
  if (baseSet.length()) {
    checkSymbols = baseSet & tokensAdded;
  } else if (baseLength == 2) {
    checkSymbols = _String("01*?-.") & tokensAdded;
  } else {
    checkSymbols = _String("ABCDEFGHIJKLMNOPQRSTUVWXYZ*?-.") & tokensAdded;
  }

  for (long i = 0; i < checkSymbols.length(); i++) {
    checkTable[(unsigned char)checkSymbols(i)] = (char)1;
  }
}

//_________________________________________________________
bool _TranslationTable::IsCharLegal(char c) {
  if (!checkTable) {
    PrepareForChecks();
  }
  return checkTable[(unsigned char)c];
}

const _String &_TranslationTable::GetAlphabetString(void) const {
  if (baseSet.length()) {
    return baseSet;
  }

  if (baseLength == 4) {
    return _TranslationTable::GetDefaultTable(HY_TRANSLATION_TABLE_DNA);
  } else if (baseLength == 20) {
    return _TranslationTable::GetDefaultTable(HY_TRANSLATION_TABLE_PROTEIN);
  } else {
    return _TranslationTable::GetDefaultTable(HY_TRANSLATION_TABLE_BINARY);
  }

  return kEmptyString;
}

//___________________________________________

void _TranslationTable::AddTokenCode(char token, _String const &code) {
  long f, newCode = 0;

  bool killBS = false;

  if (baseSet.length() == 0)
  // fill in baseSet for standard alphabets
  {
    if (baseLength == 4) {
      baseSet = _TranslationTable::GetDefaultTable(HY_TRANSLATION_TABLE_DNA);
    } else if (baseLength == 20) {
      baseSet =
          _TranslationTable::GetDefaultTable(HY_TRANSLATION_TABLE_PROTEIN);
    } else {
      baseSet = _TranslationTable::GetDefaultTable(HY_TRANSLATION_TABLE_BINARY);
    }
    killBS = true;
  }

  if (baseSet.length()) {
    long shifter = 1;
    for (int j = 0; j < baseSet.length(); j++, shifter *= 2)
      if (code.Find(baseSet.get_char(j)) >= 0) {
        newCode += shifter;
      }
  }

  f = baseSet.Find(token);

  if (killBS) {
    baseSet = kEmptyString;
  }

  if (f >= 0) {
    return;
  }
  // see if the character being added is a base
  // character; those cannot be redefined

  f = tokensAdded.Find(token, 0, -1);
  // new definition or redefinition?

  if (f == -1) { // new
    tokensAdded = tokensAdded & token;
    translationsAdded << 0;
    f = tokensAdded.length() - 1;
  }

  translationsAdded.list_data[f] = newCode;
}

//_________________________________________________________

void _TranslationTable::AddBaseSet(_String const &code) {
  baseSet = code;
  baseSet.StripQuotes();
  baseLength = baseSet.length();
  if (baseLength > HY_WIDTH_OF_LONG) {
    // longer than the bit size of 'long'
    // can't handle those
    HandleApplicationError(_String("Alphabets with more than ") &
                           HY_WIDTH_OF_LONG & " characters are not supported");
  }
}

//_________________________________________________________

char _TranslationTable::GetSkipChar(void) {
  if (baseSet.length() == 0 && translationsAdded.lLength == 0) {
    return '?'; // this is the default
  }

  // see if there is a symbol
  // which maps to all '1'

  long all = 0, ul = baseSet.length() ? baseSet.length() : baseLength,
       shifter = 1;

  for (long f = 0; f < ul; f++, shifter <<= 1) {
    all |= shifter;
  }

  if ((all = translationsAdded.Find(all)) == -1) {
    return '?';
  } else {
    return tokensAdded[all];
  }
}

//_________________________________________________________

char _TranslationTable::GetGapChar(void) const {
  if (baseSet.length() == 0 && translationsAdded.lLength == 0) {
    return '-'; // default gap character
  }

  long f = translationsAdded.Find(0L);

  return f >= 0 ? tokensAdded[f] : '\0';
}

//_________________________________________________________
const _String
_TranslationTable::ConvertCodeToLetters(long code, unsigned char base) const {

  _String res ((unsigned long)base);

  if (code >= 0) {
    // OPTIMIZE FLAG; repeated memory allocation/deallocation
    if (baseSet.length())
      for (long k = 1; k <= base; k++, code /= baseLength) {
        res.set_char(base - k,baseSet.char_at(code % baseLength));
      }
    else if (baseLength == 4) {
      for (long k = 1; k <= base; k++, code /= baseLength) {
        switch (code % baseLength) {
        case 0:
          res[base - k] = 'A';
          break;
        case 1:
          res[base - k] = 'C';
          break;
        case 2:
          res[base - k] = 'G';
          break;
        case 3:
          res[base - k] = 'T';
          break;
        }
      }
    } else if (baseLength == 20) {
      for (long k = 1; k <= base; k++, code /= baseLength) {
        char out = code % baseLength;
        if (out == 0) {
          res[base - k] = 'A';
        } else if (out <= 7) {
          res[base - k] = 'B' + out;
        } else if (out <= 11) {
          res[base - k] = 'C' + out;
        } else if (out <= 16) {
          res[base - k] = 'D' + out;
        } else if (out <= 18) {
          res[base - k] = 'E' + out;
        } else {
          res[base - k] = 'Y';
        }
      }
    } else if (baseLength == 2)
      for (long k = 1; k <= base; k++, code /= baseLength) {
        switch (code % baseLength) {
        case 0:
          res[base - k] = '0';
          break;
        case 1:
          res[base - k] = '1';
          break;
        }
      }
  } else {
    char c = GetGapChar();
    for (long k = 0L; k < base; k++) {
      res.set_char(k,c);
    }
  }
  return res;
}

//_________________________________________________________

bool _TranslationTable::operator == (const _TranslationTable& rhs) const {
    
    if (baseSet.length() == rhs.baseSet.length()) {
        if (baseSet.empty()) { // standard alphabet
            if (baseLength != rhs.baseLength) {
                return false;
            }
        } else if (baseSet != rhs.baseSet) {
            return false;
        }
        
        if (tokensAdded.length() == rhs.tokensAdded.length()) {
            
            for (unsigned i = 0; i < tokensAdded.length(); i++) {
                if (ExpandToken (tokensAdded.get_char(i)) != rhs.ExpandToken (tokensAdded.get_char(i))) {
                    return false;
                }
            }
            
            return true;
        }
        
    }
    return false;

}

//_________________________________________________________

_TranslationTable *
_TranslationTable::MergeTables(_TranslationTable const *table2) const
// merge the translation tables if they are compatible, return the result,
// otherwise return nil
{
  if (baseSet.length() == table2->baseSet.length()) {
    if (baseSet.empty()) { // standard alphabet
      if (baseLength != table2->baseLength) {
        return nil;
      }
    } else if (baseSet != table2->baseSet) {
      return nil;
    }

    _TranslationTable *result = new _TranslationTable(*this);
    if (table2->tokensAdded.length()) {
      for (long i = 0; i < table2->tokensAdded.length(); i++) {
        long f = tokensAdded.Find(table2->tokensAdded[i]);
        if (f == -1) {
          result->tokensAdded = result->tokensAdded & table2->tokensAdded[i];
          // SLKP 20071002 added the next line;
          // was not adding the translation for the new token
          result->translationsAdded << table2->translationsAdded(i);
        } else if (translationsAdded.list_data[f] !=
                   table2->translationsAdded.list_data[i]) {
          DeleteObject(result);
          return nil;
        }
      }
      return result;
    } else {
      return result;
    }
  }
  return nil;
}


