/Users/dbr/ma/src/lmu/cis/sis/indexer/DocumentIndexingAutomaton.cpp

Go to the documentation of this file.

#ifndef DOCUMENTINDEXINGAUTOMATON_CPP
#define DOCUMENTINDEXINGAUTOMATON_CPP

#include "../cppbase.hpp"
#include "DocumentIndexingAutomaton.hpp"

namespace lmu { namespace cis { namespace sis {




/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/* * * * * * * * * * * * Specializations * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */

template<typename T>
void
DocumentIndexingAutomaton<T>::
AddDocuments( const std::vector<DocumentName>& documents ) {
    if (DEBUG_LEVEL >= 3) std::cerr << "virtual void DocumentIndexer::AddDocuments(std::vector<std::string>& documents)" << std::endl;

    assert( ManagedAutomatonStage_ < CLOSED );

    UINT total_length = 0;
    UINT number_of_docs_read = 0;

    _document_names = std::vector<std::string>(documents);

    for (auto& doc: _document_names) {
        number_of_docs_read++;
        if (DEBUG_LEVEL >= 3) std::cerr << "Adding document: " << doc << std::endl;

        VoidSequenceAdapterDollarized v2( T::get_symbol_size() );
        std::ifstream is(doc);
        is >> v2;

        State sinkstate = T::Add(v2.get()); // call to template's Add() (i.e. CDAWG::Add or SCDAWG::Add)!

        if (DEBUG_LEVEL >= 4) std::cerr << "Document " << doc << "'s sinkstate is " << sinkstate << std::endl;
        assert(sinkstate != NO);

        _document_sinkstate[doc] = sinkstate;

        if (DEBUG_LEVEL >= 4) std::cerr << doc << " -- current  length = " << v2.length() - 2                                         << std::endl;
        if (DEBUG_LEVEL >= 4) std::cerr << doc << " -- absolute length = " << v2.length() - 2 + total_length                          << std::endl;
        if (DEBUG_LEVEL >= 4) std::cerr << doc << " -- actual   length = " << v2.length() - 2 + total_length + number_of_docs_read*2-1<< std::endl;

        PositionsTuple pt(
            v2.length(),
            v2.length() + total_length,
            v2.length() + total_length + number_of_docs_read*2-1
        );

        _document_length[doc] = pt;

        total_length += v2.length() - 2;
    }

    ManagedAutomatonStage_ = UNSORTED;
}

template<typename T>
void
DocumentIndexingAutomaton<T>::Close() {
    if (DEBUG_LEVEL >= 4) std::cerr << "DocumentIndexingAutomaton<typename T>::Close()" << std::endl;
    T::Close();
    ManagedAutomatonStage_ = CLOSED;
}

template<typename T>
void
DocumentIndexingAutomaton<T>::
Index() {
    if (DEBUG_LEVEL >= 4) std::cerr << "virtual void DocumentIndexingAutomaton<T>::Index()" << std::endl;

    assert( ManagedAutomatonStage_ > EMPTY && ManagedAutomatonStage_ < CLOSED );

    if ( ManagedAutomatonStage_ < SORTED ) {
        if (DEBUG_LEVEL >= 4) std::cerr << "----UNSORTED-----" << std::endl;
        T::SortTransitions();
    } else {
        if (DEBUG_LEVEL >= 4) std::cerr << "----SORTED-----" << std::endl;
    }
    assert( ManagedAutomatonStage_ >= SORTED );

    if (ManagedAutomatonStage_ < INDEXED ) {
        State source = 0;
        State state  = 0;

        for (auto doc: _document_names) {
            state = _document_sinkstate[doc];

            if (DEBUG_LEVEL >= 4) std::cerr << "doc is " << doc << std::endl;
            if (DEBUG_LEVEL >= 4) std::cerr << "state is " << state << std::endl;
            if (DEBUG_LEVEL >= 4) std::cerr << "source is " << source << std::endl;

            // documents
            while(state != source){
                if (DEBUG_LEVEL >= 4) std::cerr << "state is " << state << std::endl;
                if (DEBUG_LEVEL >= 4) std::cerr << "........doc is " << doc << std::endl;
                _states_documents[state].push_back(doc);
                state = suffixLink(state);
                assert(state != NO);
                if (state == NO) {
                    std::cerr << "WARN state was NO" << std::endl;
                    state = source;
                }
            }
        }
    }
    ManagedAutomatonStage_ = INDEXED;
}




template<typename T>
DocumentIndexingAutomatonFindResults&&
DocumentIndexingAutomaton < T > ::
findall( const std::string& w ) {
    // 2. find all occurrences of a given string W
    if (DEBUG_LEVEL >= 4) std::cerr << "virtual unsigned int DocumentIndexingAutomaton<InenagaCDAWGAdapter>::findall(const std::string& w)" << std::endl;

    assert( ManagedAutomatonStage_ < CLOSED );

    if ( ManagedAutomatonStage_ < SORTED ) {
        SortTransitions();
    }

    if ( ManagedAutomatonStage_ < INDEXED) {
        Index();
    }

    VoidSequenceAdapter v(w.c_str());

    PairState pairState = GetInitialPairState();
    PairState nextPairState;
    for( size_t i = 0; i < v.length(); i++ ){
        nextPairState = Delta( &pairState, v.at(i) );
        if( nextPairState.index == NO ){
            if (DEBUG_LEVEL >= 2) std::cerr << "no occurences" << std::endl;
            return std::forward<DocumentIndexingAutomatonFindResults>(DocumentIndexingAutomatonFindResults(this)); // empty results
        } else {
            if (DEBUG_LEVEL >= 5) std::cerr << "State might be implicit or explicit find() will set it to the canonical representative" << std::endl;
        }

        pairState = nextPairState;
    }

    //TODO set assertions?
    return std::forward<DocumentIndexingAutomatonFindResults>(
        find( pairState, v.length() )
    );
}



template<typename T>
DocumentIndexingAutomatonFindResults&&
DocumentIndexingAutomaton<T>::
find( const PairState state, UINT length ) {
    if (DEBUG_LEVEL >= 5) std::cerr << "virtual void find( PairState state, int length = " << length << " )" << std::endl;
    if (DEBUG_LEVEL >= 5) std::cerr << "PairState's index is "  << state.index  << " length is " << length << std::endl;

    //3. the find method
    UINT s;
    if(state.transition != NO){ // check excplicit-ness of the state: if state is not explicit yet, "read" the "rest" (i.e. find the canonical representative)
        if (DEBUG_LEVEL >= 5) std::cerr << "IMPLICIT" << std::endl;
        length += automaton()->statesEnd->seq[automaton()->transitionsTo->seq[state.transition]] - state.index + 1; // state.transition is state-ID
        s = automaton()->transitionsTo->seq[state.transition];
    } else {
        if (DEBUG_LEVEL >= 5) std::cerr << "EXPLICIT" << std::endl;
        s = state.index;
    }
    return std::forward<DocumentIndexingAutomatonFindResults>(
        findRec(s, length)
    );
}

template<typename T>
DocumentIndexingAutomatonFindResults&&
DocumentIndexingAutomaton<T>::
findRec( const UINT s, const UINT length ) { // 4. the recursive method
    if (DEBUG_LEVEL >= 4) std::cerr << "virtual void findRec(int s = " << s << ", int length = " << length << ")" << std::endl;

    UINT i, position, abs_pos;
    // static DocumentIndexingAutomatonFindResults<T> results_acc(this);
    static DocumentIndexingAutomatonFindResults results_acc(this);

    for (auto doc: _states_documents[s]) {
        if (DEBUG_LEVEL >= 4) std::cout << "_document_length[ " << doc << "] = " << std::get<0>(_document_length[doc]) << std::endl;
        DocumentPosition current_position  = std::get<0>(_document_length[doc]) - length;
        DocumentPosition absolute_position = std::get<1>(_document_length[doc]) - length;
        DocumentPosition actual_position   = std::get<2>(_document_length[doc]) - length;

        PositionsTuple pt (
            current_position,
            absolute_position,
            actual_position
        );
        std::cout << "INSERTING " << doc <<  " ++  " << current_position <<  " ++  " << absolute_position << " ++  " << actual_position << std::endl;
        results_acc.insert_position(doc, pt);
    }
    int tr;
    if (DEBUG_LEVEL >= 4) std::cout << "  --> State " << s << " number of transitions = " << number_of_transitions(s) << std::endl;

    for(UINT i = 0; i < automaton()->statesNumberOfTransitions->seq[s]; i++) {
        State to = automaton()->transitionsTo->seq[s];
        if (DEBUG_LEVEL >= 4) std::cout << "  --> State " << s << " transition to " << to << std::endl;
    }

    for(UINT i = 0; i < automaton()->statesNumberOfTransitions->seq[s]; i++) {
        tr = automaton()->statesTransitions->seq[s] + i;
        findRec(automaton()->transitionsTo->seq[tr], length + automaton()->statesEnd->seq[automaton()->transitionsTo->seq[tr]] - automaton()->transitionsStart->seq[tr] + 1);
    }

    return std::forward<DocumentIndexingAutomatonFindResults>(results_acc);
}

template<typename T>
UINT
DocumentIndexingAutomaton<T>::
suffixLink( const UINT& state ) const {
    return std::forward<UINT>(T::suffixLink(state));
}

// template<typename T>
// std::string DocumentIndexingAutomaton<T>::data_at(UINT pos) const {
//     char * ptr = (S8*)mVoidSequenceElement( T::automaton()->data, pos );
//     return std::string(
//         ptr, 1
//     );
// }

// template<typename T>
// std::string DocumentIndexingAutomaton<T>::lrcont(const UINT pos, const UINT width) const {
//     std::string res;
// 
//     mSymbolAndVariables( zero )
//     mSymbolAndVariables( one )
//     mSymbolAssignValue( zero, 0, T::get_symbol_size() )
//     mSymbolAssignValue( one, 1, T::get_symbol_size() )
// 
//     if (width > pos) {
//         for (auto i = width - pos; i > 0; i--)
//             res.append( data_at(i) );
//     }
//     for (auto i = width > pos ? 0 : pos - width; i < pos+width; i++) { // make sure the unsigned ints start from 0.
//         std::string s = data_at(i);
//         if ( (memcmp(zero, s.data(), 1) == 0) ) {
//             std::cout << "HIT LEFT BORDER." << std::endl;
//             i = 0;
//         }
//         if ( (memcmp(one, s.data(), 1) == 0) ) {
//             std::cout << "HIT RIGHT BORDER." << std::endl;
//             i = pos+width; // break.
//         }
//         res.append( s );
//     }
//     return std::forward<std::string>(res);
// }

}}} /* End of namespace lmu::cis::sis */

#endif /* end of include guard: DOCUMENTINDEXINGAUTOMATON_CPP */