#pragma once
#include "../GroupBehavior.h"
#include "../../Commands/MoveCommand.h"
#include "../../../../../Scheduler/SchedulerTaskHelper.h"
#include <time.h>

static const double EPSILON = 0.2;
static const double LAMBDA = 0.7;

/**
* @brief Un FSM pour tester l'algo d'apprentissage par renforcement SARSA 
*/
class LearningCombatBehavior : public GroupBehaviorT<LearningCombatBehavior>
{
protected:
  class NoAction;
  class AttackRandom;
  class AttackStrongest;
  class AttackDying;
  class AttackClosest;
  class Flee;
  class RemoveTargetedUnit;
  class PutBackSafeUnit;
  friend class NoAction;
  friend class AttackRandom;
  friend class AttackStrongest;
  friend class AttackDying;
  friend class AttackClosest;
  friend class Flee;
  friend class RemoveTargetedUnit;
  friend class PutBackSafeUnit;

  class NoAction : public BehaviorStateT<LearningCombatBehavior, NoAction>
  {
  public:
    NoAction(LearningCombatBehavior& behavior);
    void onEnter();
    void onExit() {}

    void onTimer()
    {
      m_context.calculateNextAction();
      //if current action = NoAction
      if(m_context.currentAction == NO_ACTION)
      {
        //Do nothing
      }
      //else
      else
      {
        //if current action = AttackRandom
        //transition to AttackRandom
        if(m_context.currentAction == ATTACK_RANDOM){m_transitionToARState.testTransition(boost::make_tuple());}
        //if current action = AttackStrongest
        //transition to AttackStrongest
        if(m_context.currentAction == ATTACK_STRONGEST){m_transitionToASState.testTransition(boost::make_tuple());}
        //if current action = AttackDying
        //transition to AttackDying
        if(m_context.currentAction == ATTACK_DYING){m_transitionToADState.testTransition(boost::make_tuple());}
        //if current action = AttackClosest
        //transition to AttackClosest
        if(m_context.currentAction == ATTACK_CLOSEST){m_transitionToACState.testTransition(boost::make_tuple());}
        //if current action = Flee
        //transition to Flee
        if(m_context.currentAction == FLEE){m_transitionToFleeState.testTransition(boost::make_tuple());}
        //if current action = RemoveTargetedUnit
        //transition to RemoveTargetedUnit
        if(m_context.currentAction == REMOVE_TARGETED_UNIT){m_transitionToRemoveState.testTransition(boost::make_tuple());}
        //if current action = PutBackSafeUnit
        //transition to PutBackSafeUnit   
        if(m_context.currentAction == PUT_BACK_SAFE_UNIT){m_transitionToPutBackState.testTransition(boost::make_tuple());}
      }
    }

    typedef SCHEDULER_EVENT_HANDLER(NoAction, onTimer) TimerEvent;

    typedef boost::tuple<
      TimerEvent&
    > Events;
    const Events m_events;

    typedef boost::tuple<
      Transition<NoAction, NoCondition, NoTransitionAction, AttackRandom>& ,
      Transition<NoAction, NoCondition, NoTransitionAction, AttackStrongest>& ,
      Transition<NoAction, NoCondition, NoTransitionAction, AttackDying>& ,
      Transition<NoAction, NoCondition, NoTransitionAction, AttackClosest>& ,
      Transition<NoAction, NoCondition, NoTransitionAction, Flee>& ,
      Transition<NoAction, NoCondition, NoTransitionAction, RemoveTargetedUnit>& ,
      Transition<NoAction, NoCondition, NoTransitionAction, PutBackSafeUnit>&
    > Transitions;
    const Transitions m_transitions;

  protected:
    // Variables

    // Actions

    // Transitions
    TimerEvent m_timerEvent;
    Transition<NoAction, NoCondition, NoTransitionAction, AttackRandom> m_transitionToARState;
    Transition<NoAction, NoCondition, NoTransitionAction, AttackStrongest> m_transitionToASState;
    Transition<NoAction, NoCondition, NoTransitionAction, AttackDying> m_transitionToADState;
    Transition<NoAction, NoCondition, NoTransitionAction, AttackClosest> m_transitionToACState;
    Transition<NoAction, NoCondition, NoTransitionAction, Flee> m_transitionToFleeState;
    Transition<NoAction, NoCondition, NoTransitionAction, RemoveTargetedUnit> m_transitionToRemoveState;
    Transition<NoAction, NoCondition, NoTransitionAction, PutBackSafeUnit> m_transitionToPutBackState;
  };

  class AttackRandom : public BehaviorStateT<LearningCombatBehavior, AttackRandom>
  {
  public:
    AttackRandom(LearningCombatBehavior& behavior);
    void onEnter();
    void onExit() {}

    void onTimer()
    {
      m_context.calculateNextAction();
      //if current action = NoAction
      if(m_context.currentAction == NO_ACTION)
      {
        //Check each unit target, if dead find a new random one
      }
      //else
      else
      {
        //if current action = AttackRandom
        //transition to AttackRandom
        if(m_context.currentAction == ATTACK_RANDOM){m_transitionToARState.testTransition(boost::make_tuple());}
        //if current action = AttackStrongest
        //transition to AttackStrongest
        if(m_context.currentAction == ATTACK_STRONGEST){m_transitionToASState.testTransition(boost::make_tuple());}
        //if current action = AttackDying
        //transition to AttackDying
        if(m_context.currentAction == ATTACK_DYING){m_transitionToADState.testTransition(boost::make_tuple());}
        //if current action = AttackClosest
        //transition to AttackClosest
        if(m_context.currentAction == ATTACK_CLOSEST){m_transitionToACState.testTransition(boost::make_tuple());}
        //if current action = Flee
        //transition to Flee
        if(m_context.currentAction == FLEE){m_transitionToFleeState.testTransition(boost::make_tuple());}
        //if current action = RemoveTargetedUnit
        //transition to RemoveTargetedUnit
        if(m_context.currentAction == REMOVE_TARGETED_UNIT){m_transitionToRemoveState.testTransition(boost::make_tuple());}
        //if current action = PutBackSafeUnit
        //transition to PutBackSafeUnit   
        if(m_context.currentAction == PUT_BACK_SAFE_UNIT){m_transitionToPutBackState.testTransition(boost::make_tuple());}
      }
    }

    typedef SCHEDULER_EVENT_HANDLER(AttackRandom, onTimer) TimerEvent;

    typedef boost::tuple<
      TimerEvent&
    > Events;
    const Events m_events;

    typedef boost::tuple<
      Transition<AttackRandom, NoCondition, NoTransitionAction, AttackRandom>& ,
      Transition<AttackRandom, NoCondition, NoTransitionAction, AttackStrongest>& ,
      Transition<AttackRandom, NoCondition, NoTransitionAction, AttackDying>& ,
      Transition<AttackRandom, NoCondition, NoTransitionAction, AttackClosest>& ,
      Transition<AttackRandom, NoCondition, NoTransitionAction, Flee>& ,
      Transition<AttackRandom, NoCondition, NoTransitionAction, RemoveTargetedUnit>& ,
      Transition<AttackRandom, NoCondition, NoTransitionAction, PutBackSafeUnit>&
    > Transitions;
    const Transitions m_transitions;

  protected:
    // Variables

    // Actions

    // Transitions
    TimerEvent m_timerEvent;
    Transition<AttackRandom, NoCondition, NoTransitionAction, AttackRandom> m_transitionToARState;
    Transition<AttackRandom, NoCondition, NoTransitionAction, AttackStrongest> m_transitionToASState;
    Transition<AttackRandom, NoCondition, NoTransitionAction, AttackDying> m_transitionToADState;
    Transition<AttackRandom, NoCondition, NoTransitionAction, AttackClosest> m_transitionToACState;
    Transition<AttackRandom, NoCondition, NoTransitionAction, Flee> m_transitionToFleeState;
    Transition<AttackRandom, NoCondition, NoTransitionAction, RemoveTargetedUnit> m_transitionToRemoveState;
    Transition<AttackRandom, NoCondition, NoTransitionAction, PutBackSafeUnit> m_transitionToPutBackState;
  };


  class AttackStrongest : public BehaviorStateT<LearningCombatBehavior, AttackStrongest>
  {
  public:
    AttackStrongest(LearningCombatBehavior& behavior);
    void onEnter();
    void onExit() {}

    void onTimer()
    {
       m_context.calculateNextAction();
      //if current action = NoAction
      if(m_context.currentAction == NO_ACTION)
      {
        //Check if target is dead
        //if it is find new strongest enemy
        //else check if there is a new strongest enemy
      }
      //else
      else
      {
        //if current action = AttackRandom
        //transition to AttackRandom
        if(m_context.currentAction == ATTACK_RANDOM){m_transitionToARState.testTransition(boost::make_tuple());}
        //if current action = AttackStrongest
        //transition to AttackStrongest
        if(m_context.currentAction == ATTACK_STRONGEST){m_transitionToASState.testTransition(boost::make_tuple());}
        //if current action = AttackDying
        //transition to AttackDying
        if(m_context.currentAction == ATTACK_DYING){m_transitionToADState.testTransition(boost::make_tuple());}
        //if current action = AttackClosest
        //transition to AttackClosest
        if(m_context.currentAction == ATTACK_CLOSEST){m_transitionToACState.testTransition(boost::make_tuple());}
        //if current action = Flee
        //transition to Flee
        if(m_context.currentAction == FLEE){m_transitionToFleeState.testTransition(boost::make_tuple());}
        //if current action = RemoveTargetedUnit
        //transition to RemoveTargetedUnit
        if(m_context.currentAction == REMOVE_TARGETED_UNIT){m_transitionToRemoveState.testTransition(boost::make_tuple());}
        //if current action = PutBackSafeUnit
        //transition to PutBackSafeUnit   
        if(m_context.currentAction == PUT_BACK_SAFE_UNIT){m_transitionToPutBackState.testTransition(boost::make_tuple());}
      }
    }

    typedef SCHEDULER_EVENT_HANDLER(AttackStrongest, onTimer) TimerEvent;

    typedef boost::tuple<
      TimerEvent&
    > Events;
    const Events m_events;

    typedef boost::tuple<
      Transition<AttackStrongest, NoCondition, NoTransitionAction, AttackRandom>& ,
      Transition<AttackStrongest, NoCondition, NoTransitionAction, AttackStrongest>& ,
      Transition<AttackStrongest, NoCondition, NoTransitionAction, AttackDying>& ,
      Transition<AttackStrongest, NoCondition, NoTransitionAction, AttackClosest>& ,
      Transition<AttackStrongest, NoCondition, NoTransitionAction, Flee>& ,
      Transition<AttackStrongest, NoCondition, NoTransitionAction, RemoveTargetedUnit>& ,
      Transition<AttackStrongest, NoCondition, NoTransitionAction, PutBackSafeUnit>&
    > Transitions;
    const Transitions m_transitions;

  protected:
    // Variables

    // Actions

    // Transitions
    TimerEvent m_timerEvent;
    Transition<AttackStrongest, NoCondition, NoTransitionAction, AttackRandom> m_transitionToARState;
    Transition<AttackStrongest, NoCondition, NoTransitionAction, AttackStrongest> m_transitionToASState;
    Transition<AttackStrongest, NoCondition, NoTransitionAction, AttackDying> m_transitionToADState;
    Transition<AttackStrongest, NoCondition, NoTransitionAction, AttackClosest> m_transitionToACState;
    Transition<AttackStrongest, NoCondition, NoTransitionAction, Flee> m_transitionToFleeState;
    Transition<AttackStrongest, NoCondition, NoTransitionAction, RemoveTargetedUnit> m_transitionToRemoveState;
    Transition<AttackStrongest, NoCondition, NoTransitionAction, PutBackSafeUnit> m_transitionToPutBackState;
  };


  class AttackDying : public BehaviorStateT<LearningCombatBehavior, AttackDying>
  {
  public:
    AttackDying(LearningCombatBehavior& behavior);
    void onEnter();
    void onExit() {}

    void onTimer()
    {
      m_context.calculateNextAction();
      //if current action = NoAction
      if(m_context.currentAction == NO_ACTION)
      {
        //Check if enemy is dead
        //find a new dying enemy to attack if he is
      }
      //else
      else
      {
        //if current action = AttackRandom
        //transition to AttackRandom
        if(m_context.currentAction == ATTACK_RANDOM){m_transitionToARState.testTransition(boost::make_tuple());}
        //if current action = AttackStrongest
        //transition to AttackStrongest
        if(m_context.currentAction == ATTACK_STRONGEST){m_transitionToASState.testTransition(boost::make_tuple());}
        //if current action = AttackDying
        //transition to AttackDying
        if(m_context.currentAction == ATTACK_DYING){m_transitionToADState.testTransition(boost::make_tuple());}
        //if current action = AttackClosest
        //transition to AttackClosest
        if(m_context.currentAction == ATTACK_CLOSEST){m_transitionToACState.testTransition(boost::make_tuple());}
        //if current action = Flee
        //transition to Flee
        if(m_context.currentAction == FLEE){m_transitionToFleeState.testTransition(boost::make_tuple());}
        //if current action = RemoveTargetedUnit
        //transition to RemoveTargetedUnit
        if(m_context.currentAction == REMOVE_TARGETED_UNIT){m_transitionToRemoveState.testTransition(boost::make_tuple());}
        //if current action = PutBackSafeUnit
        //transition to PutBackSafeUnit   
        if(m_context.currentAction == PUT_BACK_SAFE_UNIT){m_transitionToPutBackState.testTransition(boost::make_tuple());}
      }
    }

    typedef SCHEDULER_EVENT_HANDLER(AttackDying, onTimer) TimerEvent;

    typedef boost::tuple<
      TimerEvent&
    > Events;
    const Events m_events;

    typedef boost::tuple<
      Transition<AttackDying, NoCondition, NoTransitionAction, AttackRandom>& ,
      Transition<AttackDying, NoCondition, NoTransitionAction, AttackStrongest>& ,
      Transition<AttackDying, NoCondition, NoTransitionAction, AttackDying>& ,
      Transition<AttackDying, NoCondition, NoTransitionAction, AttackClosest>& ,
      Transition<AttackDying, NoCondition, NoTransitionAction, Flee>& ,
      Transition<AttackDying, NoCondition, NoTransitionAction, RemoveTargetedUnit>& ,
      Transition<AttackDying, NoCondition, NoTransitionAction, PutBackSafeUnit>&
    > Transitions;
    const Transitions m_transitions;

  protected:
    // Variables

    // Actions

    // Transitions
    TimerEvent m_timerEvent;
    Transition<AttackDying, NoCondition, NoTransitionAction, AttackRandom> m_transitionToARState;
    Transition<AttackDying, NoCondition, NoTransitionAction, AttackStrongest> m_transitionToASState;
    Transition<AttackDying, NoCondition, NoTransitionAction, AttackDying> m_transitionToADState;
    Transition<AttackDying, NoCondition, NoTransitionAction, AttackClosest> m_transitionToACState;
    Transition<AttackDying, NoCondition, NoTransitionAction, Flee> m_transitionToFleeState;
    Transition<AttackDying, NoCondition, NoTransitionAction, RemoveTargetedUnit> m_transitionToRemoveState;
    Transition<AttackDying, NoCondition, NoTransitionAction, PutBackSafeUnit> m_transitionToPutBackState;
  };

  class AttackClosest : public BehaviorStateT<LearningCombatBehavior, AttackClosest>
  {
  public:
    AttackClosest(LearningCombatBehavior& behavior);
    void onEnter();
    void onExit() {}

    void onTimer()
    {
      m_context.calculateNextAction();
      //if current action = NoAction
      if(m_context.currentAction == NO_ACTION)
      {
        //For each unit check its target and determine if there is a closer one
      }
      //else
      else
      {
        //if current action = AttackRandom
        //transition to AttackRandom
        if(m_context.currentAction == ATTACK_RANDOM){m_transitionToARState.testTransition(boost::make_tuple());}
        //if current action = AttackStrongest
        //transition to AttackStrongest
        if(m_context.currentAction == ATTACK_STRONGEST){m_transitionToASState.testTransition(boost::make_tuple());}
        //if current action = AttackDying
        //transition to AttackDying
        if(m_context.currentAction == ATTACK_DYING){m_transitionToADState.testTransition(boost::make_tuple());}
        //if current action = AttackClosest
        //transition to AttackClosest
        if(m_context.currentAction == ATTACK_CLOSEST){m_transitionToACState.testTransition(boost::make_tuple());}
        //if current action = Flee
        //transition to Flee
        if(m_context.currentAction == FLEE){m_transitionToFleeState.testTransition(boost::make_tuple());}
        //if current action = RemoveTargetedUnit
        //transition to RemoveTargetedUnit
        if(m_context.currentAction == REMOVE_TARGETED_UNIT){m_transitionToRemoveState.testTransition(boost::make_tuple());}
        //if current action = PutBackSafeUnit
        //transition to PutBackSafeUnit   
        if(m_context.currentAction == PUT_BACK_SAFE_UNIT){m_transitionToPutBackState.testTransition(boost::make_tuple());}
      }
    }

    typedef SCHEDULER_EVENT_HANDLER(AttackClosest, onTimer) TimerEvent;

    typedef boost::tuple<
      TimerEvent&
    > Events;
    const Events m_events;

    typedef boost::tuple<
      Transition<AttackClosest, NoCondition, NoTransitionAction, AttackRandom>& ,
      Transition<AttackClosest, NoCondition, NoTransitionAction, AttackStrongest>& ,
      Transition<AttackClosest, NoCondition, NoTransitionAction, AttackDying>& ,
      Transition<AttackClosest, NoCondition, NoTransitionAction, AttackClosest>& ,
      Transition<AttackClosest, NoCondition, NoTransitionAction, Flee>& ,
      Transition<AttackClosest, NoCondition, NoTransitionAction, RemoveTargetedUnit>& ,
      Transition<AttackClosest, NoCondition, NoTransitionAction, PutBackSafeUnit>&
    > Transitions;
    const Transitions m_transitions;

  protected:
    // Variables

    // Actions

    // Transitions
    TimerEvent m_timerEvent;
    Transition<AttackClosest, NoCondition, NoTransitionAction, AttackRandom> m_transitionToARState;
    Transition<AttackClosest, NoCondition, NoTransitionAction, AttackStrongest> m_transitionToASState;
    Transition<AttackClosest, NoCondition, NoTransitionAction, AttackDying> m_transitionToADState;
    Transition<AttackClosest, NoCondition, NoTransitionAction, AttackClosest> m_transitionToACState;
    Transition<AttackClosest, NoCondition, NoTransitionAction, Flee> m_transitionToFleeState;
    Transition<AttackClosest, NoCondition, NoTransitionAction, RemoveTargetedUnit> m_transitionToRemoveState;
    Transition<AttackClosest, NoCondition, NoTransitionAction, PutBackSafeUnit> m_transitionToPutBackState;
  };

  class Flee : public BehaviorStateT<LearningCombatBehavior, Flee>
  {
  public:
    Flee(LearningCombatBehavior& behavior);
    void onEnter();
    void onExit() {}

    void onTimer()
    {
      m_context.calculateNextAction();
      //if current action = NoAction
      if(m_context.currentAction == NO_ACTION)
      {
        //Find new position and flee there
      }
      //else
      else
      {
        //if current action = AttackRandom
        //transition to AttackRandom
        if(m_context.currentAction == ATTACK_RANDOM){m_transitionToARState.testTransition(boost::make_tuple());}
        //if current action = AttackStrongest
        //transition to AttackStrongest
        if(m_context.currentAction == ATTACK_STRONGEST){m_transitionToASState.testTransition(boost::make_tuple());}
        //if current action = AttackDying
        //transition to AttackDying
        if(m_context.currentAction == ATTACK_DYING){m_transitionToADState.testTransition(boost::make_tuple());}
        //if current action = AttackClosest
        //transition to AttackClosest
        if(m_context.currentAction == ATTACK_CLOSEST){m_transitionToACState.testTransition(boost::make_tuple());}
        //if current action = Flee
        //transition to Flee
        if(m_context.currentAction == FLEE){m_transitionToFleeState.testTransition(boost::make_tuple());}
        //if current action = RemoveTargetedUnit
        //transition to RemoveTargetedUnit
        if(m_context.currentAction == REMOVE_TARGETED_UNIT){m_transitionToRemoveState.testTransition(boost::make_tuple());}
        //if current action = PutBackSafeUnit
        //transition to PutBackSafeUnit   
        if(m_context.currentAction == PUT_BACK_SAFE_UNIT){m_transitionToPutBackState.testTransition(boost::make_tuple());}
      }
    }

    typedef SCHEDULER_EVENT_HANDLER(Flee, onTimer) TimerEvent;

    typedef boost::tuple<
      TimerEvent&
    > Events;
    const Events m_events;

    typedef boost::tuple<
      Transition<Flee, NoCondition, NoTransitionAction, AttackRandom>& ,
      Transition<Flee, NoCondition, NoTransitionAction, AttackStrongest>& ,
      Transition<Flee, NoCondition, NoTransitionAction, AttackDying>& ,
      Transition<Flee, NoCondition, NoTransitionAction, AttackClosest>& ,
      Transition<Flee, NoCondition, NoTransitionAction, Flee>& ,
      Transition<Flee, NoCondition, NoTransitionAction, RemoveTargetedUnit>& ,
      Transition<Flee, NoCondition, NoTransitionAction, PutBackSafeUnit>&
    > Transitions;
    const Transitions m_transitions;

  protected:
    // Variables

    // Actions
    //MoveCommand m_moveCommand;

    // Transitions
    TimerEvent m_timerEvent;
    Transition<Flee, NoCondition, NoTransitionAction, AttackRandom> m_transitionToARState;
    Transition<Flee, NoCondition, NoTransitionAction, AttackStrongest> m_transitionToASState;
    Transition<Flee, NoCondition, NoTransitionAction, AttackDying> m_transitionToADState;
    Transition<Flee, NoCondition, NoTransitionAction, AttackClosest> m_transitionToACState;
    Transition<Flee, NoCondition, NoTransitionAction, Flee> m_transitionToFleeState;
    Transition<Flee, NoCondition, NoTransitionAction, RemoveTargetedUnit> m_transitionToRemoveState;
    Transition<Flee, NoCondition, NoTransitionAction, PutBackSafeUnit> m_transitionToPutBackState;
  };

  class RemoveTargetedUnit : public BehaviorStateT<LearningCombatBehavior, RemoveTargetedUnit>
  {
  public:
    RemoveTargetedUnit(LearningCombatBehavior& behavior);
    void onEnter();
    void onExit() {}

    void onTimer()
    {
       m_context.calculateNextAction();
      //if current action = NoAction
      if(m_context.currentAction == NO_ACTION)
      {
        //remove the new targeted unit
      }
      //else
      else
      {
        //if current action = AttackRandom
        //transition to AttackRandom
        if(m_context.currentAction == ATTACK_RANDOM){m_transitionToARState.testTransition(boost::make_tuple());}
        //if current action = AttackStrongest
        //transition to AttackStrongest
        if(m_context.currentAction == ATTACK_STRONGEST){m_transitionToASState.testTransition(boost::make_tuple());}
        //if current action = AttackDying
        //transition to AttackDying
        if(m_context.currentAction == ATTACK_DYING){m_transitionToADState.testTransition(boost::make_tuple());}
        //if current action = AttackClosest
        //transition to AttackClosest
        if(m_context.currentAction == ATTACK_CLOSEST){m_transitionToACState.testTransition(boost::make_tuple());}
        //if current action = Flee
        //transition to Flee
        if(m_context.currentAction == FLEE){m_transitionToFleeState.testTransition(boost::make_tuple());}
        //if current action = RemoveTargetedUnit
        //transition to RemoveTargetedUnit
        if(m_context.currentAction == REMOVE_TARGETED_UNIT){m_transitionToRemoveState.testTransition(boost::make_tuple());}
        //if current action = PutBackSafeUnit
        //transition to PutBackSafeUnit   
        if(m_context.currentAction == PUT_BACK_SAFE_UNIT){m_transitionToPutBackState.testTransition(boost::make_tuple());}
      }
    }

    typedef SCHEDULER_EVENT_HANDLER(RemoveTargetedUnit, onTimer) TimerEvent;

    typedef boost::tuple<
      TimerEvent&
    > Events;
    const Events m_events;

    typedef boost::tuple<
      Transition<RemoveTargetedUnit, NoCondition, NoTransitionAction, AttackRandom>& ,
      Transition<RemoveTargetedUnit, NoCondition, NoTransitionAction, AttackStrongest>& ,
      Transition<RemoveTargetedUnit, NoCondition, NoTransitionAction, AttackDying>& ,
      Transition<RemoveTargetedUnit, NoCondition, NoTransitionAction, AttackClosest>& ,
      Transition<RemoveTargetedUnit, NoCondition, NoTransitionAction, Flee>& ,
      Transition<RemoveTargetedUnit, NoCondition, NoTransitionAction, RemoveTargetedUnit>& ,
      Transition<RemoveTargetedUnit, NoCondition, NoTransitionAction, PutBackSafeUnit>&
    > Transitions;
    const Transitions m_transitions;

  protected:
    // Variables

    // Actions

    // Transitions
    TimerEvent m_timerEvent;
    Transition<RemoveTargetedUnit, NoCondition, NoTransitionAction, AttackRandom> m_transitionToARState;
    Transition<RemoveTargetedUnit, NoCondition, NoTransitionAction, AttackStrongest> m_transitionToASState;
    Transition<RemoveTargetedUnit, NoCondition, NoTransitionAction, AttackDying> m_transitionToADState;
    Transition<RemoveTargetedUnit, NoCondition, NoTransitionAction, AttackClosest> m_transitionToACState;
    Transition<RemoveTargetedUnit, NoCondition, NoTransitionAction, Flee> m_transitionToFleeState;
    Transition<RemoveTargetedUnit, NoCondition, NoTransitionAction, RemoveTargetedUnit> m_transitionToRemoveState;
    Transition<RemoveTargetedUnit, NoCondition, NoTransitionAction, PutBackSafeUnit> m_transitionToPutBackState;
  };

  class PutBackSafeUnit : public BehaviorStateT<LearningCombatBehavior, PutBackSafeUnit>
  {
  public:
    PutBackSafeUnit(LearningCombatBehavior& behavior);
    void onEnter();
    void onExit() {}

    void onTimer()
    {
      m_context.calculateNextAction();
      //if current action = NoAction
      if(m_context.currentAction == NO_ACTION)
      {
        //Check if there is new safe unit
      }
      //else
      else
      {
        //if current action = AttackRandom
        //transition to AttackRandom
        if(m_context.currentAction == ATTACK_RANDOM){m_transitionToARState.testTransition(boost::make_tuple());}
        //if current action = AttackStrongest
        //transition to AttackStrongest
        if(m_context.currentAction == ATTACK_STRONGEST){m_transitionToASState.testTransition(boost::make_tuple());}
        //if current action = AttackDying
        //transition to AttackDying
        if(m_context.currentAction == ATTACK_DYING){m_transitionToADState.testTransition(boost::make_tuple());}
        //if current action = AttackClosest
        //transition to AttackClosest
        if(m_context.currentAction == ATTACK_CLOSEST){m_transitionToACState.testTransition(boost::make_tuple());}
        //if current action = Flee
        //transition to Flee
        if(m_context.currentAction == FLEE){m_transitionToFleeState.testTransition(boost::make_tuple());}
        //if current action = RemoveTargetedUnit
        //transition to RemoveTargetedUnit
        if(m_context.currentAction == REMOVE_TARGETED_UNIT){m_transitionToRemoveState.testTransition(boost::make_tuple());}
        //if current action = PutBackSafeUnit
        //transition to PutBackSafeUnit   
        if(m_context.currentAction == PUT_BACK_SAFE_UNIT){m_transitionToPutBackState.testTransition(boost::make_tuple());}
      }
    }

    typedef SCHEDULER_EVENT_HANDLER(PutBackSafeUnit, onTimer) TimerEvent;

    typedef boost::tuple<
      TimerEvent&
    > Events;
    const Events m_events;

    typedef boost::tuple<
      Transition<PutBackSafeUnit, NoCondition, NoTransitionAction, AttackRandom>& ,
      Transition<PutBackSafeUnit, NoCondition, NoTransitionAction, AttackStrongest>& ,
      Transition<PutBackSafeUnit, NoCondition, NoTransitionAction, AttackDying>& ,
      Transition<PutBackSafeUnit, NoCondition, NoTransitionAction, AttackClosest>& ,
      Transition<PutBackSafeUnit, NoCondition, NoTransitionAction, Flee>& ,
      Transition<PutBackSafeUnit, NoCondition, NoTransitionAction, RemoveTargetedUnit>& ,
      Transition<PutBackSafeUnit, NoCondition, NoTransitionAction, PutBackSafeUnit>&
    > Transitions;
    const Transitions m_transitions;

  protected:
    // Variables

    // Actions

    // Transitions
    TimerEvent m_timerEvent;
    Transition<PutBackSafeUnit, NoCondition, NoTransitionAction, AttackRandom> m_transitionToARState;
    Transition<PutBackSafeUnit, NoCondition, NoTransitionAction, AttackStrongest> m_transitionToASState;
    Transition<PutBackSafeUnit, NoCondition, NoTransitionAction, AttackDying> m_transitionToADState;
    Transition<PutBackSafeUnit, NoCondition, NoTransitionAction, AttackClosest> m_transitionToACState;
    Transition<PutBackSafeUnit, NoCondition, NoTransitionAction, Flee> m_transitionToFleeState;
    Transition<PutBackSafeUnit, NoCondition, NoTransitionAction, RemoveTargetedUnit> m_transitionToRemoveState;
    Transition<PutBackSafeUnit, NoCondition, NoTransitionAction, PutBackSafeUnit> m_transitionToPutBackState;
  };

public:
  LearningCombatBehavior(const Process* parent, unsigned int initialPriority, const PersistentUnitGroup* units)
    : GroupBehaviorT(parent, initialPriority, units),
    GroupBehavior(parent, initialPriority, units),
    Behavior(parent, initialPriority),
    FSM(parent),
    Process(parent),   
    m_noActionState(*this),
    m_attackRandomState(*this),
    m_attackStrongestState(*this),
    m_attackDyingState(*this),
    m_attackClosestState(*this),
    m_fleeState(*this),
    m_removeTargetedUnitState(*this),
    m_putBackSafeUnitState(*this),
    m_states(m_noActionState,m_attackRandomState,m_attackStrongestState,m_attackDyingState,m_attackClosestState,m_fleeState,m_removeTargetedUnitState,m_putBackSafeUnitState)
  {
  }

  typedef boost::tuple<
    NoAction&,
    AttackRandom&,
    AttackStrongest&,
    AttackDying&,
    AttackClosest&,
    Flee&,
    RemoveTargetedUnit&,
    PutBackSafeUnit&
  > States;
  const States m_states;

  enum Action
  {
    NO_ACTION,
    ATTACK_RANDOM,
    ATTACK_STRONGEST,
    ATTACK_DYING,
    ATTACK_CLOSEST,
    FLEE,
    REMOVE_TARGETED_UNIT,
    PUT_BACK_SAFE_UNIT,
    ActionFirst = NO_ACTION,
    ActionLast = PUT_BACK_SAFE_UNIT
  };

  class StateOfTheWorld
  {
  public :
    StateOfTheWorld(Action action,BWAPI::UnitType groupType,double nbUnitInGroup,double healthOfGroup,
      double positionXOfAllied,double positionYOfAllied,double healthOfAllied,double effectiveDammageOfAllied,
      double positionXOfEnemy,double positionYOfEnemy,double healthOfEnemy,double effectiveDammageOfEnemy)
    {
      m_action = action;
      m_groupType = groupType;
      m_nbUnitInGroup = nbUnitInGroup;
      m_healthOfGroup = healthOfGroup;
      m_positionXOfAllied = positionXOfAllied;
      m_positionYOfAllied = positionYOfAllied;
      m_healthOfAllied = healthOfAllied;
      m_effectiveDammageOfAllied = effectiveDammageOfAllied;
      m_positionXOfEnemy = positionXOfEnemy;
      m_positionYOfEnemy = positionYOfEnemy;
      m_healthOfEnemy = healthOfEnemy;
      m_effectiveDammageOfEnemy = effectiveDammageOfEnemy;   

      // initialize random seed
      srand ( static_cast<unsigned int>(time(NULL)) );
    }

    StateOfTheWorld()
    {
      m_action = NO_ACTION;
      m_groupType = BWAPI::UnitTypes::None;
      m_nbUnitInGroup = 0;
      m_healthOfGroup = 0;
      m_positionXOfAllied = 0;
      m_positionYOfAllied = 0;
      m_healthOfAllied = 0;
      m_effectiveDammageOfAllied = 0;
      m_positionXOfEnemy = 0;
      m_positionYOfEnemy = 0;
      m_healthOfEnemy = 0;
      m_effectiveDammageOfEnemy = 0;    
    }

    Action m_action;
    BWAPI::UnitType m_groupType;
    double m_nbUnitInGroup;
    double m_healthOfGroup;
    double m_positionXOfAllied;
    double m_positionYOfAllied;
    double m_healthOfAllied;
    double m_effectiveDammageOfAllied;
    double m_positionXOfEnemy;
    double m_positionYOfEnemy;
    double m_healthOfEnemy;
    double m_effectiveDammageOfEnemy; 
  };

protected:
  virtual void _output(std::ostream& out) const
  {
    out << "Combat Learning";
  }

  virtual void onStart()
  {
    GroupBehaviorT::onStart();
    m_noActionState.enter();
  }

  void calculateNextAction()
  {
    //calculate new Current State
    currentState = getCurrentStateOfTheWorld();

    //double valueOfAction[ActionLast+1];
    //for each action
    /*for(int i = ActionFirst; i <= ActionLast ; ++i)
    {
      //get value of that action in the current state
      valueOfAction[i] = getActionValue(currentState,currentAction);
    }*/

    //Choose random number between 0 and 1
    double random = (rand() % 1000)/1000;
    // if random < e
    if(random < EPSILON)
    {
      //choose on action randomly
      currentAction = getRandomAction();
    }
    //else
    else
    {
      //choose max value action
      currentAction = getBestAction(currentState);
    }

    //Calculate Reward of the pass action
    double reward = getReward();
    //if current state is terminal
    if(currentState.m_healthOfEnemy == 0 || currentState.m_healthOfGroup == 0)
    {
      //Expected value of passed state and passed action = reward
      //Save the ANN
      setValueOfActionTerminal(pastState,pastAction,reward);
    }
    //else
    else
    {
      //Expected value of passed state and passed action = reward + discountFactor * (value of current state and current action)
      setValueOfAction(pastState,pastAction,reward + (LAMBDA * getActionValue(currentState,currentAction)));
      //passed state = current state
      pastState = currentState;
      //passed action = current action
      pastAction = currentAction;
    }
  }

  virtual void onUnitAddedToGroup(void*, BWAPI::Unit* unit)
  {
    mapOfTarget.insert(std::pair<BWAPI::Unit*,BWAPI::Unit*>::pair(unit,NULL));
  }

  virtual void onUnitRemovedFromGroup(void*, BWAPI::Unit* unit)
  { 
    mapOfTarget.erase(unit);
    fleeingUnit.erase(unit);
  }

  Action getRandomAction()
  {
    int act = rand() % ActionLast + 1;   
    return (Action) act;
  }

  Action getBestAction(StateOfTheWorld currentState)
  {
    unused(currentState);

    return NO_ACTION;
  }

  double getActionValue(StateOfTheWorld currentState,Action action)
  {
    unused(currentState);
    unused(action);

    //Get the Q aproximation of the policy
    return 0;
  }

  void setValueOfAction(StateOfTheWorld currentState,Action action,double value)
  {
    unused(currentState);
    unused(action);
    unused(value);

    //set Q at expected value
  }

  void setValueOfActionTerminal(StateOfTheWorld currentState,Action action,double value)
  {
    unused(currentState);
    unused(action);
    unused(value);

    //set Q at expected value
    //Save the ANN
  }

  StateOfTheWorld getCurrentStateOfTheWorld()
  {
    return StateOfTheWorld();
  }

  double getReward()
  {
    return 0;
  }

  // Variables
  StateOfTheWorld currentState;
  Action currentAction;
  StateOfTheWorld pastState;
  Action pastAction;

  std::map<BWAPI::Unit*,BWAPI::Unit*> mapOfTarget;
  std::set<BWAPI::Unit*> fleeingUnit;

  //Constante important

  // States
  NoAction m_noActionState;
  AttackRandom m_attackRandomState;
  AttackStrongest m_attackStrongestState;
  AttackDying m_attackDyingState;
  AttackClosest m_attackClosestState;
  Flee m_fleeState;
  RemoveTargetedUnit m_removeTargetedUnitState;
  PutBackSafeUnit m_putBackSafeUnitState;
};

