Watten_Train+Match.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "dmhGW2i1wf36"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import gc\n",
    "import glob\n",
    "import numpy as np\n",
    "import random\n",
    "import shutil\n",
    "import warnings\n",
    "import scipy.signal as sg\n",
    "from collections import deque\n",
    "from matplotlib import pyplot as plt\n",
    "#from parallel_process import mp_map\n",
    "from yogi_opt import Yogi\n",
    "\n",
    "import tensorflow as tf\n",
    "from tensorflow.python.keras.models import Model\n",
    "from tensorflow.python.keras.layers import Input, Dense, Add, BatchNormalization\n",
    "from tensorflow.python.keras.models import load_model\n",
    "from tensorflow.contrib.keras import backend as K\n",
    "tf.logging.set_verbosity(tf.logging.ERROR)\n",
    "\n",
    "os.environ[\"CUDA_DEVICE_ORDER\"] = \"PCI_BUS_ID\"\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "seed = 42\n",
    "root = '.'\n",
    "single_fit = False\n",
    "\n",
    "\n",
    "def clean_checkpoints():\n",
    "    try:\n",
    "        shutil.rmtree('%s/checkpoints'%root)\n",
    "    except Exception as ex:\n",
    "        pass\n",
    "    os.mkdir('%s/checkpoints'%root)\n",
    "\n",
    "    try:\n",
    "        shutil.rmtree('%s/tb_logs'%root)\n",
    "    except Exception as ex:\n",
    "        pass\n",
    "    os.mkdir('%s/tb_logs'%root)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import traceback\n",
    "from tqdm import tqdm\n",
    "from concurrent.futures import ProcessPoolExecutor, as_completed\n",
    "\n",
    "\n",
    "def mp_map(function, array, n_jobs=4, use_kwargs=False, front_num=1):\n",
    "    \"\"\"\n",
    "        A parallel version of the map function with a progress bar. \n",
    "\n",
    "        Args:\n",
    "            array (array-like): An array to iterate over.\n",
    "            function (function): A python function to apply to the elements of array\n",
    "            n_jobs (int, default=16): The number of cores to use\n",
    "            use_kwargs (boolean, default=False): Whether to consider the elements of array as dictionaries of \n",
    "                keyword arguments to function \n",
    "            front_num (int, default=3): The number of iterations to run serially before kicking off the parallel job. \n",
    "                Useful for catching bugs\n",
    "        Returns:\n",
    "            [function(array[0]), function(array[1]), ...]\n",
    "    \"\"\"\n",
    "    # We run the first few iterations serially to catch bugs\n",
    "    front = []\n",
    "    out = []\n",
    "    if front_num > 0:\n",
    "        front = [function(**a) if use_kwargs else function(a) for a in array[:front_num]]\n",
    "    # If we set n_jobs to 1, just run a list comprehension. This is useful for benchmarking and debugging.\n",
    "    if n_jobs==1:\n",
    "        return front + [function(**a) if use_kwargs else function(a) for a in tqdm(array[front_num:])]\n",
    "    # Assemble the workers\n",
    "    with ProcessPoolExecutor(max_workers=n_jobs) as pool:\n",
    "        # Pass the elements of array into function\n",
    "        if use_kwargs:\n",
    "            futures = [pool.submit(function, **a) for a in array[front_num:]]\n",
    "        else:\n",
    "            futures = [pool.submit(function, a) for a in array[front_num:]]\n",
    "        kwargs = {\n",
    "            'total': len(futures),\n",
    "            'unit': 'it',\n",
    "            'unit_scale': True,\n",
    "            'leave': True\n",
    "        }\n",
    "        # Print out the progress as tasks complete\n",
    "        for future in tqdm(as_completed(futures), **kwargs):\n",
    "            try:\n",
    "                out.append(future.result())\n",
    "            except Exception as e:\n",
    "                print(e)\n",
    "                traceback.print_exc()\n",
    "                out.append(e)    \n",
    "    return front + out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "V9DmT0X4wf4E"
   },
   "outputs": [],
   "source": [
    "def translate_card(card_id):\n",
    "    # lab, oachl, herz, schell\n",
    "    if int(card_id) == 0:\n",
    "        return 'None'\n",
    "    colors = ['lab', 'oachl', 'herz', 'schell']\n",
    "    hits = ['junge', 'dame', 'kinig', 'ass']\n",
    "    color = colors[int(card_id+2)%4]\n",
    "    hit = int(card_id+2)//4 + 6\n",
    "    if hit < 11:\n",
    "        return color + ' ' + str(hit)\n",
    "    else:\n",
    "        return color + ' ' + hits[hit-11]\n",
    "\n",
    "\n",
    "def encode_card(card_id):\n",
    "    none = np.zeros((1,))\n",
    "    colors = np.zeros((4,))\n",
    "    hits = np.zeros((9,))\n",
    "    if int(card_id) == 0:\n",
    "        none[0] = 1\n",
    "        return np.hstack((none, colors, hits))\n",
    "    else:\n",
    "        color = int(card_id+2)%4\n",
    "        colors[color] = 1\n",
    "        hit = int(card_id+2)//4\n",
    "        hits[hit] = 1\n",
    "        return np.hstack((none, colors, hits))\n",
    "\n",
    "    \n",
    "def decode_card(vec):\n",
    "    if vec[0] == 1:\n",
    "        return 0\n",
    "    forbe = np.argmax(vec[1:5])\n",
    "    schlog = np.argmax(vec[5:])\n",
    "    card_id = (schlog*4 + forbe -2)\n",
    "    return card_id\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "V9DmT0X4wf4E"
   },
   "outputs": [],
   "source": [
    "class WattenEnv():\n",
    "    \n",
    "    def __init__(self, n_players=2, seed=42, debug=False):\n",
    "        self.seed = seed\n",
    "        self.debug = debug\n",
    "        self.n_players = n_players\n",
    "        self.played_cards = list(np.zeros(self.n_players*5, dtype=np.int))\n",
    "        # print(self.action_space.shape)\n",
    "        # print(self.observation_space.shape)\n",
    "        \n",
    "    def __str__(self):\n",
    "        ret_val = 'Cards: ' + str([[translate_card(j) for j in hand] for hand in self.player_cards]) + '\\nStates: ' + str(self.states) + '\\nActionSpace: ' + str(self.actions)\n",
    "        return ret_val\n",
    "\n",
    "    def close(self):\n",
    "        pass\n",
    "\n",
    "    def reset(self, player):\n",
    "        \"\"\"\n",
    "        Reset environment and setup for new episode.\n",
    "        Returns: initial state of reset environment.\n",
    "        \"\"\"\n",
    "        self.turn = -self.n_players          # prepare for initial setup 'unsogn'\n",
    "        self.deck = [i for i in range(1, 33+1)]\n",
    "        random.shuffle(self.deck)\n",
    "        self.hands = [self.deck[5*p:5*p+5] for p in range(self.n_players)]       # giving cards\n",
    "        self.player_hands = self.hands\n",
    "        #print(self.player_hands)\n",
    "        self.played_cards = list()\n",
    "        self.winners = list()\n",
    "        self.price = 2        #value of win / loss\n",
    "        self.schlog = None\n",
    "        self.trumpf = None\n",
    "        self.rechter = 0\n",
    "        self.offered = False     # geboten player ID\n",
    "        self.stings = [0,0]      # gemachte stiche pro team\n",
    "        self.done = False        # counter until everybody got reward\n",
    "        self.last_winner = 0\n",
    "        return self.get_state(player)\n",
    "\n",
    "    def choose_valid_card(self, player, orig_logits):\n",
    "        if self.turn < 0 or (self.turn % self.n_players == 0):\n",
    "            return np.argmax(orig_logits[:len(self.player_hands[player])])\n",
    "        else:\n",
    "            first_card = self.played_cards[-(self.turn%self.n_players)]\n",
    "            first_trumpf = (first_card+2)%4\n",
    "            if first_trumpf == self.trumpf and player in [0, self.n_players-1]:\n",
    "                logits = orig_logits.copy()\n",
    "                for n, l in enumerate(logits[:len(self.player_hands[player])]):\n",
    "                    _, _, trumpf, schlog, _, guater = self.rank_card(self.player_hands[player][n])\n",
    "                    logits[n] -= 10000 * (not trumpf or not schlog or not guater)\n",
    "                return np.argmax(logits[:len(self.player_hands[player])])\n",
    "            else:\n",
    "                return np.argmax(orig_logits[:len(self.player_hands[player])])\n",
    "    \n",
    "    def rank_card(self, card):\n",
    "        farbe = (card+2)%4\n",
    "        trumpf = farbe == self.trumpf\n",
    "        schlog = ((card+2)//4 + 6) == self.schlog\n",
    "        rechter = trumpf and schlog\n",
    "        guater = trumpf and (((card+2)//4 + 6) == self.schlog+1 or (self.schlog == 14 and schlog == 7-(farbe==3)))\n",
    "        return card, farbe, trumpf, schlog, rechter, guater\n",
    "        \n",
    "    def validate_sting(self, cards):\n",
    "        best_card = None\n",
    "        for card in cards:\n",
    "            if best_card is None:\n",
    "                best_card, best_farbe, best_trumpf, best_schlog, best_rechter, best_guater = self.rank_card(card)\n",
    "                if best_guater:\n",
    "                    break\n",
    "            else:\n",
    "                card, farbe, trumpf, schlog, rechter, guater = self.rank_card(card)\n",
    "                if guater:\n",
    "                    best_card, best_farbe, best_trumpf, best_schlog, best_rechter, best_guater = card, farbe, trumpf, schlog, rechter, guater\n",
    "                    break\n",
    "                elif rechter and not best_guater:\n",
    "                    best_card, best_farbe, best_trumpf, best_schlog, best_rechter, best_guater = card, farbe, trumpf, schlog, rechter, guater\n",
    "                elif schlog and not (best_schlog or best_rechter or best_guater):\n",
    "                    best_card, best_farbe, best_trumpf, best_schlog, best_rechter, best_guater = card, farbe, trumpf, schlog, rechter, guater\n",
    "                elif (trumpf or card == 1) and not (best_trumpf and ((card+2)//4 + 6) < ((best_card+2)//4 + 6)) and not (best_schlog or best_rechter or best_guater):\n",
    "                    best_card, best_farbe, best_trumpf, best_schlog, best_rechter, best_guater = card, farbe, trumpf, schlog, rechter, guater\n",
    "                elif (card+2)%4 == (best_card+2)%4 and ((card+2)//4 + 6 > (best_card+2)//4 + 6) and not (best_schlog or best_rechter or best_guater):\n",
    "                    best_card, best_farbe, best_trumpf, best_schlog, best_rechter, best_guater = card, farbe, trumpf, schlog, rechter, guater\n",
    "        return cards.index(best_card)\n",
    "    \n",
    "    def step(self, action, player):\n",
    "#         if self.debug:\n",
    "#             print(action)\n",
    "        team = self.turn % 2\n",
    "        go = np.argmax(action[-2:])\n",
    "        offer = np.argmax(action[-4:-2])\n",
    "        hold = self.choose_valid_card(player, action[:5])\n",
    "        chosen_action = np.zeros(9)\n",
    "        chosen_action[np.array([hold, 5 + offer, 7 + go])] = 1\n",
    "        if self.debug and (self.turn == -self.n_players or self.turn == -1  or self.turn >= 0 or go or offer):\n",
    "            message = 'Player ' + str(player)\n",
    "            if (self.turn == -self.n_players or self.turn == -1  or self.turn >= 0):\n",
    "                if self.turn < 0:\n",
    "                    message += ' sog un'\n",
    "                else:\n",
    "                    message += ' wirft'\n",
    "                message += ': ' + translate_card(self.player_hands[player][hold])\n",
    "            if offer:\n",
    "                if (self.turn == -self.n_players or self.turn == -1  or self.turn >= 0):\n",
    "                    message += ' und'\n",
    "                message += ' biatet ' + str(self.price + 1 + (self.offered>0))\n",
    "            elif go:\n",
    "                if (self.turn == -self.n_players or self.turn == -1  or self.turn >= 0):\n",
    "                    message += ' und'\n",
    "                message += ' will gian'\n",
    "            print(message)\n",
    "        \n",
    "        if self.turn < 0:\n",
    "            card_id = self.player_hands[player][hold]\n",
    "            if self.turn == -self.n_players and self.schlog is None:\n",
    "                self.schlog = (card_id+2)//4+6\n",
    "            elif self.turn == -1 and self.trumpf is None:\n",
    "                self.trumpf = (card_id+2)%4\n",
    "                self.rechter = (self.schlog-6)*4 - 2 + self.trumpf\n",
    "                if self.debug:\n",
    "                    print('Ungsog:', translate_card(self.rechter), '\\n')\n",
    "        else:\n",
    "            card_id = self.player_hands[player].pop(hold)\n",
    "            self.played_cards.append(card_id)  \n",
    "            \n",
    "        if self.offered:\n",
    "            if self.offered%2==1 and go:\n",
    "                self.stings[(team+1)%2] = 3\n",
    "                self.done = True\n",
    "                self.last_winner = (team+1)%2\n",
    "                self.offered = False\n",
    "            elif offer:\n",
    "                if self.offered:\n",
    "                    self.price += 1\n",
    "                    self.offered = True\n",
    "            else:\n",
    "                self.offered += 1\n",
    "        elif offer:\n",
    "            self.offered = True\n",
    "        elif go:\n",
    "            self.stings[(team+1)%2] = 3\n",
    "            self.done = True\n",
    "            self.last_winner = (team+1)%2\n",
    "            \n",
    "        if not self.done and self.offered >= self.n_players-1 or (self.turn+self.n_players)%self.n_players==self.n_players-1:\n",
    "            self.price += 1\n",
    "            self.offered = False\n",
    "        \n",
    "        if not self.done and self.turn > 0 and (self.turn) % self.n_players == self.n_players-1:\n",
    "            self.last_winner = self.validate_sting(self.played_cards[-self.n_players:])\n",
    "            self.winners.append(self.last_winner+1)\n",
    "            self.stings[self.last_winner%2] += 1\n",
    "            if self.debug:\n",
    "                print('Stich mit korte ', self.last_winner+1, ',   Cards: ', [translate_card(c) for c in self.played_cards[-self.n_players:]], 'Stiche: ', self.stings, '\\n')\n",
    "        \n",
    "        if self.done or 3 in self.stings:\n",
    "            self.done = True\n",
    "            rewards = [(s==3)*self.price or -(s!=3)*self.price for s in self.stings]*(self.n_players//2)\n",
    "            if self.debug:\n",
    "                print('Spiel vobei! Punkte:', rewards, 'Stiche:', self.stings, 'Geboten woren:', self.price, '\\n\\n')\n",
    "        else:\n",
    "            rewards = [0]*self.n_players\n",
    "\n",
    "        self.turn += 1\n",
    "        return self.get_state(player), [r/25 for r in rewards], chosen_action, self.done, self.last_winner\n",
    "        \n",
    "    def get_state(self, player):\n",
    "        \"\"\"\n",
    "        player_state is composed of an array of length 372:\n",
    "        for 4 players it looks as shown:\n",
    "        0-14: rechter for players allowed to see it\n",
    "        14: amount of offered points\n",
    "        15: turn number\n",
    "        15-85: player hand\n",
    "        85-365: cards already played\n",
    "        365-372: winners\n",
    "        \"\"\"\n",
    "        player_state = [\n",
    "                encode_card(self.rechter*(((self.turn+self.n_players)%self.n_players) in [0, self.n_players-1])),\n",
    "                np.array([self.offered/25]),\n",
    "                np.array([self.turn/(5*self.n_players)]),\n",
    "                np.hstack([encode_card(c) for c in self.player_hands[player]] + [encode_card(0)] * (5-len(self.player_hands[player]))),\n",
    "                np.hstack([encode_card(c) for c in self.played_cards] + [encode_card(0)] * (self.n_players*5 - len(self.played_cards))),\n",
    "                np.array([w/(self.n_players+1) for w in self.winners] + [0] * (5 - len(self.winners)))\n",
    "            ]\n",
    "        player_state = np.hstack(player_state)\n",
    "        return np.expand_dims(player_state, axis=0)\n",
    "        \n",
    "    @property\n",
    "    def observation_space(self):\n",
    "        return np.array([0]*14+[0,0]+14*[0]*5+[0]*self.n_players*5*14+[0]*5)\n",
    "    \n",
    "    @property\n",
    "    def action_space(self):\n",
    "        hold = [0]*5\n",
    "        offer = [0]*2\n",
    "        go = [0]*2\n",
    "        return np.expand_dims(np.array(hold + offer + go), axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Rv7QGdhvGPW0"
   },
   "outputs": [],
   "source": [
    "class CollectiveMemory:\n",
    "    \n",
    "    def __init__(self, memsize):\n",
    "        self.size = memsize\n",
    "        self._memories = deque(maxlen=memsize)\n",
    "        \n",
    "    def get(self,):\n",
    "        return self._memories\n",
    "    \n",
    "    def append(self, mem):\n",
    "        self._memories.append(mem)\n",
    "        \n",
    "    def resize(self, new_size):\n",
    "        self.size = new_size\n",
    "        self._memories = deque(self._memories, maxlen=self.size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class HumanPlayer:\n",
    "       \n",
    "    def __init__(self, env, name):\n",
    "        self.env = env\n",
    "        self.name = name\n",
    "        \n",
    "    def act(self, state):\n",
    "        \"\"\"\n",
    "        player_state is composed of an array of length 372:\n",
    "        for 4 players it looks as shown:\n",
    "        0-14: rechter for players allowed to see it\n",
    "        14: amount of offered points\n",
    "        15: turn number\n",
    "        16-87: player hand\n",
    "        87-367: cards already played\n",
    "        368-372: winners\n",
    "        \"\"\"\n",
    "        rechter = decode_card(state[0][:14])\n",
    "        bet = state[0][14] > 0\n",
    "        turn = state[0][15]*(5*self.env.n_players)\n",
    "        played_cards = [True for s in np.split(state[0][87:367], 20) if np.any(s[1:])]\n",
    "        player = (len(played_cards)+1)%self.env.n_players\n",
    "        hand = [decode_card(s) for s in np.split(state[0][16:86], 5)]\n",
    "        cards = ['%s: %s'%(i+1,translate_card(c)) for i,c in enumerate(hand) if c != 0]       # [translate_card(c*34) for c in hand if c != 0]\n",
    "        if self.env.turn >= 0 or self.env.turn == -1 or self.env.turn == -self.env.n_players:\n",
    "            if bet:\n",
    "                geboten = '      Gebotn isch!'\n",
    "            else:\n",
    "                geboten = ''\n",
    "            print('Deine Korten: %s                 Ungsog: %s%s'%(cards, rechter, geboten))\n",
    "\n",
    "            choice = 0\n",
    "            if bet:\n",
    "                while choice not in [1,2,3]:\n",
    "                    go = [0,0]\n",
    "                    bet = [0,0]\n",
    "                    choice = int(float(input('Hebn: 1      Gian: 2      Biatn: 3')))\n",
    "                    go[choice == 2] = 1\n",
    "                    bet[choice == 3] = 1\n",
    "            else:\n",
    "                while choice not in [1,3]:\n",
    "                    bet = [0,0]\n",
    "                    choice = int(float(input('Hebn: 1      Biatn: 3')))\n",
    "                    go =[1,0]\n",
    "                    bet[choice == 3] = 1\n",
    "            if choice in [1,3]:\n",
    "                card = None\n",
    "                while card not in range(1,len(cards)+1):\n",
    "                    try:\n",
    "                        card = int(float(input('Wos wirfschn?')))\n",
    "                    except Exception as ex:\n",
    "                        pass\n",
    "                chosen = [0]*5\n",
    "                chosen[card-1] = 1\n",
    "            else:\n",
    "                chosen = [0]*5\n",
    "               \n",
    "            chosen = chosen + bet + go\n",
    "        else:\n",
    "            chosen = [0]*5 + [0,0] + [0,0]\n",
    "                \n",
    "        return np.array([chosen])\n",
    "\n",
    "    def remember(self, state, action, reward, new_state, done):\n",
    "        pass\n",
    "\n",
    "    def replay(self):\n",
    "        return 0\n",
    "            \n",
    "    def eval(self):\n",
    "        return 0\n",
    "\n",
    "    def target_train(self):\n",
    "        pass\n",
    "\n",
    "    def save_model(self, fn):\n",
    "        pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "BUf1w52hcbT_"
   },
   "outputs": [],
   "source": [
    "class DQN:\n",
    "       \n",
    "    def __init__(self, env, name, memory, neurons, res_blocks, restore=None, alpha=1.0):\n",
    "        self.env = env\n",
    "        self.neurons = neurons\n",
    "        self.res_blocks = res_blocks\n",
    "        self.alpha = alpha\n",
    "        self.name = name\n",
    "        self.memory = memory\n",
    "        \n",
    "        '''\n",
    "        self.gamma = 0.9\n",
    "        self.epsilon = 0.2\n",
    "        self.epsilon_min = 0.01\n",
    "        self.epsilon_decay = 0.99\n",
    "        self.learning_rate = 0.0051\n",
    "        self.tau = .125\n",
    "        '''\n",
    "        \n",
    "        self.gamma = 0.75                     # discount for future rewards \n",
    "        self.epsilon = 0.25                   # probability of a random action\n",
    "        self.epsilon_min = 0.001              # min epsilon\n",
    "        self.epsilon_decay = 0.99             # rate of decay for epsilon\n",
    "        self.learning_rate = 0.000081\n",
    "        self.tau = .125\n",
    "        \n",
    "        self.model = self.create_model(restore)\n",
    "        self.target_model = self.create_model(restore)\n",
    "        self.batch_size = 512\n",
    "    \n",
    "\n",
    "    def create_model(self, restore):\n",
    "        \n",
    "        def res_block(x, n):\n",
    "            x1 = Dense(n, activation='relu')(x)\n",
    "            x1 = BatchNormalization()(x1)\n",
    "            x2 = Dense(n, activation='relu')(x1)\n",
    "            x2 = BatchNormalization()(x2)\n",
    "            \n",
    "            x = Add()([x, x2])\n",
    "            return x\n",
    "        \n",
    "        if restore is None:\n",
    "            n = self.env.observation_space.size\n",
    "            inputs = Input((n,), name='main_input')\n",
    "            x = Dense(n, activation='relu')(inputs)\n",
    "            x = BatchNormalization()(x)\n",
    "            for rb in range(self.res_blocks):\n",
    "                x = res_block(x, int(n/self.alpha**(rb)))\n",
    "                x = BatchNormalization()(x)\n",
    "                x = Dense(int(n/self.alpha**(rb+1)), activation='relu')(x)\n",
    "            predictions = Dense(self.env.action_space.size, activation='sigmoid')(x)\n",
    "            \n",
    "            model = Model(inputs=inputs, outputs=predictions)\n",
    "            model.compile(loss='mse',\n",
    "                          optimizer=Yogi(lr=self.learning_rate))\n",
    "        else:\n",
    "            model = load_model(restore,  custom_objects={'Yogi': Yogi})\n",
    "            \n",
    "        return model\n",
    "\n",
    "    def act(self, state):\n",
    "        self.epsilon *= self.epsilon_decay\n",
    "        self.epsilon = max(self.epsilon_min, self.epsilon)\n",
    "        if train:\n",
    "            if np.random.random() < self.epsilon:\n",
    "                return np.random.normal(0, 0.1, self.env.action_space.shape)\n",
    "        return self.model.predict(state)\n",
    "\n",
    "    def remember(self, state, action, reward, new_state, done):\n",
    "        self.memory.append([state, action, reward, new_state, done])\n",
    "\n",
    "    def replay(self):\n",
    "        if single_fit and len(self.memory.get()) < 1:   #self.batch_size: \n",
    "            return None\n",
    "        elif len(self.memory.get()) < self.batch_size: \n",
    "            return None\n",
    "\n",
    "        samples = random.sample(self.memory.get(), min(self.batch_size, len(self.memory.get())))\n",
    "        states = []\n",
    "        actions = []\n",
    "        rewards = []\n",
    "        new_states = []\n",
    "        dones = []\n",
    "        targets = []  # only for single train\n",
    "        for sample in samples:\n",
    "            \n",
    "            state, action, reward, new_state, done = sample\n",
    "            states.append(state)\n",
    "            actions.append(action)\n",
    "            rewards.append(reward/25)\n",
    "            new_states.append(new_state)\n",
    "            dones.append(done)\n",
    "            \n",
    "            \n",
    "            if single_fit:\n",
    "                target = self.target_model.predict(state)\n",
    "                if done:\n",
    "                    target[0][action==1] = reward\n",
    "                else:\n",
    "                    Q_future = max(self.target_model.predict(new_state)[0])\n",
    "                    target[0][action == 1] = reward + Q_future * self.gamma\n",
    "                targets.append(target)\n",
    "                self.model.fit(state, target, epochs=1, verbose=0)\n",
    "        if not single_fit:\n",
    "            targets = self.target_model.predict_on_batch(np.concatenate(states))\n",
    "            Q_futures = [max(fut) for fut in self.target_model.predict_on_batch(np.concatenate(new_states))]\n",
    "            for i, target in enumerate(targets):\n",
    "                for j, action in enumerate(actions[i]):\n",
    "                    if dones[i]:\n",
    "                        if action==1:\n",
    "                            targets[i][j] = rewards[i]\n",
    "                    else:\n",
    "                        targets[i][j] = rewards[i] + Q_futures[i] * self.gamma\n",
    "        if single_fit:\n",
    "            loss = (self.model.train_on_batch(np.concatenate(states), np.concatenate(targets)))\n",
    "        else:\n",
    "            loss = (self.model.train_on_batch(np.concatenate(states), targets))\n",
    "        \n",
    "        return loss\n",
    "            \n",
    "    \n",
    "    def eval(self):\n",
    "        if len(self.memory.get()) < self.batch_size: \n",
    "            return None\n",
    "        \n",
    "        mse = []\n",
    "        for s in range(len(self.memory.get())//self.batch_size):\n",
    "            samples = random.sample(self.memory.get(), self.batch_size)\n",
    "            states = []\n",
    "            actions = []\n",
    "            rewards = []\n",
    "            new_states = []\n",
    "            dones = []\n",
    "            for sample in samples:\n",
    "                state, action, reward, new_state, done = sample\n",
    "                states.append(state)\n",
    "                actions.append(action)\n",
    "                rewards.append(reward)\n",
    "                new_states.append(new_state)\n",
    "                dones.append(done)\n",
    "            targets = self.target_model.predict_on_batch(np.concatenate(states))\n",
    "            Q_futures = [max(fut) for fut in self.target_model.predict_on_batch(np.concatenate(new_states))]\n",
    "            for i, target in enumerate(targets):\n",
    "                for j, action in enumerate(actions[i]):\n",
    "                    if dones[i]:\n",
    "                        if action:\n",
    "                            targets[i][j] = rewards[i]\n",
    "                    else:\n",
    "                        targets[i][j] = rewards[i] + Q_futures[i] * self.gamma\n",
    "            mse.append(self.model.test_on_batch(np.concatenate(states), targets))\n",
    "        return sum(mse)/len(mse)\n",
    "\n",
    "    def target_train(self):\n",
    "        weights = self.model.get_weights()\n",
    "        target_weights = self.target_model.get_weights()\n",
    "        for i in range(len(target_weights)):\n",
    "            target_weights[i] = weights[i] * self.tau + target_weights[i] * (1 - self.tau)\n",
    "        self.target_model.set_weights(target_weights)\n",
    "\n",
    "    def save_model(self, fn):\n",
    "        self.model.save(fn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "jqLj4BGWUDSt"
   },
   "outputs": [],
   "source": [
    "def match(trial_id, env, agents, trials, metric='points', show_progress=False):\n",
    "\n",
    "    if metric not in ['points', 'loss']:\n",
    "        raise Exception('unsupported metric')\n",
    "\n",
    "    trial_len = 5\n",
    "    n_players=len(agents)\n",
    "    random_order = [i for i in range(n_players)]\n",
    "    all_points = []\n",
    "    all_losses = []\n",
    "    offset = trial_id\n",
    "    order = [random_order[((offset + i)%n_players)] for i in range(n_players)]\n",
    "    played_trials = 0\n",
    "    while trials > played_trials:           #for trial in range(trials):\n",
    "        np.random.shuffle(order)\n",
    "        cur_state = env.reset(order[0])\n",
    "        winner = 0\n",
    "        round_done = False\n",
    "        while not round_done:\n",
    "            states = [cur_state]\n",
    "            actions = []\n",
    "            rewards = []\n",
    "            losses = [1 for i in range(n_players)]\n",
    "            # agents = [agents[(step + i)%n_players] for i in range(n_players)]\n",
    "            # print('')\n",
    "            order = [order[((winner + i)%n_players)] for i in range(n_players)]\n",
    "            # env.player_hands = [env.player_hands[order[i]] for i in range(n_players)]\n",
    "            completed_rounds = 0\n",
    "            for i, player in enumerate(order):\n",
    "                agent = agents[player]\n",
    "                cur_state = env.get_state(player)\n",
    "                action = agent[0].act(cur_state)[0]\n",
    "                new_state, reward, action, round_done, winner = env.step(action, player)\n",
    "                actions.append(action)\n",
    "                states.append(new_state)\n",
    "                rewards.append(reward)\n",
    "                completed_rounds += 1\n",
    "                if round_done:\n",
    "                    for o, j in enumerate(order):\n",
    "                        agents[j][1] += int(reward[o]*25)\n",
    "                    break\n",
    "            for o, i in enumerate(order):\n",
    "                agent = agents[i]\n",
    "                if i < completed_rounds:       # prevents trying to add memories for agents that have none\n",
    "                    agent[0].remember(states[i], actions[i], rewards[-1][i], new_state, round_done)\n",
    "                loss = agent[0].replay()       # internally iterates default (prediction) model\n",
    "                agent[0].target_train()        # iterates target model\n",
    "                losses[i] = loss\n",
    "                #cur_state = new_state\n",
    "            if round_done:\n",
    "                points = [agent[1] for agent in agents]\n",
    "                if None in losses:\n",
    "                    for agent in agents:\n",
    "                        agent[1] = 0\n",
    "                else:\n",
    "                    all_points.append(points)\n",
    "                    all_losses.append(losses)\n",
    "                    played_trials += 1\n",
    "                    \n",
    "                offset += 1\n",
    "                if show_progress:\n",
    "                    if played_trials == 0:\n",
    "                        print('\\rRunning warm!', str(10000*len(agents[0][0].memory.get())//agents[0][0].batch_size/100)+'%', points, end='')\n",
    "                    else:\n",
    "                        points_string = 'Points: %s'%points\n",
    "                        losses_string = '   Losses: %s'%losses\n",
    "                        percent_string = str(10000*(played_trials)//trials/100)+'%'\n",
    "                        print('\\r' + percent_string + ' '*max(0,7-len(percent_string)) + points_string + \n",
    "                              ' '*max(0, (35-len(points_string))) + losses_string, end='')\n",
    "\n",
    "    # print('\\r', end='')\n",
    "    losses = []\n",
    "    for agent in agents:\n",
    "        loss = agent[0].eval()\n",
    "        agent.append(loss)\n",
    "        losses.append(loss)\n",
    "    # print(losses)\n",
    "    if metric == 'points':\n",
    "        best_agents = sorted(agents, key=lambda x: x[1], reverse=True)\n",
    "    elif metric == 'loss':\n",
    "        best_agents = sorted(agents, key=lambda x: x[-1])     # loss=x[-1])[:2]   points=x[1])[-2:]\n",
    "\n",
    "    best_agent_names = []\n",
    "    for agent in best_agents:\n",
    "        agent_id = agent[2]\n",
    "        agent_res_blocks = agent[3]\n",
    "        agent_neurons = agent[4]\n",
    "        agent_alpha = agent[5]\n",
    "        agent_loss = agent[-1]\n",
    "        agent_points = agent[1]\n",
    "        # print('agent %s: \\tres_blocks: %s \\tneurons: %s \\talpha: %s \\tpoints: %s  \\tloss: %s'%(agent_id, agent_res_blocks, agent_neurons, agent_alpha, agent_alpha, agent_points, agent_loss))\n",
    "        agent_name = 'agent_%s_%s_%s_%s_%s_%s'%( trial_id, agent_id, agent_res_blocks, agent_neurons, int(agent_alpha*1000), agent_points)\n",
    "        agent[0].name = agent_name\n",
    "        best_agent_names.append('./checkpoints/%s.h5'%(agent_name))\n",
    "    for i in range(n_players//2):\n",
    "        best_agents[i][0].save_model(best_agent_names[i])\n",
    "    # for agent in agents:\n",
    "    #    agent[0].clear()\n",
    "\n",
    "    all_losses = [[i if i is not None else 1 for i in loss] for loss in all_losses]\n",
    "\n",
    "    plt.figure(figsize=(26, 10), facecolor='white')\n",
    "    ax = plt.subplot(1, 2, 1)\n",
    "    ax.set_title('Points')\n",
    "    for i in random_order:\n",
    "        plt.plot([p[i] for p in all_points], label='%s'%(agents[i][0].name))\n",
    "    ax.legend(loc=\"upper left\")\n",
    "    ax = plt.subplot(1, 2, 2)\n",
    "    ax.set_title('Lowpassed Losses')\n",
    "    b, a = sg.butter(4, 1/(trials*max(0.005, 1/(trials-1))))\n",
    "    plt.plot(sg.filtfilt(b, a, all_losses, axis=0))\n",
    "    #plt.plot(all_losses)\n",
    "    plt.tight_layout()\n",
    "    plt.show()\n",
    "\n",
    "    # print()\n",
    "    return best_agent_names[:n_players//2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def first_matches(args):\n",
    "\n",
    "    config = tf.ConfigProto()\n",
    "    config.gpu_options.allow_growth = True\n",
    "    sess = tf.Session(config=config)\n",
    "    K.set_session(sess)\n",
    "\n",
    "    collective_memory = args[1]\n",
    "    n_players = args[2]\n",
    "    trials = args[3]\n",
    "    metric = args[4]\n",
    "    match_id = args[5]\n",
    "    env = WattenEnv(n_players=n_players, debug=not train)\n",
    "\n",
    "    agent_attrs = []\n",
    "    for i in range(n_players):\n",
    "        layers = random.randint(5, 15)\n",
    "        neurons = random.randint(2**3,2**7)\n",
    "        alpha = 1 # random.uniform(1, 1.5)\n",
    "        agent_attrs.append([layers, neurons, alpha])\n",
    "        \n",
    "    agents = []\n",
    "    for p in range(n_players):\n",
    "        agents += [[DQN(env=env,                                                     # DQN-agent\n",
    "                       res_blocks=agent_attrs[p][0], \n",
    "                       neurons=agent_attrs[p][1],\n",
    "                       alpha=agent_attrs[p][2],\n",
    "                       name='agent_%s'%(match_id*n_players+p), \n",
    "                       memory=collective_memory\n",
    "                      ),\n",
    "                   0,                                                               # Points\n",
    "                   match_id*n_players+p,                                            # Agent-ID\n",
    "                   agent_attrs[p][0],                                               # Residual Blocks\n",
    "                   agent_attrs[p][1],                                               # Neurons\n",
    "                   agent_attrs[p][2]                                                # Alpha\n",
    "                  ] ]\n",
    "    \n",
    "    match_best_agents = match(0, env, agents, trials, metric=metric, show_progress=(workers==1 and train))\n",
    "    K.clear_session()\n",
    "    return match_best_agents\n",
    "\n",
    "\n",
    "def next_matches(args):\n",
    "    \n",
    "    # print('\\nRound', i+1)\n",
    "    # collective_memory, n_players, trials, metric, match_id, i, old_agents\n",
    "\n",
    "    config = tf.ConfigProto()\n",
    "    config.gpu_options.allow_growth = True\n",
    "    sess = tf.Session(config=config)\n",
    "    K.set_session(sess)\n",
    "    \n",
    "    collective_memory = args[0]\n",
    "    n_players = args[1]\n",
    "    trials = args[2]\n",
    "    metric = args[3]\n",
    "    match_id = args[4]\n",
    "    match_id = args[5]\n",
    "    old_agents = args[6]\n",
    "    env = WattenEnv(n_players=n_players, debug=not train)\n",
    "\n",
    "    agent_names = old_agents[match_id*n_players: match_id*n_players+n_players]\n",
    "    agents = []\n",
    "    for a in agent_names:\n",
    "        if not a.startswith('Player'):\n",
    "            splits = a.split('/')[-1].split('_')\n",
    "            # 'checkpoints/%s_%s_%s_%s_%.4f_%s.h5'%( match_id, agent_id, agent_res_blocks, agent_neurons, agent_alpha, agent_points)\n",
    "            p_id = int(float(splits[2]))\n",
    "            layers = int(float(splits[3]))\n",
    "            neurons = int(float(splits[4]))\n",
    "            alpha = float(splits[5])/1000\n",
    "            agents.append([DQN(env, name='agent_%s'%(p_id), \n",
    "                               memory=collective_memory,\n",
    "                               restore=a, \n",
    "                               neurons=neurons, \n",
    "                               res_blocks=layers,\n",
    "                               alpha=alpha\n",
    "                              ), \n",
    "                           0, \n",
    "                           p_id, \n",
    "                           layers, \n",
    "                           neurons,\n",
    "                           alpha\n",
    "                          ]\n",
    "                         )\n",
    "        else:\n",
    "            agents.append([HumanPlayer(env, a),\n",
    "                          0,\n",
    "                          a,\n",
    "                          'inf',\n",
    "                          'inf',\n",
    "                          1.0\n",
    "                          ]\n",
    "                         )\n",
    "    match_best_agents = match(match_id, env, agents, trials, metric=metric, show_progress=(workers==1 and train))\n",
    "    K.clear_session()\n",
    "    return match_best_agents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "GDQzdGlwwf4N"
   },
   "outputs": [],
   "source": [
    "def main():\n",
    "    global workers, max_workers\n",
    "    n_players = 4\n",
    "    humans = 1\n",
    "    trial_id = 0\n",
    "    \n",
    "    if not train:\n",
    "        max_workers = 1\n",
    "    print('Running on max %s process%s.\\n'%(max_workers, 'es'*(max_workers>1)))\n",
    "    \n",
    "    first_trials = 2**5\n",
    "    trials = 2000\n",
    "    memsize = 200000\n",
    "    if train:\n",
    "        best_agents = glob.glob('%s/checkpoints/*.h5'%root)\n",
    "        # best_agents = glob.glob('%s/checkpoints/agent_3*.h5'%root)\n",
    "    else:\n",
    "        best_agents = ['Player_%s'%i for i in range(humans)]\n",
    "        npc_players =  glob.glob('./saved/*.h5')\n",
    "        while len(npc_players) < 4:\n",
    "            npc_players *= 2\n",
    "        random.shuffle(npc_players)\n",
    "        best_agents = best_agents + npc_players[:n_players-humans]\n",
    "    \n",
    "    metric = 'points'     # 'loss'    'points'\n",
    "    \n",
    "    collective_memory = CollectiveMemory(memsize)\n",
    "    args = []\n",
    "    if train:\n",
    "        clean = None\n",
    "        while clean not in ['y', 'Y', '', 'n', 'N']:\n",
    "            clean = input('Do you want to delete all checkpoints and train from scratch? Y/n')\n",
    "            if clean in ['y', 'Y', '']:\n",
    "                clean_checkpoints()\n",
    "                best_agents = []\n",
    "        print('\\n')\n",
    "\n",
    "        if clean in ['y', 'Y', '', None]:\n",
    "            for match_id in range(first_trials):\n",
    "                args.append([None, collective_memory, n_players, trials, metric, match_id])\n",
    "            workers = min(len(args), max_workers)\n",
    "            if workers > 1:\n",
    "                winners = mp_map(first_matches, args, n_jobs=workers, front_num=0)\n",
    "            else:\n",
    "                winners = [first_matches(arg) for arg in args]\n",
    "            for w in winners:\n",
    "                best_agents += w\n",
    "    \n",
    "    print('Best Agents:')\n",
    "    for agent in best_agents:\n",
    "        print(agent)\n",
    "    print('\\n')\n",
    "    \n",
    "    #try:\n",
    "    if True:\n",
    "        while True:\n",
    "            while len(best_agents) < n_players * max_workers:\n",
    "                best_agents = best_agents * 2      # n_players\n",
    "            old_agents = best_agents\n",
    "            random.shuffle(old_agents)\n",
    "            best_agents = []\n",
    "            trial_id += 1\n",
    "            # trials *= 2\n",
    "            # trials += 1000\n",
    "            # collective_memory.resize(int(collective_memory.size*1.2))\n",
    "            args = []\n",
    "            for i in range(len(old_agents)//n_players):\n",
    "                args.append([collective_memory, n_players, trials, metric, trial_id, i, old_agents])\n",
    "            workers = min(len(args), max_workers)\n",
    "            if workers > 1:\n",
    "                winners = mp_map(next_matches, args, n_jobs=workers, front_num=0)\n",
    "            else:\n",
    "                winners = [next_matches(arg) for arg in args]\n",
    "            for w in winners:\n",
    "                best_agents += w\n",
    "\n",
    "            print('Best Agents:')\n",
    "            for agent in best_agents:\n",
    "                print(agent)\n",
    "            print('\\n')\n",
    "#     except KeyboardInterrupt as kix:\n",
    "#         if train:\n",
    "#             print('\\n\\nTraining stopped.')\n",
    "        \n",
    "    print('Done.')    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 7683
    },
    "colab_type": "code",
    "id": "nxXJr8HTVWWU",
    "outputId": "0e3e8c41-efda-4b66-f092-9c746d815279"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running on max 4 processes.\n",
      "\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Do you want to delete all checkpoints and train from scratch? Y/n \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0.00/32.0 [00:00<?, ?it/s]"
     ]
    }
   ],
   "source": [
    "train = True\n",
    "if train:\n",
    "    max_workers = 4\n",
    "    main()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "WsqsvO_U_0X7"
   },
   "outputs": [],
   "source": [
    "# before playing with your saved checkpoints move at least one of them into ./saved\n",
    "train = False\n",
    "\n",
    "main()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "wcU4ILkb89tM"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "U3yl-N8k89tO"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "BulFuDJb89tQ"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "yQYsv5jz89tR"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "MjhFbZAF89tT"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "WYrDuT4K89tV"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "jsH_R61l89tX"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ZADe0omg89tY"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "Watten.ipynb",
   "provenance": [],
   "version": "0.3.2"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}