QWOP.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Play QWOP Game with RL"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## CS496 madcamp week4 qwop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import read_qwop_game as qwop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from PIL import Image\n",
    "import cv2\n",
    "import io\n",
    "import time\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from IPython.display import clear_output\n",
    "from random import randint\n",
    "import os\n",
    "\n",
    "from keras.models import model_from_json\n",
    "from keras.models import Sequential\n",
    "from keras.layers.core import Dense, Dropout, Activation, Flatten\n",
    "from keras.layers.convolutional import Conv2D, MaxPooling2D\n",
    "from keras.optimizers import SGD , Adam\n",
    "from keras.callbacks import TensorBoard\n",
    "from collections import deque\n",
    "import random\n",
    "import pickle\n",
    "from io import BytesIO\n",
    "import base64\n",
    "import json\n",
    "import queue\n",
    "import threading"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss_file_path = \"./objects/loss_df.csv\"\n",
    "# actions_file_path = \"./objects/actions_df.csv\"\n",
    "q_value_file_path = \"./objects/q_vales.csv\"\n",
    "scores_file_path = \"./objects/scores_df.csv\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# game parameters\n",
    "ACTIONS = 4 # possible actions: q, w, o, p\n",
    "GAMMA = 0.99 # decay rate of past observations original 0.99\n",
    "OBSERVATION = 100. # timesteps to observe before training\n",
    "EXPLORE = 60000  # frames over which to anneal epsilon\n",
    "FINAL_EPSILON = 0.0001 # final value of epsilon\n",
    "INITIAL_EPSILON = 0.1 # starting value of epsilon\n",
    "REPLAY_MEMORY = 50000 # number of previous transitions to remember\n",
    "BATCH = 16 # size of minibatch\n",
    "FRAME_PER_ACTION = 1\n",
    "LEARNING_RATE = 1e-04\n",
    "img_rows, img_cols = 80, 80\n",
    "img_channels = 4 # We stack 4 frames"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'actions_file_path' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-19-2ac98701026f>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[0mscores_df\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mscores_file_path\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mos\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0misfile\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mloss_file_path\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32melse\u001b[0m \u001b[0mpd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mDataFrame\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcolumns\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;34m'scores'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[1;31m# actions_df = pd.read_csv(actions_file_path) if os.path.isfile(actions_file_path) else pd.DataFrame(columns = ['actions'])\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 5\u001b[1;33m \u001b[0mq_values_df\u001b[0m \u001b[1;33m=\u001b[0m\u001b[0mpd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mactions_file_path\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mos\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0misfile\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mq_value_file_path\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32melse\u001b[0m \u001b[0mpd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mDataFrame\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcolumns\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;34m'qvalues'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m: name 'actions_file_path' is not defined"
     ]
    }
   ],
   "source": [
    "# Intialize log structures from file if exists else create new\n",
    "loss_df = pd.read_csv(loss_file_path) if os.path.isfile(loss_file_path) else pd.DataFrame(columns =['loss'])\n",
    "scores_df = pd.read_csv(scores_file_path) if os.path.isfile(loss_file_path) else pd.DataFrame(columns = ['scores'])\n",
    "# actions_df = pd.read_csv(actions_file_path) if os.path.isfile(actions_file_path) else pd.DataFrame(columns = ['actions'])\n",
    "q_values_df =pd.read_csv(actions_file_path) if os.path.isfile(q_value_file_path) else pd.DataFrame(columns = ['qvalues'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def save_obj(obj, name):\n",
    "    with open('objects/' + name + '.pkl', 'wb') as f:\n",
    "        pickle.dump(obj, f, pickle.HIGHEST_PROTOCOL)\n",
    "        \n",
    "def load_obj(name):\n",
    "    with open('objects/' + name + '.pkl', 'rb') as f:\n",
    "        return pickle.load(f)\n",
    "\n",
    "# Show images in new window\n",
    "def show_img(graphs = False):\n",
    "    while True:\n",
    "        screen = (yield)\n",
    "        window_title = \"logs\" if graphs else \"game_play\"\n",
    "        cv2.namedWindow(window_title, cv2.WINDOW_NORMAL)\n",
    "        cv2.resizeWindow(window_title, 300,300)\n",
    "        cv2.imshow(window_title, screen)\n",
    "        if (cv2.waitKey(1) & 0xFF == ord('q')):\n",
    "            cv2.destroyAllWindows()\n",
    "            break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# training variables saved as checkpoints to filesystem to resume training from the same step\n",
    "def init_cache():\n",
    "    \"\"\"initial variable caching, done only once\"\"\"\n",
    "    save_obj(INITIAL_EPSILON,\"epsilon\")\n",
    "    t = 0\n",
    "    save_obj(t,\"time\")\n",
    "    D = deque()\n",
    "    save_obj(D,\"D\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''Call only once to init file structure\n",
    "'''\n",
    "#init_cache()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Game_state:\n",
    "    def __init__(self, game):\n",
    "        self._game = game\n",
    "        self._display = show_img()\n",
    "        self._display.__next__()\n",
    "        self.high_score = 0\n",
    "        self.score_array = []\n",
    "        \n",
    "    def get_state(self, actions):\n",
    "#         actions_df.loc[len(actions_df)] = actions[1:]\n",
    "        state, score, done, high_score = self._game.get()\n",
    "#         reward = 0.1 # Reward because he is alive\n",
    "        reward = 0\n",
    "        \n",
    "        # Do action\n",
    "        if actions[0] == 1:\n",
    "            t = threading.Thread(target=self._game.press_key, args=('q'))\n",
    "            t.start()\n",
    "#             self._game.press_key('q')\n",
    "\n",
    "        elif actions[1] == 1:\n",
    "            t = threading.Thread(target=self._game.press_key, args=('w'))\n",
    "            t.start()\n",
    "\n",
    "        elif actions[2] == 1:\n",
    "            t = threading.Thread(target=self._game.press_key, args=('o'))\n",
    "            t.start()\n",
    " \n",
    "        elif actions[3] == 1:\n",
    "            t = threading.Thread(target=self._game.press_key, args=('p'))\n",
    "            t.start()\n",
    "        \n",
    "        self.score_array.append(score)\n",
    "        \n",
    "        if len(self.score_array) > 5:\n",
    "            self.score_array.pop(0)\n",
    "            if (self.score_array[0] == self.score_array[1] \n",
    "                and self.score_array[1] == self.score_array[2] \n",
    "                and self.score_array[2] == self.score_array[3] \n",
    "                and self.score_array[3] == self.score_array[4]):\n",
    "                reward -= 0.1\n",
    "        \n",
    "        self._display.send(state)\n",
    "        \n",
    "        if done:\n",
    "            scores_df.loc\n",
    "            self._game.restart()\n",
    "            reward += high_score * 10\n",
    "        \n",
    "        return state, reward, done"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def buildmodel():\n",
    "    print(\"Build the model.\")\n",
    "    model = Sequential()\n",
    "    model.add(Conv2D(32, (5, 5), strides=(3, 3), padding='same',input_shape=(img_cols,img_rows,img_channels), kernel_initializer='he_uniform'))  #300*300*4\n",
    "    model.add(MaxPooling2D(pool_size=(2,2)))\n",
    "    model.add(Activation('relu'))\n",
    "    model.add(Conv2D(64, (3, 3), strides=(1, 1), padding='same', kernel_initializer='he_uniform'))\n",
    "    model.add(MaxPooling2D(pool_size=(2,2)))\n",
    "    model.add(Activation('relu'))\n",
    "    model.add(Conv2D(64, (3, 3), strides=(1, 1), padding='same', kernel_initializer='he_uniform'))\n",
    "    model.add(MaxPooling2D(pool_size=(2,2)))\n",
    "    model.add(Activation('relu'))\n",
    "    model.add(Flatten())\n",
    "    model.add(Dense(512))\n",
    "    model.add(Activation('relu'))\n",
    "    model.add(Dense(ACTIONS))\n",
    "    model.compile(loss='mse',optimizer=Adam(lr=LEARNING_RATE))\n",
    "    \n",
    "    #create model file if not present\n",
    "    if not os.path.isfile(loss_file_path):\n",
    "        model.save_weights('model.h5')\n",
    "    print(\"Finish building the model. \")\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def trainNetwork(model, game_state, observe=False):\n",
    "    last_time = time.time()\n",
    "    # store the previous observations in replay memory\n",
    "    D = load_obj(\"D\") #load from file system\n",
    "    # get the first state by doing nothing\n",
    "    do_nothing = np.zeros(ACTIONS)\n",
    "    do_nothing[0] = 1\n",
    "    \n",
    "    x_t, r_0, terminal = game_state.get_state(do_nothing) # get next step after performing the action\n",
    "    \n",
    "    s_t = np.stack((x_t, x_t, x_t, x_t), axis=2) # stack 4 images to create placeholder input\n",
    "    \n",
    "\n",
    "    s_t = s_t.reshape(1, s_t.shape[0], s_t.shape[1], s_t.shape[2])  #1*20*40*4\n",
    "    \n",
    "    initial_state = s_t \n",
    "\n",
    "    if observe :\n",
    "        OBSERVE = 999999999    #We keep observe, never train\n",
    "        epsilon = FINAL_EPSILON\n",
    "        print (\"Now we load weight\")\n",
    "        model.load_weights(\"model.h5\")\n",
    "        adam = Adam(lr=LEARNING_RATE)\n",
    "        model.compile(loss='mse',optimizer=adam)\n",
    "        print (\"Weight load successfully\")    \n",
    "    else:                       #We go to training mode\n",
    "        OBSERVE = OBSERVATION\n",
    "        epsilon = load_obj(\"epsilon\") \n",
    "        model.load_weights(\"model.h5\")\n",
    "        adam = Adam(lr=LEARNING_RATE)\n",
    "        model.compile(loss='mse',optimizer=adam)\n",
    "\n",
    "    t = load_obj(\"time\") # resume from the previous time step stored in file system\n",
    "    while (True): #endless running\n",
    "        \n",
    "        loss = 0\n",
    "        Q_sa = 0\n",
    "        action_index = 0\n",
    "        r_t = 0 #reward at 4\n",
    "        a_t = np.zeros([ACTIONS]) # action at t\n",
    "        \n",
    "        #choose an action epsilon greedy\n",
    "        if t % FRAME_PER_ACTION == 0: #parameter to skip frames for actions\n",
    "            if  random.random() <= epsilon: #randomly explore an action\n",
    "                print(\"----------Random Action----------\")\n",
    "                action_index = random.randrange(ACTIONS)\n",
    "                a_t[action_index] = 1\n",
    "            else: # predict the output\n",
    "                q = model.predict(s_t)       #input a stack of 4 images, get the prediction\n",
    "                max_Q = np.argmax(q)         # chosing index with maximum q value\n",
    "                action_index = max_Q \n",
    "                a_t[action_index] = 1        # o=> do nothing, 1=> jump\n",
    "                \n",
    "        #We reduced the epsilon (exploration parameter) gradually\n",
    "        if epsilon > FINAL_EPSILON and t > OBSERVE:\n",
    "            epsilon -= (INITIAL_EPSILON - FINAL_EPSILON) / EXPLORE \n",
    "\n",
    "        #run the selected action and observed next state and reward\n",
    "        x_t1, r_t, terminal = game_state.get_state(a_t)\n",
    "        print('fps: {0}'.format(1 / (time.time()-last_time))) # helpful for measuring frame rate\n",
    "        last_time = time.time()\n",
    "        x_t1 = x_t1.reshape(1, x_t1.shape[0], x_t1.shape[1], 1) #1x20x40x1\n",
    "        s_t1 = np.append(x_t1, s_t[:, :, :, :3], axis=3) # append the new image to input stack and remove the first one\n",
    "        \n",
    "        \n",
    "        # store the transition in D\n",
    "        D.append((s_t, action_index, r_t, s_t1, terminal))\n",
    "        if len(D) > REPLAY_MEMORY:\n",
    "            D.popleft()\n",
    "\n",
    "        #only train if done observing\n",
    "        if t > OBSERVE: \n",
    "            \n",
    "            #sample a minibatch to train on\n",
    "            minibatch = random.sample(D, BATCH)\n",
    "            inputs = np.zeros((BATCH, s_t.shape[1], s_t.shape[2], s_t.shape[3]))   #32, 20, 40, 4\n",
    "            targets = np.zeros((inputs.shape[0], ACTIONS))                         #32, 2\n",
    "\n",
    "            #Now we do the experience replay\n",
    "            for i in range(0, len(minibatch)):\n",
    "                state_t = minibatch[i][0]    # 4D stack of images\n",
    "                action_t = minibatch[i][1]   #This is action index\n",
    "                reward_t = minibatch[i][2]   #reward at state_t due to action_t\n",
    "                state_t1 = minibatch[i][3]   #next state\n",
    "                terminal = minibatch[i][4]   #wheather the agent died or survided due the action\n",
    "                \n",
    "\n",
    "                inputs[i:i + 1] = state_t    \n",
    "\n",
    "                targets[i] = model.predict(state_t)  # predicted q values\n",
    "                Q_sa = model.predict(state_t1)      #predict q values for next step\n",
    "                \n",
    "                if terminal:\n",
    "                    targets[i, action_t] = reward_t # if terminated, only equals reward\n",
    "                else:\n",
    "                    targets[i, action_t] = reward_t + GAMMA * np.max(Q_sa)\n",
    "\n",
    "            loss += model.train_on_batch(inputs, targets)\n",
    "            loss_df.loc[len(loss_df)] = loss\n",
    "            q_values_df.loc[len(q_values_df)] = np.max(Q_sa)\n",
    "        s_t = initial_state if terminal else s_t1 #reset game to initial frame if terminate\n",
    "        t = t + 1\n",
    "        \n",
    "        # save progress every 1000 iterations\n",
    "        if t % 1000 == 0:\n",
    "            print(\"Now we save model\")\n",
    "            model.save_weights(\"model.h5\", overwrite=True)\n",
    "            save_obj(D,\"D\") #saving episodes\n",
    "            save_obj(t,\"time\") #caching time steps\n",
    "            save_obj(epsilon,\"epsilon\") #cache epsilon to avoid repeated randomness in actions\n",
    "            loss_df.to_csv(\"./objects/loss_df.csv\",index=False)\n",
    "            scores_df.to_csv(\"./objects/scores_df.csv\",index=False)\n",
    "#             actions_df.to_csv(\"./objects/actions_df.csv\",index=False)\n",
    "            q_values_df.to_csv(q_value_file_path,index=False)\n",
    "            with open(\"model.json\", \"w\") as outfile:\n",
    "                json.dump(model.to_json(), outfile)\n",
    "            clear_output()\n",
    "        # print info\n",
    "        state = \"\"\n",
    "        if t <= OBSERVE:\n",
    "            state = \"observe\"\n",
    "        elif t > OBSERVE and t <= OBSERVE + EXPLORE:\n",
    "            state = \"explore\"\n",
    "        else:\n",
    "            state = \"train\"\n",
    "\n",
    "        print(\"TIMESTEP\", t, \"/ STATE\", state,             \"/ EPSILON\", epsilon, \"/ ACTION\", action_index, \"/ REWARD\", r_t,             \"/ Q_MAX \" , np.max(Q_sa), \"/ Loss \", loss)\n",
    "\n",
    "    print(\"Episode finished!\")\n",
    "    print(\"************************\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "#main function\n",
    "def playGame(observe=False):\n",
    "    game = qwop.Game()\n",
    "    game_state = Game_state(game)    \n",
    "    model = buildmodel()\n",
    "    try:\n",
    "        trainNetwork(model,game_state,observe=observe)\n",
    "    except StopIteration:\n",
    "        game.end()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TIMESTEP 54000 / STATE explore / EPSILON 0.04615489899991773 / ACTION 1 / REWARD 10.0 / Q_MAX  129.54584 / Loss  41.823036193847656\n",
      "fps: 0.027487569411813694\n",
      "TIMESTEP 54001 / STATE explore / EPSILON 0.04615389999991773 / ACTION 3 / REWARD 0 / Q_MAX  136.53079 / Loss  47.516815185546875\n",
      "fps: 0.7885982300505595\n",
      "TIMESTEP 54002 / STATE explore / EPSILON 0.04615290099991773 / ACTION 2 / REWARD 0 / Q_MAX  124.00511 / Loss  27.419479370117188\n",
      "fps: 0.7412474109471868\n",
      "TIMESTEP 54003 / STATE explore / EPSILON 0.04615190199991773 / ACTION 2 / REWARD 0 / Q_MAX  142.401 / Loss  55.99506378173828\n",
      "fps: 0.7727533006370976\n",
      "TIMESTEP 54004 / STATE explore / EPSILON 0.046150902999917726 / ACTION 1 / REWARD 0 / Q_MAX  124.18278 / Loss  39.7996940612793\n",
      "fps: 0.7781655955142024\n",
      "TIMESTEP 54005 / STATE explore / EPSILON 0.046149903999917724 / ACTION 2 / REWARD 0 / Q_MAX  115.12438 / Loss  83.25408935546875\n",
      "fps: 0.7467833135760494\n",
      "TIMESTEP 54006 / STATE explore / EPSILON 0.04614890499991772 / ACTION 2 / REWARD 0 / Q_MAX  129.34578 / Loss  45.07707977294922\n",
      "fps: 0.7563862577788165\n",
      "TIMESTEP 54007 / STATE explore / EPSILON 0.04614790599991772 / ACTION 3 / REWARD 0 / Q_MAX  123.423325 / Loss  65.19931030273438\n",
      "fps: 0.7256479623728457\n",
      "TIMESTEP 54008 / STATE explore / EPSILON 0.04614690699991772 / ACTION 0 / REWARD 0 / Q_MAX  144.04276 / Loss  154.4535675048828\n",
      "fps: 0.7685954926259395\n",
      "TIMESTEP 54009 / STATE explore / EPSILON 0.04614590799991772 / ACTION 2 / REWARD 0 / Q_MAX  133.11012 / Loss  69.09993743896484\n",
      "fps: 0.7656527583772113\n",
      "TIMESTEP 54010 / STATE explore / EPSILON 0.046144908999917716 / ACTION 2 / REWARD 0 / Q_MAX  137.24957 / Loss  41.20472717285156\n",
      "fps: 0.7942358637588455\n",
      "TIMESTEP 54011 / STATE explore / EPSILON 0.046143909999917715 / ACTION 3 / REWARD 0 / Q_MAX  126.60975 / Loss  74.63367462158203\n",
      "fps: 0.774548918895611\n",
      "TIMESTEP 54012 / STATE explore / EPSILON 0.04614291099991771 / ACTION 2 / REWARD 0 / Q_MAX  37.359287 / Loss  61.721710205078125\n",
      "fps: 0.7106913166390024\n",
      "TIMESTEP 54013 / STATE explore / EPSILON 0.04614191199991771 / ACTION 3 / REWARD 15.0 / Q_MAX  138.17357 / Loss  199.33489990234375\n",
      "error:  could not convert string to float:  \n",
      "score:  \n",
      "fps: 0.831207704423829\n",
      "TIMESTEP 54014 / STATE explore / EPSILON 0.04614091299991771 / ACTION 1 / REWARD 0 / Q_MAX  10.930737 / Loss  128.68557739257812\n",
      "fps: 0.7650673042616669\n",
      "TIMESTEP 54015 / STATE explore / EPSILON 0.04613991399991771 / ACTION 3 / REWARD 0 / Q_MAX  118.25001 / Loss  60.73064422607422\n",
      "fps: 0.7836542395909898\n",
      "TIMESTEP 54016 / STATE explore / EPSILON 0.04613891499991771 / ACTION 1 / REWARD 0 / Q_MAX  125.9329 / Loss  244.41378784179688\n",
      "fps: 0.7417973575605585\n",
      "TIMESTEP 54017 / STATE explore / EPSILON 0.046137915999917706 / ACTION 1 / REWARD 0 / Q_MAX  108.68518 / Loss  27.783599853515625\n",
      "fps: 0.7974024132127265\n",
      "TIMESTEP 54018 / STATE explore / EPSILON 0.046136916999917704 / ACTION 1 / REWARD 0 / Q_MAX  137.49156 / Loss  87.80860900878906\n",
      "fps: 0.7680052500358889\n",
      "TIMESTEP 54019 / STATE explore / EPSILON 0.0461359179999177 / ACTION 2 / REWARD 0 / Q_MAX  119.36142 / Loss  33.06410598754883\n",
      "fps: 0.7668273401484729\n",
      "TIMESTEP 54020 / STATE explore / EPSILON 0.0461349189999177 / ACTION 1 / REWARD 0 / Q_MAX  44.319138 / Loss  63.5444450378418\n",
      "fps: 0.773949500595182\n",
      "TIMESTEP 54021 / STATE explore / EPSILON 0.0461339199999177 / ACTION 2 / REWARD 0 / Q_MAX  152.39786 / Loss  84.3352279663086\n",
      "fps: 0.7954994004789341\n",
      "TIMESTEP 54022 / STATE explore / EPSILON 0.0461329209999177 / ACTION 1 / REWARD 0 / Q_MAX  137.48961 / Loss  38.666770935058594\n",
      "fps: 0.7961326776439218\n",
      "TIMESTEP 54023 / STATE explore / EPSILON 0.0461319219999177 / ACTION 1 / REWARD 0 / Q_MAX  131.94263 / Loss  30.55009651184082\n",
      "fps: 0.781816205983387\n",
      "TIMESTEP 54024 / STATE explore / EPSILON 0.046130922999917695 / ACTION 2 / REWARD 0 / Q_MAX  140.42142 / Loss  170.9352264404297\n",
      "fps: 0.759258059155565\n",
      "TIMESTEP 54025 / STATE explore / EPSILON 0.046129923999917694 / ACTION 2 / REWARD 0 / Q_MAX  126.78274 / Loss  64.60964965820312\n",
      "fps: 0.7507077422355974\n",
      "TIMESTEP 54026 / STATE explore / EPSILON 0.04612892499991769 / ACTION 2 / REWARD 0 / Q_MAX  139.13107 / Loss  111.9062271118164\n",
      "----------Random Action----------\n",
      "fps: 0.7861185352340754\n",
      "TIMESTEP 54027 / STATE explore / EPSILON 0.04612792599991769 / ACTION 2 / REWARD 0 / Q_MAX  114.84811 / Loss  66.64759826660156\n",
      "fps: 0.8012361907094285\n",
      "TIMESTEP 54028 / STATE explore / EPSILON 0.04612692699991769 / ACTION 2 / REWARD 0 / Q_MAX  139.59435 / Loss  39.28752517700195\n",
      "fps: 0.7668273401484729\n",
      "TIMESTEP 54029 / STATE explore / EPSILON 0.04612592799991769 / ACTION 1 / REWARD 0 / Q_MAX  152.19478 / Loss  90.75846099853516\n",
      "fps: 0.7569589061022095\n",
      "TIMESTEP 54030 / STATE explore / EPSILON 0.046124928999917686 / ACTION 1 / REWARD 0 / Q_MAX  138.37938 / Loss  127.01422882080078\n",
      "fps: 0.7910940656590661\n",
      "TIMESTEP 54031 / STATE explore / EPSILON 0.046123929999917684 / ACTION 3 / REWARD 0 / Q_MAX  133.48627 / Loss  73.64241790771484\n",
      "fps: 0.7967671152415213\n",
      "TIMESTEP 54032 / STATE explore / EPSILON 0.04612293099991768 / ACTION 3 / REWARD 0 / Q_MAX  136.66586 / Loss  46.81487274169922\n",
      "fps: 0.7674156324810283\n",
      "TIMESTEP 54033 / STATE explore / EPSILON 0.04612193199991768 / ACTION 1 / REWARD 0 / Q_MAX  148.42128 / Loss  49.510581970214844\n",
      "fps: 0.759258059155565\n",
      "TIMESTEP 54034 / STATE explore / EPSILON 0.04612093299991768 / ACTION 3 / REWARD 0 / Q_MAX  149.20752 / Loss  63.54344940185547\n",
      "fps: 0.806405401793997\n",
      "TIMESTEP 54035 / STATE explore / EPSILON 0.04611993399991768 / ACTION 2 / REWARD 0 / Q_MAX  127.264694 / Loss  68.30497741699219\n",
      "error:  could not convert string to float: 'wy' \n",
      "score:  wy\n",
      "fps: 0.7357931087110783\n",
      "TIMESTEP 54036 / STATE explore / EPSILON 0.04611893499991768 / ACTION 3 / REWARD -2.0 / Q_MAX  155.93399 / Loss  327.26165771484375\n",
      "fps: 0.7967672665986848\n",
      "TIMESTEP 54037 / STATE explore / EPSILON 0.046117935999917675 / ACTION 1 / REWARD 0 / Q_MAX  144.10217 / Loss  62.53129577636719\n",
      "fps: 0.7818160602532572\n",
      "TIMESTEP 54038 / STATE explore / EPSILON 0.046116936999917674 / ACTION 3 / REWARD 0 / Q_MAX  122.01341 / Loss  198.2441864013672\n",
      "----------Random Action----------\n",
      "fps: 0.8116419018367631\n",
      "TIMESTEP 54039 / STATE explore / EPSILON 0.04611593799991767 / ACTION 3 / REWARD 0 / Q_MAX  36.256268 / Loss  88.45307922363281\n",
      "fps: 0.7936055994145076\n",
      "TIMESTEP 54040 / STATE explore / EPSILON 0.04611493899991767 / ACTION 3 / REWARD 0 / Q_MAX  123.114586 / Loss  36.621849060058594\n",
      "fps: 0.7873566519681551\n",
      "TIMESTEP 54041 / STATE explore / EPSILON 0.04611393999991767 / ACTION 3 / REWARD 0 / Q_MAX  136.21585 / Loss  95.52592468261719\n",
      "fps: 0.8012361907094285\n",
      "TIMESTEP 54042 / STATE explore / EPSILON 0.04611294099991767 / ACTION 1 / REWARD -0.1 / Q_MAX  139.6641 / Loss  32.49448013305664\n",
      "fps: 0.8038126397562904\n",
      "TIMESTEP 54043 / STATE explore / EPSILON 0.046111941999917666 / ACTION 0 / REWARD -0.1 / Q_MAX  146.67876 / Loss  107.44249725341797\n",
      "fps: 0.7781654511418389\n",
      "TIMESTEP 54044 / STATE explore / EPSILON 0.046110942999917665 / ACTION 2 / REWARD 0 / Q_MAX  133.07217 / Loss  156.36944580078125\n",
      "fps: 0.7691866431636111\n",
      "TIMESTEP 54045 / STATE explore / EPSILON 0.04610994399991766 / ACTION 1 / REWARD 0 / Q_MAX  104.67837 / Loss  39.79283142089844\n",
      "fps: 0.7967672665986848\n",
      "TIMESTEP 54046 / STATE explore / EPSILON 0.04610894499991766 / ACTION 1 / REWARD 0 / Q_MAX  89.51237 / Loss  344.4570007324219\n",
      "fps: 0.7569590427129294\n",
      "TIMESTEP 54047 / STATE explore / EPSILON 0.04610794599991766 / ACTION 3 / REWARD 0 / Q_MAX  110.90156 / Loss  317.79705810546875\n",
      "fps: 0.8070564796608223\n",
      "TIMESTEP 54048 / STATE explore / EPSILON 0.04610694699991766 / ACTION 0 / REWARD 0 / Q_MAX  123.444016 / Loss  475.7218322753906\n",
      "fps: 0.7581068079527846\n",
      "TIMESTEP 54049 / STATE explore / EPSILON 0.04610594799991766 / ACTION 3 / REWARD 0 / Q_MAX  123.54113 / Loss  80.66844940185547\n",
      "fps: 0.7848843062585963\n",
      "TIMESTEP 54050 / STATE explore / EPSILON 0.046104948999917655 / ACTION 2 / REWARD 0 / Q_MAX  145.19691 / Loss  89.44145202636719\n",
      "fps: 0.7936052990978834\n",
      "TIMESTEP 54051 / STATE explore / EPSILON 0.046103949999917654 / ACTION 1 / REWARD 0 / Q_MAX  125.8384 / Loss  238.50732421875\n",
      "fps: 0.7910939164496377\n",
      "TIMESTEP 54052 / STATE explore / EPSILON 0.04610295099991765 / ACTION 1 / REWARD 0 / Q_MAX  46.246243 / Loss  115.12676239013672\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fps: 0.7961329798764166\n",
      "TIMESTEP 54053 / STATE explore / EPSILON 0.04610195199991765 / ACTION 0 / REWARD 0 / Q_MAX  143.50638 / Loss  92.31919860839844\n",
      "fps: 0.7867371064841083\n",
      "TIMESTEP 54054 / STATE explore / EPSILON 0.04610095299991765 / ACTION 3 / REWARD 0 / Q_MAX  159.25093 / Loss  47.91556167602539\n",
      "fps: 0.7948672806195498\n",
      "TIMESTEP 54055 / STATE explore / EPSILON 0.04609995399991765 / ACTION 3 / REWARD 0 / Q_MAX  43.225647 / Loss  41.93206024169922\n",
      "fps: 0.8051070106224577\n",
      "TIMESTEP 54056 / STATE explore / EPSILON 0.046098954999917646 / ACTION 1 / REWARD 0 / Q_MAX  48.17327 / Loss  117.47930145263672\n",
      "fps: 0.8250353525826318\n",
      "TIMESTEP 54057 / STATE explore / EPSILON 0.046097955999917645 / ACTION 1 / REWARD 0 / Q_MAX  137.92683 / Loss  72.64205932617188\n",
      "fps: 0.7867371064841083\n",
      "TIMESTEP 54058 / STATE explore / EPSILON 0.04609695699991764 / ACTION 3 / REWARD 0 / Q_MAX  146.02002 / Loss  58.10223388671875\n",
      "fps: 0.7680052500358889\n",
      "TIMESTEP 54059 / STATE explore / EPSILON 0.04609595799991764 / ACTION 3 / REWARD 0 / Q_MAX  51.014008 / Loss  64.14598083496094\n",
      "----------Random Action----------\n",
      "fps: 0.8236761651600416\n",
      "TIMESTEP 54060 / STATE explore / EPSILON 0.04609495899991764 / ACTION 2 / REWARD 0 / Q_MAX  155.0137 / Loss  40.37202072143555\n",
      "fps: 0.8070564796608223\n",
      "TIMESTEP 54061 / STATE explore / EPSILON 0.04609395999991764 / ACTION 2 / REWARD -0.1 / Q_MAX  143.99255 / Loss  87.4411849975586\n",
      "fps: 0.7309524784620787\n",
      "TIMESTEP 54062 / STATE explore / EPSILON 0.04609296099991764 / ACTION 3 / REWARD 0 / Q_MAX  144.19458 / Loss  31.309886932373047\n",
      "fps: 0.8136230273156945\n",
      "TIMESTEP 54063 / STATE explore / EPSILON 0.046091961999917636 / ACTION 1 / REWARD 0 / Q_MAX  152.06245 / Loss  244.3716278076172\n",
      "fps: 0.7769563376258573\n",
      "TIMESTEP 54064 / STATE explore / EPSILON 0.046090962999917634 / ACTION 1 / REWARD 0 / Q_MAX  124.57084 / Loss  47.0428352355957\n",
      "fps: 0.7967672665986848\n",
      "TIMESTEP 54065 / STATE explore / EPSILON 0.04608996399991763 / ACTION 0 / REWARD 0 / Q_MAX  127.41106 / Loss  73.84567260742188\n",
      "fps: 0.7674159133032903\n",
      "TIMESTEP 54066 / STATE explore / EPSILON 0.04608896499991763 / ACTION 3 / REWARD 0 / Q_MAX  139.52266 / Loss  185.215576171875\n",
      "fps: 0.7954994004789341\n",
      "TIMESTEP 54067 / STATE explore / EPSILON 0.04608796599991763 / ACTION 3 / REWARD 0 / Q_MAX  129.90837 / Loss  82.11552429199219\n",
      "fps: 0.7769563376258573\n",
      "TIMESTEP 54068 / STATE explore / EPSILON 0.04608696699991763 / ACTION 3 / REWARD 0 / Q_MAX  117.406 / Loss  57.72295379638672\n",
      "fps: 0.7368775382133087\n",
      "TIMESTEP 54069 / STATE explore / EPSILON 0.046085967999917626 / ACTION 3 / REWARD 0 / Q_MAX  101.79578 / Loss  76.48197937011719\n",
      "fps: 0.8012363437692821\n",
      "TIMESTEP 54070 / STATE explore / EPSILON 0.046084968999917625 / ACTION 2 / REWARD 0 / Q_MAX  142.22034 / Loss  111.84548950195312\n",
      "fps: 0.7942358637588455\n",
      "TIMESTEP 54071 / STATE explore / EPSILON 0.04608396999991762 / ACTION 3 / REWARD 0 / Q_MAX  78.60352 / Loss  35.51792526245117\n",
      "fps: 0.7818159145231819\n",
      "TIMESTEP 54072 / STATE explore / EPSILON 0.04608297099991762 / ACTION 3 / REWARD 0 / Q_MAX  83.175514 / Loss  258.20745849609375\n",
      "fps: 0.7942357133619053\n",
      "TIMESTEP 54073 / STATE explore / EPSILON 0.04608197199991762 / ACTION 1 / REWARD -0.1 / Q_MAX  130.56871 / Loss  133.96730041503906\n",
      "fps: 0.8012361907094285\n",
      "TIMESTEP 54074 / STATE explore / EPSILON 0.04608097299991762 / ACTION 0 / REWARD 0 / Q_MAX  134.3328 / Loss  125.48816680908203\n",
      "fps: 0.8012361907094285\n",
      "TIMESTEP 54075 / STATE explore / EPSILON 0.04607997399991762 / ACTION 1 / REWARD 0 / Q_MAX  110.66057 / Loss  257.193115234375\n",
      "fps: 0.7836542395909898\n",
      "TIMESTEP 54076 / STATE explore / EPSILON 0.046078974999917616 / ACTION 1 / REWARD 0 / Q_MAX  116.43139 / Loss  326.8431091308594\n",
      "fps: 0.7763530451138745\n",
      "TIMESTEP 54077 / STATE explore / EPSILON 0.046077975999917614 / ACTION 1 / REWARD 0 / Q_MAX  76.47027 / Loss  52.96464538574219\n",
      "fps: 0.7942357133619053\n",
      "TIMESTEP 54078 / STATE explore / EPSILON 0.04607697699991761 / ACTION 3 / REWARD 0 / Q_MAX  145.7374 / Loss  69.31463623046875\n",
      "fps: 0.793605449256167\n",
      "TIMESTEP 54079 / STATE explore / EPSILON 0.04607597799991761 / ACTION 3 / REWARD 0 / Q_MAX  116.13429 / Loss  68.85028839111328\n",
      "fps: 0.7961329798764166\n",
      "TIMESTEP 54080 / STATE explore / EPSILON 0.04607497899991761 / ACTION 0 / REWARD 0 / Q_MAX  51.13908 / Loss  203.76806640625\n",
      "fps: 0.7842687906083047\n",
      "TIMESTEP 54081 / STATE explore / EPSILON 0.04607397999991761 / ACTION 3 / REWARD 0 / Q_MAX  127.899734 / Loss  47.78801727294922\n",
      "fps: 0.8090152563740535\n",
      "TIMESTEP 54082 / STATE explore / EPSILON 0.04607298099991761 / ACTION 1 / REWARD 0 / Q_MAX  142.61841 / Loss  71.11466217041016\n",
      "fps: 0.821645518761799\n",
      "TIMESTEP 54083 / STATE explore / EPSILON 0.046071981999917605 / ACTION 1 / REWARD 0 / Q_MAX  128.41136 / Loss  92.25601196289062\n",
      "fps: 0.7936052990978834\n",
      "TIMESTEP 54084 / STATE explore / EPSILON 0.046070982999917603 / ACTION 1 / REWARD 0 / Q_MAX  144.13148 / Loss  61.378929138183594\n",
      "fps: 0.7751494096613504\n",
      "TIMESTEP 54085 / STATE explore / EPSILON 0.0460699839999176 / ACTION 1 / REWARD 0 / Q_MAX  140.67891 / Loss  42.7701416015625\n",
      "fps: 0.7824278764405731\n",
      "TIMESTEP 54086 / STATE explore / EPSILON 0.0460689849999176 / ACTION 1 / REWARD 0 / Q_MAX  109.35998 / Loss  54.134788513183594\n",
      "fps: 0.765067164708627\n",
      "TIMESTEP 54087 / STATE explore / EPSILON 0.0460679859999176 / ACTION 1 / REWARD 0 / Q_MAX  127.916 / Loss  236.08438110351562\n",
      "fps: 0.781816205983387\n",
      "TIMESTEP 54088 / STATE explore / EPSILON 0.0460669869999176 / ACTION 1 / REWARD 0 / Q_MAX  104.47636 / Loss  46.68834686279297\n",
      "fps: 0.754104709396769\n",
      "TIMESTEP 54089 / STATE explore / EPSILON 0.046065987999917596 / ACTION 0 / REWARD -0.1 / Q_MAX  76.62736 / Loss  297.2696838378906\n",
      "fps: 0.797402564811354\n",
      "TIMESTEP 54090 / STATE explore / EPSILON 0.046064988999917594 / ACTION 1 / REWARD -0.1 / Q_MAX  104.06438 / Loss  59.58353805541992\n",
      "fps: 0.7842687906083047\n",
      "TIMESTEP 54091 / STATE explore / EPSILON 0.04606398999991759 / ACTION 3 / REWARD -0.1 / Q_MAX  135.8794 / Loss  29.670743942260742\n",
      "fps: 0.754104709396769\n",
      "TIMESTEP 54092 / STATE explore / EPSILON 0.04606299099991759 / ACTION 3 / REWARD 0 / Q_MAX  132.29768 / Loss  31.518579483032227\n",
      "fps: 0.7775607126344644\n",
      "TIMESTEP 54093 / STATE explore / EPSILON 0.04606199199991759 / ACTION 2 / REWARD 0 / Q_MAX  124.617676 / Loss  53.003509521484375\n",
      "fps: 0.7685954926259395\n",
      "TIMESTEP 54094 / STATE explore / EPSILON 0.04606099299991759 / ACTION 0 / REWARD 0 / Q_MAX  108.78302 / Loss  46.18048095703125\n",
      "fps: 0.7812053456350289\n",
      "TIMESTEP 54095 / STATE explore / EPSILON 0.04605999399991759 / ACTION 2 / REWARD 0 / Q_MAX  92.199005 / Loss  50.2611083984375\n",
      "fps: 0.7954992496030853\n",
      "TIMESTEP 54096 / STATE explore / EPSILON 0.046058994999917585 / ACTION 3 / REWARD 0 / Q_MAX  134.1989 / Loss  140.48719787597656\n",
      "fps: 0.7775604243385098\n",
      "TIMESTEP 54097 / STATE explore / EPSILON 0.046057995999917584 / ACTION 0 / REWARD 0 / Q_MAX  123.21893 / Loss  21.38588523864746\n",
      "fps: 0.8077082986301344\n",
      "TIMESTEP 54098 / STATE explore / EPSILON 0.04605699699991758 / ACTION 2 / REWARD 0 / Q_MAX  148.9699 / Loss  251.3167266845703\n",
      "fps: 0.7685956334691258\n",
      "TIMESTEP 54099 / STATE explore / EPSILON 0.04605599799991758 / ACTION 0 / REWARD 0 / Q_MAX  141.31775 / Loss  22.06328582763672\n",
      "fps: 0.7451138534587725\n",
      "TIMESTEP 54100 / STATE explore / EPSILON 0.04605499899991758 / ACTION 0 / REWARD 0 / Q_MAX  143.43475 / Loss  43.746803283691406\n",
      "fps: 0.7357931087110783\n",
      "TIMESTEP 54101 / STATE explore / EPSILON 0.04605399999991758 / ACTION 3 / REWARD 15.0 / Q_MAX  120.26192 / Loss  37.35491943359375\n",
      "error:  could not convert string to float:  \n",
      "score:  \n",
      "fps: 0.8402881133894032\n",
      "TIMESTEP 54102 / STATE explore / EPSILON 0.046053000999917576 / ACTION 1 / REWARD 0 / Q_MAX  130.05779 / Loss  112.35525512695312\n",
      "fps: 0.8025222705792627\n",
      "TIMESTEP 54103 / STATE explore / EPSILON 0.046052001999917574 / ACTION 3 / REWARD 0 / Q_MAX  103.52627 / Loss  30.683748245239258\n",
      "fps: 0.8038124857105569\n",
      "TIMESTEP 54104 / STATE explore / EPSILON 0.04605100299991757 / ACTION 2 / REWARD 0 / Q_MAX  129.30931 / Loss  37.97442626953125\n",
      "fps: 0.8044590730901859\n",
      "TIMESTEP 54105 / STATE explore / EPSILON 0.04605000399991757 / ACTION 1 / REWARD 0 / Q_MAX  129.27014 / Loss  55.28809356689453\n",
      "----------Random Action----------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fps: 0.7848844531348098\n",
      "TIMESTEP 54106 / STATE explore / EPSILON 0.04604900499991757 / ACTION 2 / REWARD 0 / Q_MAX  123.33811 / Loss  41.42634963989258\n",
      "fps: 0.8169466950726921\n",
      "TIMESTEP 54107 / STATE explore / EPSILON 0.04604800599991757 / ACTION 2 / REWARD 0 / Q_MAX  145.239 / Loss  29.370014190673828\n",
      "fps: 0.8109837211748473\n",
      "TIMESTEP 54108 / STATE explore / EPSILON 0.04604700699991757 / ACTION 0 / REWARD 0 / Q_MAX  140.36803 / Loss  34.443382263183594\n",
      "fps: 0.814285500730843\n",
      "TIMESTEP 54109 / STATE explore / EPSILON 0.046046007999917565 / ACTION 1 / REWARD 0 / Q_MAX  118.76502 / Loss  92.72309875488281\n",
      "fps: 0.8129616309534123\n",
      "TIMESTEP 54110 / STATE explore / EPSILON 0.046045008999917564 / ACTION 1 / REWARD 0 / Q_MAX  137.87689 / Loss  40.134578704833984\n",
      "fps: 0.793605449256167\n",
      "TIMESTEP 54111 / STATE explore / EPSILON 0.04604400999991756 / ACTION 1 / REWARD 0 / Q_MAX  108.09003 / Loss  24.805252075195312\n",
      "fps: 0.8012361907094285\n",
      "TIMESTEP 54112 / STATE explore / EPSILON 0.04604301099991756 / ACTION 2 / REWARD 0 / Q_MAX  142.40338 / Loss  183.15762329101562\n",
      "fps: 0.7967671152415213\n",
      "TIMESTEP 54113 / STATE explore / EPSILON 0.04604201199991756 / ACTION 2 / REWARD 0 / Q_MAX  135.38516 / Loss  116.3694076538086\n",
      "fps: 0.8025222705792627\n",
      "TIMESTEP 54114 / STATE explore / EPSILON 0.04604101299991756 / ACTION 1 / REWARD 0 / Q_MAX  66.67399 / Loss  68.6804428100586\n",
      "fps: 0.793605449256167\n",
      "TIMESTEP 54115 / STATE explore / EPSILON 0.046040013999917556 / ACTION 1 / REWARD 0 / Q_MAX  118.84591 / Loss  76.34642791748047\n",
      "fps: 0.7948671299833837\n",
      "TIMESTEP 54116 / STATE explore / EPSILON 0.046039014999917555 / ACTION 2 / REWARD 0 / Q_MAX  87.79238 / Loss  63.91794204711914\n",
      "fps: 0.7948669793472747\n",
      "TIMESTEP 54117 / STATE explore / EPSILON 0.04603801599991755 / ACTION 1 / REWARD 0 / Q_MAX  116.55031 / Loss  215.69158935546875\n",
      "fps: 0.7879770259536386\n",
      "TIMESTEP 54118 / STATE explore / EPSILON 0.04603701699991755 / ACTION 0 / REWARD -0.1 / Q_MAX  123.86949 / Loss  48.65753173828125\n",
      "fps: 0.8263990331398641\n",
      "TIMESTEP 54119 / STATE explore / EPSILON 0.04603601799991755 / ACTION 1 / REWARD 0 / Q_MAX  125.79999 / Loss  29.851226806640625\n",
      "fps: 0.8044592273838481\n",
      "TIMESTEP 54120 / STATE explore / EPSILON 0.04603501899991755 / ACTION 1 / REWARD 0 / Q_MAX  116.36527 / Loss  38.58642578125\n",
      "fps: 0.8149493705195848\n",
      "TIMESTEP 54121 / STATE explore / EPSILON 0.04603401999991755 / ACTION 1 / REWARD 0 / Q_MAX  111.1362 / Loss  63.36963653564453\n",
      "----------Random Action----------\n",
      "fps: 0.8149493705195848\n",
      "TIMESTEP 54122 / STATE explore / EPSILON 0.046033020999917545 / ACTION 0 / REWARD 0 / Q_MAX  153.9267 / Loss  37.122562408447266\n",
      "fps: 0.8044592273838481\n",
      "TIMESTEP 54123 / STATE explore / EPSILON 0.046032021999917544 / ACTION 2 / REWARD 0 / Q_MAX  127.15112 / Loss  81.3812484741211\n",
      "fps: 0.8025222705792627\n",
      "TIMESTEP 54124 / STATE explore / EPSILON 0.04603102299991754 / ACTION 2 / REWARD 0 / Q_MAX  149.45831 / Loss  99.50898742675781\n",
      "fps: 0.7929760347265811\n",
      "TIMESTEP 54125 / STATE explore / EPSILON 0.04603002399991754 / ACTION 3 / REWARD 0 / Q_MAX  137.63544 / Loss  64.77777099609375\n",
      "fps: 0.7942358637588455\n",
      "TIMESTEP 54126 / STATE explore / EPSILON 0.04602902499991754 / ACTION 0 / REWARD 0 / Q_MAX  89.90193 / Loss  58.01887512207031\n",
      "fps: 0.7668273401484729\n",
      "TIMESTEP 54127 / STATE explore / EPSILON 0.04602802599991754 / ACTION 1 / REWARD 0 / Q_MAX  139.09431 / Loss  212.85362243652344\n",
      "fps: 0.7848844531348098\n",
      "TIMESTEP 54128 / STATE explore / EPSILON 0.046027026999917536 / ACTION 3 / REWARD 0 / Q_MAX  123.96461 / Loss  441.08587646484375\n",
      "fps: 0.8057556831531081\n",
      "TIMESTEP 54129 / STATE explore / EPSILON 0.046026027999917535 / ACTION 2 / REWARD 0 / Q_MAX  132.92416 / Loss  202.33740234375\n",
      "fps: 0.7917203455304925\n",
      "TIMESTEP 54130 / STATE explore / EPSILON 0.04602502899991753 / ACTION 3 / REWARD 0 / Q_MAX  150.77606 / Loss  79.95733642578125\n",
      "fps: 0.793605449256167\n",
      "TIMESTEP 54131 / STATE explore / EPSILON 0.04602402999991753 / ACTION 2 / REWARD 0 / Q_MAX  141.90898 / Loss  46.33838653564453\n",
      "fps: 0.797402564811354\n",
      "TIMESTEP 54132 / STATE explore / EPSILON 0.04602303099991753 / ACTION 3 / REWARD 0 / Q_MAX  115.30244 / Loss  47.8438720703125\n",
      "fps: 0.7936052990978834\n",
      "TIMESTEP 54133 / STATE explore / EPSILON 0.04602203199991753 / ACTION 3 / REWARD 0 / Q_MAX  153.37962 / Loss  91.38721466064453\n",
      "fps: 0.8142856588169057\n",
      "TIMESTEP 54134 / STATE explore / EPSILON 0.04602103299991753 / ACTION 0 / REWARD 0 / Q_MAX  148.03328 / Loss  57.65803527832031\n",
      "fps: 0.7910940656590661\n",
      "TIMESTEP 54135 / STATE explore / EPSILON 0.046020033999917526 / ACTION 2 / REWARD 0 / Q_MAX  99.9146 / Loss  62.18821716308594\n",
      "fps: 0.8031669368908586\n",
      "TIMESTEP 54136 / STATE explore / EPSILON 0.046019034999917524 / ACTION 3 / REWARD 0 / Q_MAX  145.99277 / Loss  222.94961547851562\n",
      "fps: 0.8057556831531081\n",
      "TIMESTEP 54137 / STATE explore / EPSILON 0.04601803599991752 / ACTION 3 / REWARD 0 / Q_MAX  154.11038 / Loss  95.29330444335938\n",
      "fps: 0.7923477674754426\n",
      "TIMESTEP 54138 / STATE explore / EPSILON 0.04601703699991752 / ACTION 1 / REWARD 0 / Q_MAX  155.86612 / Loss  295.712890625\n",
      "fps: 0.7923477674754426\n",
      "TIMESTEP 54139 / STATE explore / EPSILON 0.04601603799991752 / ACTION 3 / REWARD 0 / Q_MAX  136.98671 / Loss  70.6309814453125\n",
      "fps: 0.7974024132127265\n",
      "TIMESTEP 54140 / STATE explore / EPSILON 0.04601503899991752 / ACTION 3 / REWARD 0 / Q_MAX  139.51631 / Loss  37.515167236328125\n",
      "fps: 0.814285500730843\n",
      "TIMESTEP 54141 / STATE explore / EPSILON 0.046014039999917516 / ACTION 0 / REWARD 0 / Q_MAX  132.31256 / Loss  191.2905731201172\n",
      "fps: 0.811641744775556\n",
      "TIMESTEP 54142 / STATE explore / EPSILON 0.046013040999917515 / ACTION 3 / REWARD 0 / Q_MAX  131.23903 / Loss  161.00291442871094\n",
      "fps: 0.8038126397562904\n",
      "TIMESTEP 54143 / STATE explore / EPSILON 0.04601204199991751 / ACTION 3 / REWARD 0 / Q_MAX  134.20605 / Loss  57.06149673461914\n",
      "fps: 0.8142858169030299\n",
      "TIMESTEP 54144 / STATE explore / EPSILON 0.04601104299991751 / ACTION 2 / REWARD -0.1 / Q_MAX  132.34235 / Loss  154.10076904296875\n",
      "fps: 0.8025225776825349\n",
      "TIMESTEP 54145 / STATE explore / EPSILON 0.04601004399991751 / ACTION 3 / REWARD -0.1 / Q_MAX  121.037766 / Loss  55.54033660888672\n",
      "fps: 0.8038127938020828\n",
      "TIMESTEP 54146 / STATE explore / EPSILON 0.04600904499991751 / ACTION 2 / REWARD 0 / Q_MAX  153.00127 / Loss  42.421905517578125\n",
      "fps: 0.8136230273156945\n",
      "TIMESTEP 54147 / STATE explore / EPSILON 0.04600804599991751 / ACTION 3 / REWARD 0 / Q_MAX  167.2352 / Loss  265.4015197753906\n",
      "fps: 0.8109837211748473\n",
      "TIMESTEP 54148 / STATE explore / EPSILON 0.046007046999917506 / ACTION 3 / REWARD 0 / Q_MAX  162.16806 / Loss  70.00603485107422\n",
      "fps: 0.8012361907094285\n",
      "TIMESTEP 54149 / STATE explore / EPSILON 0.046006047999917504 / ACTION 1 / REWARD 0 / Q_MAX  108.34254 / Loss  65.96533966064453\n",
      "fps: 0.8044592273838481\n",
      "TIMESTEP 54150 / STATE explore / EPSILON 0.0460050489999175 / ACTION 1 / REWARD 0 / Q_MAX  138.62125 / Loss  102.79878234863281\n",
      "fps: 0.7910939164496377\n",
      "TIMESTEP 54151 / STATE explore / EPSILON 0.0460040499999175 / ACTION 0 / REWARD 0 / Q_MAX  152.3008 / Loss  74.00254821777344\n",
      "fps: 0.8038124857105569\n",
      "TIMESTEP 54152 / STATE explore / EPSILON 0.0460030509999175 / ACTION 1 / REWARD 0 / Q_MAX  113.667175 / Loss  31.484712600708008\n",
      "fps: 0.7942357133619053\n",
      "TIMESTEP 54153 / STATE explore / EPSILON 0.0460020519999175 / ACTION 2 / REWARD 0 / Q_MAX  66.04773 / Loss  306.5002136230469\n",
      "fps: 0.814285500730843\n",
      "TIMESTEP 54154 / STATE explore / EPSILON 0.0460010529999175 / ACTION 2 / REWARD 0 / Q_MAX  141.02937 / Loss  118.22474670410156\n",
      "fps: 0.8136230273156945\n",
      "TIMESTEP 54155 / STATE explore / EPSILON 0.046000053999917495 / ACTION 3 / REWARD 0 / Q_MAX  134.52849 / Loss  64.13001251220703\n",
      "fps: 0.8216458406753384\n",
      "TIMESTEP 54156 / STATE explore / EPSILON 0.045999054999917494 / ACTION 1 / REWARD 0 / Q_MAX  152.16974 / Loss  164.145751953125\n",
      "fps: 0.793605449256167\n",
      "TIMESTEP 54157 / STATE explore / EPSILON 0.04599805599991749 / ACTION 0 / REWARD 0 / Q_MAX  149.37596 / Loss  148.95028686523438\n",
      "fps: 0.7812052001325382\n",
      "TIMESTEP 54158 / STATE explore / EPSILON 0.04599705699991749 / ACTION 3 / REWARD 0 / Q_MAX  150.11328 / Loss  67.30535125732422\n",
      "fps: 0.8044593816775695\n",
      "TIMESTEP 54159 / STATE explore / EPSILON 0.04599605799991749 / ACTION 2 / REWARD 0 / Q_MAX  140.45395 / Loss  87.20065307617188\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fps: 0.8129617885258587\n",
      "TIMESTEP 54160 / STATE explore / EPSILON 0.04599505899991749 / ACTION 3 / REWARD 0 / Q_MAX  149.30211 / Loss  86.38479614257812\n",
      "fps: 0.7961328287601405\n",
      "TIMESTEP 54161 / STATE explore / EPSILON 0.045994059999917486 / ACTION 3 / REWARD 0 / Q_MAX  146.39233 / Loss  143.9739227294922\n",
      "fps: 0.8038126397562904\n",
      "TIMESTEP 54162 / STATE explore / EPSILON 0.045993060999917484 / ACTION 1 / REWARD 0 / Q_MAX  141.99776 / Loss  134.0864715576172\n",
      "fps: 0.8136233429736379\n",
      "TIMESTEP 54163 / STATE explore / EPSILON 0.04599206199991748 / ACTION 0 / REWARD 0 / Q_MAX  76.78285 / Loss  50.317291259765625\n",
      "fps: 0.7917201960847234\n",
      "TIMESTEP 54164 / STATE explore / EPSILON 0.04599106299991748 / ACTION 1 / REWARD 0 / Q_MAX  135.70982 / Loss  116.49728393554688\n",
      "fps: 0.8109837211748473\n",
      "TIMESTEP 54165 / STATE explore / EPSILON 0.04599006399991748 / ACTION 1 / REWARD 0 / Q_MAX  130.41011 / Loss  63.82291030883789\n",
      "fps: 0.8129614733810272\n",
      "TIMESTEP 54166 / STATE explore / EPSILON 0.04598906499991748 / ACTION 1 / REWARD 0 / Q_MAX  133.70224 / Loss  33.89183044433594\n",
      "fps: 0.7942357133619053\n",
      "TIMESTEP 54167 / STATE explore / EPSILON 0.04598806599991748 / ACTION 3 / REWARD 0 / Q_MAX  140.57315 / Loss  59.54456329345703\n",
      "fps: 0.8149493705195848\n",
      "TIMESTEP 54168 / STATE explore / EPSILON 0.045987066999917475 / ACTION 2 / REWARD 0 / Q_MAX  160.55876 / Loss  57.211463928222656\n",
      "fps: 0.764482187434384\n",
      "TIMESTEP 54169 / STATE explore / EPSILON 0.045986067999917474 / ACTION 0 / REWARD 0 / Q_MAX  135.51566 / Loss  25.477821350097656\n",
      "fps: 0.7910940656590661\n",
      "TIMESTEP 54170 / STATE explore / EPSILON 0.04598506899991747 / ACTION 0 / REWARD 0 / Q_MAX  158.45055 / Loss  319.4342346191406\n",
      "fps: 0.8018786383275178\n",
      "TIMESTEP 54171 / STATE explore / EPSILON 0.04598406999991747 / ACTION 1 / REWARD 0 / Q_MAX  129.70795 / Loss  136.89846801757812\n",
      "fps: 0.7781657398866194\n",
      "TIMESTEP 54172 / STATE explore / EPSILON 0.04598307099991747 / ACTION 0 / REWARD 0 / Q_MAX  152.53754 / Loss  49.831634521484375\n",
      "fps: 0.7929760347265811\n",
      "TIMESTEP 54173 / STATE explore / EPSILON 0.04598207199991747 / ACTION 0 / REWARD 0 / Q_MAX  144.27078 / Loss  72.03031921386719\n",
      "fps: 0.8038124857105569\n",
      "TIMESTEP 54174 / STATE explore / EPSILON 0.045981072999917466 / ACTION 3 / REWARD 0 / Q_MAX  141.883 / Loss  305.5491943359375\n",
      "fps: 0.8142858169030299\n",
      "TIMESTEP 54175 / STATE explore / EPSILON 0.045980073999917465 / ACTION 2 / REWARD 0 / Q_MAX  160.55421 / Loss  40.63855743408203\n",
      "fps: 0.8025222705792627\n",
      "TIMESTEP 54176 / STATE explore / EPSILON 0.04597907499991746 / ACTION 2 / REWARD 0 / Q_MAX  151.56 / Loss  359.1920471191406\n",
      "fps: 0.8243552796285992\n",
      "TIMESTEP 54177 / STATE explore / EPSILON 0.04597807599991746 / ACTION 3 / REWARD 0 / Q_MAX  44.890343 / Loss  90.80343627929688\n",
      "fps: 0.8223212874806443\n",
      "TIMESTEP 54178 / STATE explore / EPSILON 0.04597707699991746 / ACTION 1 / REWARD 0 / Q_MAX  172.82018 / Loss  116.0728759765625\n",
      "fps: 0.8012361907094285\n",
      "TIMESTEP 54179 / STATE explore / EPSILON 0.04597607799991746 / ACTION 3 / REWARD 0 / Q_MAX  133.3246 / Loss  146.1644287109375\n",
      "fps: 0.8025222705792627\n",
      "TIMESTEP 54180 / STATE explore / EPSILON 0.04597507899991746 / ACTION 1 / REWARD 0 / Q_MAX  112.721695 / Loss  36.80128479003906\n",
      "fps: 0.8025222705792627\n",
      "TIMESTEP 54181 / STATE explore / EPSILON 0.045974079999917455 / ACTION 1 / REWARD 0 / Q_MAX  155.45976 / Loss  76.70914459228516\n",
      "fps: 0.8129617885258587\n",
      "TIMESTEP 54182 / STATE explore / EPSILON 0.045973080999917454 / ACTION 1 / REWARD 0 / Q_MAX  139.8675 / Loss  322.2559509277344\n",
      "fps: 0.8176147535822634\n",
      "TIMESTEP 54183 / STATE explore / EPSILON 0.04597208199991745 / ACTION 1 / REWARD 0 / Q_MAX  164.13194 / Loss  91.02490234375\n",
      "fps: 0.7855010830289753\n",
      "TIMESTEP 54184 / STATE explore / EPSILON 0.04597108299991745 / ACTION 1 / REWARD 0 / Q_MAX  151.97849 / Loss  160.04103088378906\n",
      "fps: 0.7721565151276663\n",
      "TIMESTEP 54185 / STATE explore / EPSILON 0.04597008399991745 / ACTION 3 / REWARD 0 / Q_MAX  144.42139 / Loss  50.63877868652344\n",
      "fps: 0.8051068560802085\n",
      "TIMESTEP 54186 / STATE explore / EPSILON 0.04596908499991745 / ACTION 2 / REWARD 0 / Q_MAX  118.426346 / Loss  175.75164794921875\n",
      "fps: 0.7942357133619053\n",
      "TIMESTEP 54187 / STATE explore / EPSILON 0.045968085999917446 / ACTION 0 / REWARD 0 / Q_MAX  148.49614 / Loss  140.10684204101562\n",
      "error:  could not convert string to float:  \n",
      "score:  \n",
      "fps: 0.7147553912490903\n",
      "TIMESTEP 54188 / STATE explore / EPSILON 0.045967086999917445 / ACTION 2 / REWARD 33.0 / Q_MAX  133.47229 / Loss  110.47235107421875\n",
      "fps: 0.8051068560802085\n",
      "TIMESTEP 54189 / STATE explore / EPSILON 0.04596608799991744 / ACTION 2 / REWARD 0 / Q_MAX  138.59515 / Loss  53.16802978515625\n",
      "fps: 0.8044592273838481\n",
      "TIMESTEP 54190 / STATE explore / EPSILON 0.04596508899991744 / ACTION 1 / REWARD 0 / Q_MAX  121.06938 / Loss  118.28387451171875\n",
      "fps: 0.8136233429736379\n",
      "TIMESTEP 54191 / STATE explore / EPSILON 0.04596408999991744 / ACTION 2 / REWARD 0 / Q_MAX  125.60069 / Loss  158.74769592285156\n",
      "fps: 0.7923476177927143\n",
      "TIMESTEP 54192 / STATE explore / EPSILON 0.04596309099991744 / ACTION 2 / REWARD 0 / Q_MAX  136.00706 / Loss  232.2392578125\n",
      "fps: 0.8012361907094285\n",
      "TIMESTEP 54193 / STATE explore / EPSILON 0.04596209199991744 / ACTION 0 / REWARD 0 / Q_MAX  65.56822 / Loss  72.672119140625\n",
      "fps: 0.7674157728921336\n",
      "TIMESTEP 54194 / STATE explore / EPSILON 0.045961092999917436 / ACTION 1 / REWARD 0 / Q_MAX  151.03087 / Loss  132.365478515625\n",
      "fps: 0.8012361907094285\n",
      "TIMESTEP 54195 / STATE explore / EPSILON 0.045960093999917434 / ACTION 2 / REWARD 0 / Q_MAX  155.01971 / Loss  131.30804443359375\n",
      "fps: 0.7484601379286716\n",
      "TIMESTEP 54196 / STATE explore / EPSILON 0.04595909499991743 / ACTION 0 / REWARD 0 / Q_MAX  116.61056 / Loss  147.063720703125\n",
      "fps: 0.8031667830925157\n",
      "TIMESTEP 54197 / STATE explore / EPSILON 0.04595809599991743 / ACTION 1 / REWARD 0 / Q_MAX  151.89793 / Loss  323.9179992675781\n",
      "fps: 0.7727534430082212\n",
      "TIMESTEP 54198 / STATE explore / EPSILON 0.04595709699991743 / ACTION 2 / REWARD 0 / Q_MAX  155.64072 / Loss  76.97407531738281\n",
      "fps: 0.7812050546301018\n",
      "TIMESTEP 54199 / STATE explore / EPSILON 0.04595609799991743 / ACTION 3 / REWARD 0 / Q_MAX  162.02327 / Loss  331.2740173339844\n",
      "fps: 0.7954994004789341\n",
      "TIMESTEP 54200 / STATE explore / EPSILON 0.045955098999917426 / ACTION 1 / REWARD 0 / Q_MAX  153.59491 / Loss  125.34086608886719\n",
      "fps: 0.8038124857105569\n",
      "TIMESTEP 54201 / STATE explore / EPSILON 0.045954099999917425 / ACTION 2 / REWARD 0 / Q_MAX  128.39073 / Loss  21.55797576904297\n",
      "fps: 0.7910939164496377\n",
      "TIMESTEP 54202 / STATE explore / EPSILON 0.04595310099991742 / ACTION 3 / REWARD 0 / Q_MAX  144.38763 / Loss  84.31995391845703\n",
      "fps: 0.8018787916329229\n",
      "TIMESTEP 54203 / STATE explore / EPSILON 0.04595210199991742 / ACTION 3 / REWARD 0 / Q_MAX  82.39547 / Loss  32.76521301269531\n",
      "fps: 0.7861185352340754\n",
      "TIMESTEP 54204 / STATE explore / EPSILON 0.04595110299991742 / ACTION 2 / REWARD 0 / Q_MAX  37.118927 / Loss  457.3143310546875\n",
      "fps: 0.793605449256167\n",
      "TIMESTEP 54205 / STATE explore / EPSILON 0.04595010399991742 / ACTION 1 / REWARD 0 / Q_MAX  142.91132 / Loss  29.67478370666504\n",
      "fps: 0.7929758848063863\n",
      "TIMESTEP 54206 / STATE explore / EPSILON 0.04594910499991742 / ACTION 2 / REWARD 0 / Q_MAX  85.66716 / Loss  53.09409713745117\n",
      "fps: 0.791720494976318\n",
      "TIMESTEP 54207 / STATE explore / EPSILON 0.045948105999917416 / ACTION 2 / REWARD 0 / Q_MAX  131.95383 / Loss  33.865055084228516\n",
      "fps: 0.8136230273156945\n",
      "TIMESTEP 54208 / STATE explore / EPSILON 0.045947106999917414 / ACTION 1 / REWARD 0 / Q_MAX  157.95752 / Loss  141.21240234375\n",
      "fps: 0.8018786383275178\n",
      "TIMESTEP 54209 / STATE explore / EPSILON 0.04594610799991741 / ACTION 2 / REWARD 0 / Q_MAX  157.1942 / Loss  140.99252319335938\n",
      "fps: 0.8012361907094285\n",
      "TIMESTEP 54210 / STATE explore / EPSILON 0.04594510899991741 / ACTION 2 / REWARD 0 / Q_MAX  153.8153 / Loss  66.00606536865234\n",
      "----------Random Action----------\n",
      "fps: 0.7842687906083047\n",
      "TIMESTEP 54211 / STATE explore / EPSILON 0.04594410999991741 / ACTION 0 / REWARD 0 / Q_MAX  142.18398 / Loss  71.79434967041016\n",
      "fps: 0.814285500730843\n",
      "TIMESTEP 54212 / STATE explore / EPSILON 0.04594311099991741 / ACTION 1 / REWARD 0 / Q_MAX  144.06873 / Loss  334.5823974609375\n",
      "fps: 0.8025224241308694\n",
      "TIMESTEP 54213 / STATE explore / EPSILON 0.04594211199991741 / ACTION 1 / REWARD 0 / Q_MAX  153.10341 / Loss  163.95721435546875\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fps: 0.7942357133619053\n",
      "TIMESTEP 54214 / STATE explore / EPSILON 0.045941112999917405 / ACTION 0 / REWARD 0 / Q_MAX  148.79825 / Loss  123.35508728027344\n",
      "fps: 0.7910940656590661\n",
      "TIMESTEP 54215 / STATE explore / EPSILON 0.045940113999917404 / ACTION 1 / REWARD 0 / Q_MAX  163.56215 / Loss  114.16903686523438\n",
      "fps: 0.8077079875451751\n",
      "TIMESTEP 54216 / STATE explore / EPSILON 0.0459391149999174 / ACTION 1 / REWARD 0 / Q_MAX  67.611786 / Loss  96.5322494506836\n",
      "fps: 0.8031669368908586\n",
      "TIMESTEP 54217 / STATE explore / EPSILON 0.0459381159999174 / ACTION 3 / REWARD 0 / Q_MAX  127.69089 / Loss  77.70310974121094\n",
      "fps: 0.7936052990978834\n",
      "TIMESTEP 54218 / STATE explore / EPSILON 0.0459371169999174 / ACTION 1 / REWARD 0 / Q_MAX  124.56328 / Loss  25.59928321838379\n",
      "fps: 0.8038123316648824\n",
      "TIMESTEP 54219 / STATE explore / EPSILON 0.0459361179999174 / ACTION 3 / REWARD -0.1 / Q_MAX  131.18636 / Loss  352.692138671875\n",
      "fps: 0.8123013090195126\n",
      "TIMESTEP 54220 / STATE explore / EPSILON 0.045935118999917396 / ACTION 3 / REWARD -0.1 / Q_MAX  162.21907 / Loss  181.78201293945312\n",
      "fps: 0.8038126397562904\n",
      "TIMESTEP 54221 / STATE explore / EPSILON 0.045934119999917394 / ACTION 3 / REWARD -0.1 / Q_MAX  145.62907 / Loss  71.09884643554688\n",
      "fps: 0.7917203455304925\n",
      "TIMESTEP 54222 / STATE explore / EPSILON 0.04593312099991739 / ACTION 0 / REWARD -0.1 / Q_MAX  127.17889 / Loss  158.69189453125\n",
      "fps: 0.7942357133619053\n",
      "TIMESTEP 54223 / STATE explore / EPSILON 0.04593212199991739 / ACTION 3 / REWARD 0 / Q_MAX  24.39385 / Loss  165.38967895507812\n",
      "fps: 0.793605449256167\n",
      "TIMESTEP 54224 / STATE explore / EPSILON 0.04593112299991739 / ACTION 1 / REWARD 0 / Q_MAX  145.17987 / Loss  143.67718505859375\n",
      "fps: 0.8012363437692821\n",
      "TIMESTEP 54225 / STATE explore / EPSILON 0.04593012399991739 / ACTION 1 / REWARD 0 / Q_MAX  149.79851 / Loss  51.83440399169922\n",
      "fps: 0.7873565041652345\n",
      "TIMESTEP 54226 / STATE explore / EPSILON 0.04592912499991739 / ACTION 0 / REWARD 0 / Q_MAX  142.29259 / Loss  64.60466003417969\n",
      "fps: 0.7751494096613504\n",
      "TIMESTEP 54227 / STATE explore / EPSILON 0.045928125999917385 / ACTION 3 / REWARD 0 / Q_MAX  140.99399 / Loss  217.57366943359375\n",
      "fps: 0.7942357133619053\n",
      "TIMESTEP 54228 / STATE explore / EPSILON 0.045927126999917384 / ACTION 3 / REWARD 0 / Q_MAX  123.09256 / Loss  66.777099609375\n",
      "fps: 0.793605449256167\n",
      "TIMESTEP 54229 / STATE explore / EPSILON 0.04592612799991738 / ACTION 3 / REWARD 0 / Q_MAX  137.18492 / Loss  87.64883422851562\n",
      "fps: 0.7691867842235344\n",
      "TIMESTEP 54230 / STATE explore / EPSILON 0.04592512899991738 / ACTION 3 / REWARD -0.1 / Q_MAX  71.49117 / Loss  61.175376892089844\n",
      "fps: 0.7569590427129294\n",
      "TIMESTEP 54231 / STATE explore / EPSILON 0.04592412999991738 / ACTION 0 / REWARD -0.1 / Q_MAX  148.88062 / Loss  106.59945678710938\n",
      "fps: 0.7923476177927143\n",
      "TIMESTEP 54232 / STATE explore / EPSILON 0.04592313099991738 / ACTION 3 / REWARD 0 / Q_MAX  142.76346 / Loss  358.6374816894531\n",
      "fps: 0.774548918895611\n",
      "TIMESTEP 54233 / STATE explore / EPSILON 0.045922131999917376 / ACTION 1 / REWARD 0 / Q_MAX  133.20314 / Loss  104.48126220703125\n",
      "fps: 0.7942357133619053\n",
      "TIMESTEP 54234 / STATE explore / EPSILON 0.045921132999917375 / ACTION 1 / REWARD 0 / Q_MAX  133.91444 / Loss  46.214988708496094\n",
      "fps: 0.8064055568351665\n",
      "TIMESTEP 54235 / STATE explore / EPSILON 0.04592013399991737 / ACTION 0 / REWARD 0 / Q_MAX  107.54791 / Loss  55.71809005737305\n",
      "fps: 0.8064055568351665\n",
      "TIMESTEP 54236 / STATE explore / EPSILON 0.04591913499991737 / ACTION 1 / REWARD 0 / Q_MAX  147.24254 / Loss  21.8790283203125\n",
      "fps: 0.8038124857105569\n",
      "TIMESTEP 54237 / STATE explore / EPSILON 0.04591813599991737 / ACTION 0 / REWARD 0 / Q_MAX  139.8176 / Loss  86.03140258789062\n",
      "fps: 0.81494921217571\n",
      "TIMESTEP 54238 / STATE explore / EPSILON 0.04591713699991737 / ACTION 1 / REWARD 0 / Q_MAX  152.17464 / Loss  15.310588836669922\n",
      "fps: 0.7929760347265811\n",
      "TIMESTEP 54239 / STATE explore / EPSILON 0.04591613799991737 / ACTION 3 / REWARD 0 / Q_MAX  -31.759953 / Loss  81.98091125488281\n",
      "fps: 0.8038124857105569\n",
      "TIMESTEP 54240 / STATE explore / EPSILON 0.045915138999917365 / ACTION 0 / REWARD 0 / Q_MAX  144.75371 / Loss  77.72727966308594\n",
      "fps: 0.7917203455304925\n",
      "TIMESTEP 54241 / STATE explore / EPSILON 0.045914139999917364 / ACTION 3 / REWARD 0 / Q_MAX  116.77071 / Loss  47.072662353515625\n",
      "fps: 0.8136231851446356\n",
      "TIMESTEP 54242 / STATE explore / EPSILON 0.04591314099991736 / ACTION 3 / REWARD 0 / Q_MAX  148.10791 / Loss  124.51884460449219\n",
      "fps: 0.7830405047538166\n",
      "TIMESTEP 54243 / STATE explore / EPSILON 0.04591214199991736 / ACTION 3 / REWARD 0 / Q_MAX  107.742645 / Loss  195.80149841308594\n",
      "fps: 0.8116419018367631\n",
      "TIMESTEP 54244 / STATE explore / EPSILON 0.04591114299991736 / ACTION 0 / REWARD -0.1 / Q_MAX  143.30801 / Loss  52.07306671142578\n",
      "fps: 0.8038124857105569\n",
      "TIMESTEP 54245 / STATE explore / EPSILON 0.04591014399991736 / ACTION 1 / REWARD 0 / Q_MAX  152.14479 / Loss  76.11585998535156\n",
      "fps: 0.7855009359218901\n",
      "TIMESTEP 54246 / STATE explore / EPSILON 0.045909144999917356 / ACTION 3 / REWARD 0 / Q_MAX  153.8206 / Loss  75.87907409667969\n",
      "fps: 0.7954994004789341\n",
      "TIMESTEP 54247 / STATE explore / EPSILON 0.045908145999917355 / ACTION 0 / REWARD 0 / Q_MAX  149.15726 / Loss  23.446632385253906\n",
      "fps: 0.7929760347265811\n",
      "TIMESTEP 54248 / STATE explore / EPSILON 0.04590714699991735 / ACTION 3 / REWARD 0 / Q_MAX  146.80884 / Loss  308.4975891113281\n",
      "fps: 0.7923477674754426\n",
      "TIMESTEP 54249 / STATE explore / EPSILON 0.04590614799991735 / ACTION 0 / REWARD 0 / Q_MAX  81.48831 / Loss  135.2552032470703\n",
      "fps: 0.7507078765992999\n",
      "TIMESTEP 54250 / STATE explore / EPSILON 0.04590514899991735 / ACTION 1 / REWARD -0.1 / Q_MAX  125.59898 / Loss  60.8355827331543\n",
      "fps: 0.7490208431999051\n",
      "TIMESTEP 54251 / STATE explore / EPSILON 0.04590414999991735 / ACTION 1 / REWARD 46.0 / Q_MAX  153.2893 / Loss  69.35044860839844\n",
      "fps: 0.7999542262520074\n",
      "TIMESTEP 54252 / STATE explore / EPSILON 0.04590315099991735 / ACTION 2 / REWARD 0 / Q_MAX  131.78006 / Loss  123.65406799316406\n",
      "fps: 0.7668271999526115\n",
      "TIMESTEP 54253 / STATE explore / EPSILON 0.045902151999917346 / ACTION 2 / REWARD 0 / Q_MAX  163.39896 / Loss  99.73100280761719\n",
      "fps: 0.7662396691195277\n",
      "TIMESTEP 54254 / STATE explore / EPSILON 0.045901152999917344 / ACTION 0 / REWARD 0 / Q_MAX  143.98412 / Loss  64.075927734375\n",
      "fps: 0.7650673042616669\n",
      "TIMESTEP 54255 / STATE explore / EPSILON 0.04590015399991734 / ACTION 1 / REWARD 0 / Q_MAX  154.7648 / Loss  52.93479919433594\n",
      "fps: 0.8038126397562904\n",
      "TIMESTEP 54256 / STATE explore / EPSILON 0.04589915499991734 / ACTION 3 / REWARD 0 / Q_MAX  140.05692 / Loss  81.60385131835938\n",
      "fps: 0.8012361907094285\n",
      "TIMESTEP 54257 / STATE explore / EPSILON 0.04589815599991734 / ACTION 2 / REWARD -0.1 / Q_MAX  57.285564 / Loss  464.3692932128906\n",
      "----------Random Action----------\n",
      "error:  could not convert string to float: 'm' \n",
      "score:  m\n",
      "fps: 0.7524022566843961\n",
      "TIMESTEP 54258 / STATE explore / EPSILON 0.04589715699991734 / ACTION 2 / REWARD 4.0 / Q_MAX  166.45953 / Loss  42.81214141845703\n",
      "fps: 0.8096704007771037\n",
      "TIMESTEP 54259 / STATE explore / EPSILON 0.045896157999917336 / ACTION 1 / REWARD 0 / Q_MAX  52.119026 / Loss  94.01304626464844\n",
      "fps: 0.8031670906892604\n",
      "TIMESTEP 54260 / STATE explore / EPSILON 0.045895158999917335 / ACTION 1 / REWARD 0 / Q_MAX  172.35742 / Loss  76.74613952636719\n",
      "fps: 0.793605449256167\n",
      "TIMESTEP 54261 / STATE explore / EPSILON 0.04589415999991733 / ACTION 1 / REWARD 0 / Q_MAX  166.24648 / Loss  75.68846130371094\n",
      "fps: 0.8038126397562904\n",
      "TIMESTEP 54262 / STATE explore / EPSILON 0.04589316099991733 / ACTION 0 / REWARD 0 / Q_MAX  138.07472 / Loss  475.0350341796875\n",
      "fps: 0.8038127938020828\n",
      "TIMESTEP 54263 / STATE explore / EPSILON 0.04589216199991733 / ACTION 2 / REWARD -0.1 / Q_MAX  128.02905 / Loss  44.1204833984375\n",
      "fps: 0.7861186825725772\n",
      "TIMESTEP 54264 / STATE explore / EPSILON 0.04589116299991733 / ACTION 1 / REWARD 0 / Q_MAX  147.59782 / Loss  47.07454299926758\n",
      "fps: 0.7942358637588455\n",
      "TIMESTEP 54265 / STATE explore / EPSILON 0.04589016399991733 / ACTION 2 / REWARD 0 / Q_MAX  159.55972 / Loss  131.5897674560547\n",
      "fps: 0.8038126397562904\n",
      "TIMESTEP 54266 / STATE explore / EPSILON 0.045889164999917326 / ACTION 2 / REWARD 0 / Q_MAX  153.36241 / Loss  118.32051086425781\n",
      "fps: 0.7546736354064183\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TIMESTEP 54267 / STATE explore / EPSILON 0.045888165999917324 / ACTION 1 / REWARD 0 / Q_MAX  141.29396 / Loss  59.83694076538086\n",
      "fps: 0.7451139858275166\n",
      "TIMESTEP 54268 / STATE explore / EPSILON 0.04588716699991732 / ACTION 2 / REWARD 0 / Q_MAX  157.53294 / Loss  100.49368286132812\n",
      "fps: 0.8038126397562904\n",
      "TIMESTEP 54269 / STATE explore / EPSILON 0.04588616799991732 / ACTION 1 / REWARD 0 / Q_MAX  31.538364 / Loss  65.34970092773438\n",
      "fps: 0.8012361907094285\n",
      "TIMESTEP 54270 / STATE explore / EPSILON 0.04588516899991732 / ACTION 0 / REWARD 0 / Q_MAX  158.72818 / Loss  143.75979614257812\n",
      "fps: 0.7861186825725772\n",
      "TIMESTEP 54271 / STATE explore / EPSILON 0.04588416999991732 / ACTION 1 / REWARD 0 / Q_MAX  145.69215 / Loss  75.6873779296875\n",
      "fps: 0.8064055568351665\n",
      "TIMESTEP 54272 / STATE explore / EPSILON 0.045883170999917317 / ACTION 2 / REWARD 0 / Q_MAX  143.58998 / Loss  75.15202331542969\n",
      "fps: 0.8038126397562904\n",
      "TIMESTEP 54273 / STATE explore / EPSILON 0.045882171999917315 / ACTION 2 / REWARD 0 / Q_MAX  153.46983 / Loss  140.02420043945312\n",
      "fps: 0.7751494096613504\n",
      "TIMESTEP 54274 / STATE explore / EPSILON 0.04588117299991731 / ACTION 1 / REWARD 0 / Q_MAX  124.93103 / Loss  42.975128173828125\n",
      "fps: 0.7842687906083047\n",
      "TIMESTEP 54275 / STATE explore / EPSILON 0.04588017399991731 / ACTION 1 / REWARD 0 / Q_MAX  176.09834 / Loss  97.44981384277344\n",
      "----------Random Action----------\n",
      "fps: 0.8012363437692821\n",
      "TIMESTEP 54276 / STATE explore / EPSILON 0.04587917499991731 / ACTION 1 / REWARD 0 / Q_MAX  -5.749024 / Loss  119.14030456542969\n",
      "fps: 0.7917203455304925\n",
      "TIMESTEP 54277 / STATE explore / EPSILON 0.04587817599991731 / ACTION 3 / REWARD -0.1 / Q_MAX  176.00502 / Loss  223.27108764648438\n",
      "fps: 0.7967669638844153\n",
      "TIMESTEP 54278 / STATE explore / EPSILON 0.04587717699991731 / ACTION 2 / REWARD 0 / Q_MAX  160.00046 / Loss  110.15763092041016\n",
      "fps: 0.8038123316648824\n",
      "TIMESTEP 54279 / STATE explore / EPSILON 0.045876177999917306 / ACTION 2 / REWARD 0 / Q_MAX  189.7438 / Loss  173.6501922607422\n",
      "fps: 0.8038124857105569\n",
      "TIMESTEP 54280 / STATE explore / EPSILON 0.045875178999917304 / ACTION 2 / REWARD 0 / Q_MAX  139.72351 / Loss  86.36273956298828\n",
      "fps: 0.8044590730901859\n",
      "TIMESTEP 54281 / STATE explore / EPSILON 0.0458741799999173 / ACTION 2 / REWARD 0 / Q_MAX  149.55061 / Loss  277.01025390625\n",
      "fps: 0.8012361907094285\n",
      "TIMESTEP 54282 / STATE explore / EPSILON 0.0458731809999173 / ACTION 3 / REWARD 0 / Q_MAX  158.56116 / Loss  69.58289337158203\n",
      "fps: 0.8012361907094285\n",
      "TIMESTEP 54283 / STATE explore / EPSILON 0.0458721819999173 / ACTION 1 / REWARD 0 / Q_MAX  186.63281 / Loss  685.9929809570312\n",
      "fps: 0.8038126397562904\n",
      "TIMESTEP 54284 / STATE explore / EPSILON 0.0458711829999173 / ACTION 3 / REWARD 0 / Q_MAX  162.41553 / Loss  86.80204772949219\n",
      "fps: 0.7936052990978834\n",
      "TIMESTEP 54285 / STATE explore / EPSILON 0.0458701839999173 / ACTION 2 / REWARD 0 / Q_MAX  152.70169 / Loss  195.8697967529297\n",
      "fps: 0.7948669793472747\n",
      "TIMESTEP 54286 / STATE explore / EPSILON 0.045869184999917295 / ACTION 3 / REWARD 0 / Q_MAX  128.66054 / Loss  46.39569854736328\n",
      "fps: 0.7830406509407755\n",
      "TIMESTEP 54287 / STATE explore / EPSILON 0.045868185999917294 / ACTION 1 / REWARD 0 / Q_MAX  153.17207 / Loss  110.29862213134766\n",
      "----------Random Action----------\n",
      "fps: 0.7836542395909898\n",
      "TIMESTEP 54288 / STATE explore / EPSILON 0.04586718699991729 / ACTION 3 / REWARD 0 / Q_MAX  154.7781 / Loss  35.970367431640625\n",
      "fps: 0.8057556831531081\n",
      "TIMESTEP 54289 / STATE explore / EPSILON 0.04586618799991729 / ACTION 1 / REWARD 0 / Q_MAX  179.93663 / Loss  411.84564208984375\n",
      "fps: 0.8051070106224577\n",
      "TIMESTEP 54290 / STATE explore / EPSILON 0.04586518899991729 / ACTION 3 / REWARD 0 / Q_MAX  33.515648 / Loss  468.07672119140625\n",
      "fps: 0.7967671152415213\n",
      "TIMESTEP 54291 / STATE explore / EPSILON 0.04586418999991729 / ACTION 1 / REWARD 0 / Q_MAX  145.08846 / Loss  36.75308609008789\n",
      "fps: 0.8044590730901859\n",
      "TIMESTEP 54292 / STATE explore / EPSILON 0.045863190999917286 / ACTION 1 / REWARD 0 / Q_MAX  166.655 / Loss  55.51038360595703\n",
      "fps: 0.7842689372542442\n",
      "TIMESTEP 54293 / STATE explore / EPSILON 0.045862191999917284 / ACTION 2 / REWARD 0 / Q_MAX  139.7539 / Loss  123.45585632324219\n",
      "error:  could not convert string to float:  \n",
      "score:  \n",
      "fps: 0.7261748714959279\n",
      "TIMESTEP 54294 / STATE explore / EPSILON 0.04586119299991728 / ACTION 1 / REWARD 4.0 / Q_MAX  80.95575 / Loss  97.8243408203125\n",
      "fps: 0.7598349281343183\n",
      "TIMESTEP 54295 / STATE explore / EPSILON 0.04586019399991728 / ACTION 3 / REWARD 0 / Q_MAX  130.42491 / Loss  86.66888427734375\n",
      "fps: 0.8116419018367631\n",
      "TIMESTEP 54296 / STATE explore / EPSILON 0.04585919499991728 / ACTION 1 / REWARD 0 / Q_MAX  136.44618 / Loss  169.3982696533203\n",
      "fps: 0.7861186825725772\n",
      "TIMESTEP 54297 / STATE explore / EPSILON 0.04585819599991728 / ACTION 0 / REWARD 0 / Q_MAX  86.64822 / Loss  89.81256103515625\n",
      "fps: 0.7967672665986848\n",
      "TIMESTEP 54298 / STATE explore / EPSILON 0.04585719699991728 / ACTION 1 / REWARD 0 / Q_MAX  144.21071 / Loss  57.48189163208008\n",
      "fps: 0.8070564796608223\n",
      "TIMESTEP 54299 / STATE explore / EPSILON 0.045856197999917275 / ACTION 2 / REWARD 0 / Q_MAX  146.74562 / Loss  164.09457397460938\n",
      "fps: 0.7691867842235344\n",
      "TIMESTEP 54300 / STATE explore / EPSILON 0.045855198999917274 / ACTION 0 / REWARD 0 / Q_MAX  163.48521 / Loss  59.03850555419922\n",
      "fps: 0.7967671152415213\n",
      "TIMESTEP 54301 / STATE explore / EPSILON 0.04585419999991727 / ACTION 2 / REWARD 0 / Q_MAX  157.2116 / Loss  119.44145965576172\n",
      "fps: 0.7739496434074197\n",
      "TIMESTEP 54302 / STATE explore / EPSILON 0.04585320099991727 / ACTION 2 / REWARD 0 / Q_MAX  157.8648 / Loss  135.3351593017578\n",
      "fps: 0.7781655955142024\n",
      "TIMESTEP 54303 / STATE explore / EPSILON 0.04585220199991727 / ACTION 1 / REWARD 0 / Q_MAX  154.84206 / Loss  163.8477783203125\n",
      "fps: 0.7910940656590661\n",
      "TIMESTEP 54304 / STATE explore / EPSILON 0.04585120299991727 / ACTION 1 / REWARD 0 / Q_MAX  160.73056 / Loss  139.32040405273438\n",
      "fps: 0.7685956334691258\n",
      "TIMESTEP 54305 / STATE explore / EPSILON 0.045850203999917266 / ACTION 0 / REWARD 0 / Q_MAX  148.12167 / Loss  264.9018249511719\n",
      "fps: 0.7685954926259395\n",
      "TIMESTEP 54306 / STATE explore / EPSILON 0.045849204999917265 / ACTION 2 / REWARD 0 / Q_MAX  160.97356 / Loss  150.8953857421875\n",
      "fps: 0.7936052990978834\n",
      "TIMESTEP 54307 / STATE explore / EPSILON 0.04584820599991726 / ACTION 2 / REWARD 0 / Q_MAX  155.08789 / Loss  123.70880126953125\n",
      "fps: 0.7721566572789729\n",
      "TIMESTEP 54308 / STATE explore / EPSILON 0.04584720699991726 / ACTION 1 / REWARD 0 / Q_MAX  146.7157 / Loss  92.85401916503906\n",
      "fps: 0.7836542395909898\n",
      "TIMESTEP 54309 / STATE explore / EPSILON 0.04584620799991726 / ACTION 1 / REWARD 0 / Q_MAX  108.21313 / Loss  336.5733947753906\n",
      "fps: 0.7818160602532572\n",
      "TIMESTEP 54310 / STATE explore / EPSILON 0.04584520899991726 / ACTION 2 / REWARD 0 / Q_MAX  170.24947 / Loss  53.921749114990234\n",
      "fps: 0.798039028776602\n",
      "TIMESTEP 54311 / STATE explore / EPSILON 0.04584420999991726 / ACTION 2 / REWARD 0 / Q_MAX  133.13588 / Loss  87.30955505371094\n",
      "fps: 0.7910939164496377\n",
      "TIMESTEP 54312 / STATE explore / EPSILON 0.045843210999917255 / ACTION 2 / REWARD 0 / Q_MAX  142.59752 / Loss  82.84185791015625\n",
      "fps: 0.7961328287601405\n",
      "TIMESTEP 54313 / STATE explore / EPSILON 0.045842211999917254 / ACTION 0 / REWARD 0 / Q_MAX  153.28075 / Loss  68.55921936035156\n",
      "fps: 0.8038126397562904\n",
      "TIMESTEP 54314 / STATE explore / EPSILON 0.04584121299991725 / ACTION 2 / REWARD 0 / Q_MAX  145.27951 / Loss  73.50909423828125\n",
      "fps: 0.7948669793472747\n",
      "TIMESTEP 54315 / STATE explore / EPSILON 0.04584021399991725 / ACTION 2 / REWARD 0 / Q_MAX  147.81865 / Loss  55.81941223144531\n",
      "fps: 0.7685954926259395\n",
      "TIMESTEP 54316 / STATE explore / EPSILON 0.04583921499991725 / ACTION 0 / REWARD 10.0 / Q_MAX  44.845894 / Loss  38.591758728027344\n",
      "error:  could not convert string to float:  \n",
      "score:  \n",
      "fps: 0.8424119031364815\n",
      "TIMESTEP 54317 / STATE explore / EPSILON 0.04583821599991725 / ACTION 1 / REWARD 0 / Q_MAX  143.84407 / Loss  75.52304077148438\n",
      "fps: 0.7967672665986848\n",
      "TIMESTEP 54318 / STATE explore / EPSILON 0.045837216999917246 / ACTION 3 / REWARD 0 / Q_MAX  145.0177 / Loss  125.66995239257812\n",
      "fps: 0.7490207094394234\n",
      "TIMESTEP 54319 / STATE explore / EPSILON 0.045836217999917245 / ACTION 0 / REWARD 9.0 / Q_MAX  143.45757 / Loss  35.04136276245117\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fps: 0.8196251805624167\n",
      "TIMESTEP 54320 / STATE explore / EPSILON 0.04583521899991724 / ACTION 3 / REWARD 0 / Q_MAX  150.17517 / Loss  414.3126220703125\n",
      "fps: 0.8044590730901859\n",
      "TIMESTEP 54321 / STATE explore / EPSILON 0.04583421999991724 / ACTION 2 / REWARD 0 / Q_MAX  118.42811 / Loss  26.24948501586914\n",
      "fps: 0.7341723716687417\n",
      "TIMESTEP 54322 / STATE explore / EPSILON 0.04583322099991724 / ACTION 1 / REWARD 10.0 / Q_MAX  148.02876 / Loss  216.25595092773438\n",
      "----------Random Action----------\n",
      "fps: 0.8031669368908586\n",
      "TIMESTEP 54323 / STATE explore / EPSILON 0.04583222199991724 / ACTION 2 / REWARD 0 / Q_MAX  132.17444 / Loss  259.4957275390625\n",
      "fps: 0.8044592273838481\n",
      "TIMESTEP 54324 / STATE explore / EPSILON 0.04583122299991724 / ACTION 1 / REWARD 0 / Q_MAX  148.83691 / Loss  44.651695251464844\n",
      "fps: 0.7867371064841083\n",
      "TIMESTEP 54325 / STATE explore / EPSILON 0.045830223999917236 / ACTION 0 / REWARD 0 / Q_MAX  125.955345 / Loss  41.77540588378906\n",
      "fps: 0.8038126397562904\n",
      "TIMESTEP 54326 / STATE explore / EPSILON 0.045829224999917234 / ACTION 1 / REWARD 0 / Q_MAX  125.70385 / Loss  174.5050506591797\n",
      "fps: 0.8051070106224577\n",
      "TIMESTEP 54327 / STATE explore / EPSILON 0.04582822599991723 / ACTION 2 / REWARD 0 / Q_MAX  123.52183 / Loss  74.84658813476562\n",
      "fps: 0.8038126397562904\n",
      "TIMESTEP 54328 / STATE explore / EPSILON 0.04582722699991723 / ACTION 1 / REWARD 0 / Q_MAX  156.76416 / Loss  43.832054138183594\n",
      "fps: 0.7936055994145076\n",
      "TIMESTEP 54329 / STATE explore / EPSILON 0.04582622799991723 / ACTION 1 / REWARD 0 / Q_MAX  129.60728 / Loss  52.08844757080078\n",
      "fps: 0.7781655955142024\n",
      "TIMESTEP 54330 / STATE explore / EPSILON 0.04582522899991723 / ACTION 2 / REWARD 0 / Q_MAX  60.61886 / Loss  73.41383361816406\n",
      "----------Random Action----------\n",
      "fps: 0.8038124857105569\n",
      "TIMESTEP 54331 / STATE explore / EPSILON 0.045824229999917226 / ACTION 3 / REWARD 0 / Q_MAX  159.91357 / Loss  195.2216796875\n",
      "fps: 0.8123011517029974\n",
      "TIMESTEP 54332 / STATE explore / EPSILON 0.045823230999917225 / ACTION 3 / REWARD 0 / Q_MAX  137.54025 / Loss  210.38153076171875\n",
      "fps: 0.8149493705195848\n",
      "TIMESTEP 54333 / STATE explore / EPSILON 0.04582223199991722 / ACTION 0 / REWARD 0 / Q_MAX  133.1855 / Loss  362.823974609375\n",
      "fps: 0.8038126397562904\n",
      "TIMESTEP 54334 / STATE explore / EPSILON 0.04582123299991722 / ACTION 0 / REWARD 0 / Q_MAX  138.85886 / Loss  175.66371154785156\n",
      "----------Random Action----------\n",
      "fps: 0.8044593816775695\n",
      "TIMESTEP 54335 / STATE explore / EPSILON 0.04582023399991722 / ACTION 0 / REWARD 0 / Q_MAX  141.15451 / Loss  77.61170959472656\n",
      "----------Random Action----------\n",
      "fps: 0.8156140064649351\n",
      "TIMESTEP 54336 / STATE explore / EPSILON 0.04581923499991722 / ACTION 3 / REWARD 0 / Q_MAX  132.78027 / Loss  77.99504852294922\n",
      "fps: 0.7581068079527846\n",
      "TIMESTEP 54337 / STATE explore / EPSILON 0.04581823599991722 / ACTION 0 / REWARD 1.0 / Q_MAX  125.06315 / Loss  51.128292083740234\n",
      "fps: 0.7974027164100391\n",
      "TIMESTEP 54338 / STATE explore / EPSILON 0.045817236999917216 / ACTION 3 / REWARD 0 / Q_MAX  130.84962 / Loss  75.35893249511719\n",
      "fps: 0.8038126397562904\n",
      "TIMESTEP 54339 / STATE explore / EPSILON 0.045816237999917214 / ACTION 2 / REWARD 0 / Q_MAX  150.70216 / Loss  39.434654235839844\n",
      "fps: 0.7569590427129294\n",
      "TIMESTEP 54340 / STATE explore / EPSILON 0.04581523899991721 / ACTION 2 / REWARD 11.0 / Q_MAX  141.0301 / Loss  60.66637420654297\n",
      "fps: 0.8057555283617299\n",
      "TIMESTEP 54341 / STATE explore / EPSILON 0.04581423999991721 / ACTION 2 / REWARD 0 / Q_MAX  24.776567 / Loss  55.89305877685547\n",
      "fps: 0.7942357133619053\n",
      "TIMESTEP 54342 / STATE explore / EPSILON 0.04581324099991721 / ACTION 2 / REWARD 0 / Q_MAX  142.81322 / Loss  124.97993469238281\n",
      "----------Random Action----------\n",
      "fps: 0.7546739069806775\n",
      "TIMESTEP 54343 / STATE explore / EPSILON 0.04581224199991721 / ACTION 1 / REWARD 10.0 / Q_MAX  148.7132 / Loss  106.32051086425781\n",
      "error:  could not convert string to float:  \n",
      "score:  \n",
      "fps: 0.7325588868985907\n",
      "TIMESTEP 54344 / STATE explore / EPSILON 0.04581124299991721 / ACTION 0 / REWARD 2.0 / Q_MAX  93.85765 / Loss  455.7868957519531\n",
      "fps: 0.8077082986301344\n",
      "TIMESTEP 54345 / STATE explore / EPSILON 0.045810243999917205 / ACTION 1 / REWARD 0 / Q_MAX  140.29875 / Loss  57.07514953613281\n",
      "fps: 0.8038124857105569\n",
      "TIMESTEP 54346 / STATE explore / EPSILON 0.045809244999917204 / ACTION 1 / REWARD 0 / Q_MAX  126.92015 / Loss  114.75534057617188\n",
      "fps: 0.781816205983387\n",
      "TIMESTEP 54347 / STATE explore / EPSILON 0.0458082459999172 / ACTION 1 / REWARD 0 / Q_MAX  116.19782 / Loss  43.90004348754883\n",
      "fps: 0.7948669793472747\n",
      "TIMESTEP 54348 / STATE explore / EPSILON 0.0458072469999172 / ACTION 0 / REWARD -0.1 / Q_MAX  142.90771 / Loss  47.939781188964844\n",
      "fps: 0.8038124857105569\n",
      "TIMESTEP 54349 / STATE explore / EPSILON 0.0458062479999172 / ACTION 2 / REWARD -0.1 / Q_MAX  134.40483 / Loss  73.21473693847656\n",
      "fps: 0.7563863941829193\n",
      "TIMESTEP 54350 / STATE explore / EPSILON 0.0458052489999172 / ACTION 2 / REWARD 0 / Q_MAX  145.19182 / Loss  58.459877014160156\n",
      "fps: 0.7842687906083047\n",
      "TIMESTEP 54351 / STATE explore / EPSILON 0.045804249999917196 / ACTION 0 / REWARD 0 / Q_MAX  50.875446 / Loss  174.29287719726562\n",
      "fps: 0.781816205983387\n",
      "TIMESTEP 54352 / STATE explore / EPSILON 0.045803250999917194 / ACTION 1 / REWARD 0 / Q_MAX  116.03165 / Loss  109.1911849975586\n",
      "fps: 0.7910939164496377\n",
      "TIMESTEP 54353 / STATE explore / EPSILON 0.04580225199991719 / ACTION 3 / REWARD 0 / Q_MAX  161.58197 / Loss  112.97624206542969\n",
      "fps: 0.7917203455304925\n",
      "TIMESTEP 54354 / STATE explore / EPSILON 0.04580125299991719 / ACTION 3 / REWARD 0 / Q_MAX  149.09824 / Loss  110.40019226074219\n",
      "fps: 0.7775605684864603\n",
      "TIMESTEP 54355 / STATE explore / EPSILON 0.04580025399991719 / ACTION 1 / REWARD 0 / Q_MAX  140.72426 / Loss  294.14544677734375\n",
      "fps: 0.7745490619291485\n",
      "TIMESTEP 54356 / STATE explore / EPSILON 0.04579925499991719 / ACTION 2 / REWARD 0 / Q_MAX  140.05215 / Loss  92.60733032226562\n",
      "fps: 0.7830405047538166\n",
      "TIMESTEP 54357 / STATE explore / EPSILON 0.04579825599991719 / ACTION 3 / REWARD -0.1 / Q_MAX  143.07428 / Loss  112.36003112792969\n",
      "fps: 0.8012364968291941\n",
      "TIMESTEP 54358 / STATE explore / EPSILON 0.045797256999917185 / ACTION 2 / REWARD -0.1 / Q_MAX  149.61884 / Loss  60.65373229980469\n",
      "fps: 0.7855009359218901\n",
      "TIMESTEP 54359 / STATE explore / EPSILON 0.045796257999917184 / ACTION 0 / REWARD 0 / Q_MAX  141.33948 / Loss  83.67703247070312\n",
      "fps: 0.7948669793472747\n",
      "TIMESTEP 54360 / STATE explore / EPSILON 0.04579525899991718 / ACTION 3 / REWARD 0 / Q_MAX  128.42511 / Loss  60.66669464111328\n",
      "fps: 0.7954994004789341\n",
      "TIMESTEP 54361 / STATE explore / EPSILON 0.04579425999991718 / ACTION 0 / REWARD 0 / Q_MAX  160.82109 / Loss  70.05412292480469\n",
      "fps: 0.793605449256167\n",
      "TIMESTEP 54362 / STATE explore / EPSILON 0.04579326099991718 / ACTION 1 / REWARD 0 / Q_MAX  146.6592 / Loss  485.62646484375\n",
      "fps: 0.8012361907094285\n",
      "TIMESTEP 54363 / STATE explore / EPSILON 0.04579226199991718 / ACTION 3 / REWARD 0 / Q_MAX  140.47823 / Loss  269.120849609375\n",
      "fps: 0.7948671299833837\n",
      "TIMESTEP 54364 / STATE explore / EPSILON 0.045791262999917176 / ACTION 1 / REWARD 0 / Q_MAX  136.93536 / Loss  68.71791076660156\n",
      "fps: 0.793605449256167\n",
      "TIMESTEP 54365 / STATE explore / EPSILON 0.045790263999917175 / ACTION 0 / REWARD 0 / Q_MAX  115.71441 / Loss  88.5657958984375\n",
      "fps: 0.8070563243692563\n",
      "TIMESTEP 54366 / STATE explore / EPSILON 0.04578926499991717 / ACTION 3 / REWARD 0 / Q_MAX  4.618036 / Loss  83.96729278564453\n",
      "fps: 0.8038124857105569\n",
      "TIMESTEP 54367 / STATE explore / EPSILON 0.04578826599991717 / ACTION 1 / REWARD 0 / Q_MAX  136.22243 / Loss  139.9374542236328\n",
      "fps: 0.7757508322441119\n",
      "TIMESTEP 54368 / STATE explore / EPSILON 0.04578726699991717 / ACTION 0 / REWARD 0 / Q_MAX  134.13281 / Loss  166.52783203125\n",
      "fps: 0.7842687906083047\n",
      "TIMESTEP 54369 / STATE explore / EPSILON 0.04578626799991717 / ACTION 3 / REWARD 0 / Q_MAX  160.97244 / Loss  56.61524963378906\n",
      "fps: 0.7923477674754426\n",
      "TIMESTEP 54370 / STATE explore / EPSILON 0.04578526899991717 / ACTION 3 / REWARD 0 / Q_MAX  82.01289 / Loss  66.40204620361328\n",
      "fps: 0.7685956334691258\n",
      "TIMESTEP 54371 / STATE explore / EPSILON 0.045784269999917165 / ACTION 3 / REWARD 0 / Q_MAX  163.9934 / Loss  358.4954528808594\n",
      "fps: 0.7745490619291485\n",
      "TIMESTEP 54372 / STATE explore / EPSILON 0.045783270999917164 / ACTION 3 / REWARD -0.1 / Q_MAX  101.09107 / Loss  85.11345672607422\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fps: 0.747900271499861\n",
      "TIMESTEP 54373 / STATE explore / EPSILON 0.04578227199991716 / ACTION 1 / REWARD -0.1 / Q_MAX  151.48274 / Loss  185.77804565429688\n",
      "fps: 0.7282905616633687\n",
      "TIMESTEP 54374 / STATE explore / EPSILON 0.04578127299991716 / ACTION 3 / REWARD 0 / Q_MAX  118.336975 / Loss  196.66757202148438\n",
      "fps: 0.7691867842235344\n",
      "TIMESTEP 54375 / STATE explore / EPSILON 0.04578027399991716 / ACTION 1 / REWARD 0 / Q_MAX  36.210537 / Loss  92.0478286743164\n",
      "fps: 0.7867371064841083\n",
      "TIMESTEP 54376 / STATE explore / EPSILON 0.04577927499991716 / ACTION 0 / REWARD 0 / Q_MAX  148.98177 / Loss  124.33802795410156\n",
      "fps: 0.7787715648293334\n",
      "TIMESTEP 54377 / STATE explore / EPSILON 0.045778275999917156 / ACTION 3 / REWARD 0 / Q_MAX  93.43536 / Loss  72.13337707519531\n",
      "fps: 0.7451138534587725\n",
      "TIMESTEP 54378 / STATE explore / EPSILON 0.045777276999917155 / ACTION 3 / REWARD 0 / Q_MAX  76.16971 / Loss  119.7055435180664\n",
      "fps: 0.7721565151276663\n",
      "TIMESTEP 54379 / STATE explore / EPSILON 0.04577627799991715 / ACTION 2 / REWARD 0 / Q_MAX  97.17608 / Loss  138.85421752929688\n",
      "fps: 0.7873565041652345\n",
      "TIMESTEP 54380 / STATE explore / EPSILON 0.04577527899991715 / ACTION 1 / REWARD 0 / Q_MAX  139.05583 / Loss  563.7313842773438\n",
      "fps: 0.7929760347265811\n",
      "TIMESTEP 54381 / STATE explore / EPSILON 0.04577427999991715 / ACTION 2 / REWARD 0 / Q_MAX  127.24344 / Loss  102.74528503417969\n",
      "fps: 0.8070561690777502\n",
      "TIMESTEP 54382 / STATE explore / EPSILON 0.04577328099991715 / ACTION 3 / REWARD 0 / Q_MAX  126.022285 / Loss  70.76570129394531\n",
      "fps: 0.781816205983387\n",
      "TIMESTEP 54383 / STATE explore / EPSILON 0.04577228199991715 / ACTION 3 / REWARD 0 / Q_MAX  138.57364 / Loss  78.99122619628906\n",
      "fps: 0.7830406509407755\n",
      "TIMESTEP 54384 / STATE explore / EPSILON 0.045771282999917146 / ACTION 0 / REWARD 0 / Q_MAX  134.6561 / Loss  100.74040985107422\n",
      "fps: 0.7598349281343183\n",
      "TIMESTEP 54385 / STATE explore / EPSILON 0.045770283999917144 / ACTION 3 / REWARD 0 / Q_MAX  127.7132 / Loss  78.7923812866211\n",
      "fps: 0.7775604243385098\n",
      "TIMESTEP 54386 / STATE explore / EPSILON 0.04576928499991714 / ACTION 0 / REWARD 0 / Q_MAX  120.069374 / Loss  57.200042724609375\n",
      "fps: 0.7873563563623694\n",
      "TIMESTEP 54387 / STATE explore / EPSILON 0.04576828599991714 / ACTION 0 / REWARD 0 / Q_MAX  119.429436 / Loss  114.58851623535156\n",
      "fps: 0.7336337970423442\n",
      "TIMESTEP 54388 / STATE explore / EPSILON 0.04576728699991714 / ACTION 0 / REWARD 8.0 / Q_MAX  71.500496 / Loss  179.07223510742188\n",
      "fps: 0.7910939164496377\n",
      "TIMESTEP 54389 / STATE explore / EPSILON 0.04576628799991714 / ACTION 2 / REWARD 0 / Q_MAX  149.76657 / Loss  44.398189544677734\n",
      "fps: 0.7775605684864603\n",
      "TIMESTEP 54390 / STATE explore / EPSILON 0.045765288999917136 / ACTION 0 / REWARD 0 / Q_MAX  136.49263 / Loss  146.15853881835938\n",
      "fps: 0.7374210173668172\n",
      "TIMESTEP 54391 / STATE explore / EPSILON 0.045764289999917135 / ACTION 0 / REWARD 10.0 / Q_MAX  139.46318 / Loss  444.962158203125\n",
      "fps: 0.7842687906083047\n",
      "TIMESTEP 54392 / STATE explore / EPSILON 0.04576329099991713 / ACTION 2 / REWARD 0 / Q_MAX  145.54034 / Loss  57.404754638671875\n",
      "fps: 0.7604128122266388\n",
      "TIMESTEP 54393 / STATE explore / EPSILON 0.04576229199991713 / ACTION 1 / REWARD 0 / Q_MAX  108.20646 / Loss  63.97924041748047\n",
      "fps: 0.7336337970423442\n",
      "TIMESTEP 54394 / STATE explore / EPSILON 0.04576129299991713 / ACTION 1 / REWARD 10.0 / Q_MAX  130.80202 / Loss  33.39192199707031\n",
      "fps: 0.790468626851596\n",
      "TIMESTEP 54395 / STATE explore / EPSILON 0.04576029399991713 / ACTION 0 / REWARD 0 / Q_MAX  111.909325 / Loss  62.2969856262207\n",
      "fps: 0.7961329798764166\n",
      "TIMESTEP 54396 / STATE explore / EPSILON 0.04575929499991713 / ACTION 3 / REWARD 0 / Q_MAX  120.388374 / Loss  98.33009338378906\n",
      "fps: 0.7830405047538166\n",
      "TIMESTEP 54397 / STATE explore / EPSILON 0.045758295999917126 / ACTION 1 / REWARD 0 / Q_MAX  72.01614 / Loss  58.63824462890625\n",
      "fps: 0.7721565151276663\n",
      "TIMESTEP 54398 / STATE explore / EPSILON 0.045757296999917124 / ACTION 1 / REWARD 0 / Q_MAX  108.63991 / Loss  29.01995849609375\n",
      "fps: 0.781816205983387\n",
      "TIMESTEP 54399 / STATE explore / EPSILON 0.04575629799991712 / ACTION 3 / REWARD 0 / Q_MAX  116.741486 / Loss  123.82544708251953\n",
      "fps: 0.7781655955142024\n",
      "TIMESTEP 54400 / STATE explore / EPSILON 0.04575529899991712 / ACTION 3 / REWARD 0 / Q_MAX  99.02061 / Loss  319.4634704589844\n",
      "fps: 0.7656528981439648\n",
      "TIMESTEP 54401 / STATE explore / EPSILON 0.04575429999991712 / ACTION 0 / REWARD 0 / Q_MAX  124.51485 / Loss  27.061748504638672\n",
      "fps: 0.7721565151276663\n",
      "TIMESTEP 54402 / STATE explore / EPSILON 0.04575330099991712 / ACTION 0 / REWARD 0 / Q_MAX  93.247574 / Loss  80.79302978515625\n",
      "fps: 0.7563863941829193\n",
      "TIMESTEP 54403 / STATE explore / EPSILON 0.04575230199991712 / ACTION 3 / REWARD -0.1 / Q_MAX  122.1567 / Loss  67.71586608886719\n",
      "fps: 0.7781657398866194\n",
      "TIMESTEP 54404 / STATE explore / EPSILON 0.045751302999917115 / ACTION 1 / REWARD -0.1 / Q_MAX  110.329834 / Loss  198.0215301513672\n",
      "fps: 0.772156372976412\n",
      "TIMESTEP 54405 / STATE explore / EPSILON 0.045750303999917113 / ACTION 3 / REWARD -0.1 / Q_MAX  103.35187 / Loss  69.91351318359375\n",
      "fps: 0.7842687906083047\n",
      "TIMESTEP 54406 / STATE explore / EPSILON 0.04574930499991711 / ACTION 2 / REWARD -0.1 / Q_MAX  131.05057 / Loss  48.50787353515625\n",
      "fps: 0.7910940656590661\n",
      "TIMESTEP 54407 / STATE explore / EPSILON 0.04574830599991711 / ACTION 3 / REWARD -0.1 / Q_MAX  123.80394 / Loss  59.62792205810547\n",
      "fps: 0.7763530451138745\n",
      "TIMESTEP 54408 / STATE explore / EPSILON 0.04574730699991711 / ACTION 1 / REWARD -0.1 / Q_MAX  118.34277 / Loss  39.86438751220703\n",
      "fps: 0.7598350657850996\n",
      "TIMESTEP 54409 / STATE explore / EPSILON 0.04574630799991711 / ACTION 1 / REWARD 0 / Q_MAX  119.69953 / Loss  82.55378723144531\n",
      "fps: 0.7867371064841083\n",
      "TIMESTEP 54410 / STATE explore / EPSILON 0.045745308999917106 / ACTION 1 / REWARD 0 / Q_MAX  123.19888 / Loss  25.594654083251953\n",
      "fps: 0.793605449256167\n",
      "TIMESTEP 54411 / STATE explore / EPSILON 0.045744309999917104 / ACTION 0 / REWARD 0 / Q_MAX  106.32641 / Loss  300.3855285644531\n",
      "fps: 0.7473415083551321\n",
      "TIMESTEP 54412 / STATE explore / EPSILON 0.0457433109999171 / ACTION 3 / REWARD 0 / Q_MAX  127.613495 / Loss  73.38243103027344\n",
      "fps: 0.7879770259536386\n",
      "TIMESTEP 54413 / STATE explore / EPSILON 0.0457423119999171 / ACTION 0 / REWARD 0 / Q_MAX  125.69002 / Loss  56.69623565673828\n",
      "error:  could not convert string to float:  \n",
      "score:  \n",
      "fps: 0.7451138534587725\n",
      "TIMESTEP 54414 / STATE explore / EPSILON 0.0457413129999171 / ACTION 2 / REWARD 3.9 / Q_MAX  90.74282 / Loss  38.413673400878906\n",
      "----------Random Action----------\n",
      "fps: 0.8123011517029974\n",
      "TIMESTEP 54415 / STATE explore / EPSILON 0.0457403139999171 / ACTION 3 / REWARD 0 / Q_MAX  115.52297 / Loss  32.93974685668945\n",
      "fps: 0.7842687906083047\n",
      "TIMESTEP 54416 / STATE explore / EPSILON 0.0457393149999171 / ACTION 3 / REWARD 0 / Q_MAX  27.676777 / Loss  73.66001892089844\n",
      "fps: 0.781816205983387\n",
      "TIMESTEP 54417 / STATE explore / EPSILON 0.045738315999917095 / ACTION 1 / REWARD 0 / Q_MAX  0.80764455 / Loss  116.22491455078125\n",
      "fps: 0.8044592273838481\n",
      "TIMESTEP 54418 / STATE explore / EPSILON 0.045737316999917094 / ACTION 2 / REWARD 0 / Q_MAX  2.005707 / Loss  45.843788146972656\n",
      "fps: 0.7674159133032903\n",
      "TIMESTEP 54419 / STATE explore / EPSILON 0.04573631799991709 / ACTION 2 / REWARD 0 / Q_MAX  113.5413 / Loss  54.57258224487305\n",
      "fps: 0.798039028776602\n",
      "TIMESTEP 54420 / STATE explore / EPSILON 0.04573531899991709 / ACTION 1 / REWARD 0 / Q_MAX  136.24246 / Loss  104.25765991210938\n",
      "fps: 0.7818159145231819\n",
      "TIMESTEP 54421 / STATE explore / EPSILON 0.04573431999991709 / ACTION 3 / REWARD 0 / Q_MAX  99.75345 / Loss  61.72715759277344\n",
      "fps: 0.7638982431613843\n",
      "TIMESTEP 54422 / STATE explore / EPSILON 0.04573332099991709 / ACTION 3 / REWARD 0 / Q_MAX  108.66609 / Loss  28.034992218017578\n",
      "fps: 0.778771709426688\n",
      "TIMESTEP 54423 / STATE explore / EPSILON 0.045732321999917086 / ACTION 1 / REWARD 0 / Q_MAX  132.67052 / Loss  36.69380569458008\n",
      "fps: 0.7304183614633152\n",
      "TIMESTEP 54424 / STATE explore / EPSILON 0.045731322999917084 / ACTION 3 / REWARD -0.1 / Q_MAX  139.70444 / Loss  93.06683349609375\n",
      "fps: 0.7662396691195277\n",
      "TIMESTEP 54425 / STATE explore / EPSILON 0.04573032399991708 / ACTION 2 / REWARD -0.1 / Q_MAX  110.16165 / Loss  37.71175765991211\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fps: 0.7490208431999051\n",
      "TIMESTEP 54426 / STATE explore / EPSILON 0.04572932499991708 / ACTION 0 / REWARD -0.1 / Q_MAX  126.782036 / Loss  58.59253692626953\n",
      "error:  could not convert string to float:  \n",
      "score:  \n",
      "fps: 0.7336337970423442\n",
      "TIMESTEP 54427 / STATE explore / EPSILON 0.04572832599991708 / ACTION 2 / REWARD 3.9 / Q_MAX  134.91985 / Loss  86.75970458984375\n",
      "error:  could not convert string to float:  \n",
      "score:  \n",
      "fps: 0.8395825051214023\n",
      "TIMESTEP 54428 / STATE explore / EPSILON 0.04572732699991708 / ACTION 3 / REWARD -0.1 / Q_MAX  64.1963 / Loss  56.023319244384766\n",
      "fps: 0.7668274803443855\n",
      "TIMESTEP 54429 / STATE explore / EPSILON 0.04572632799991708 / ACTION 0 / REWARD 0 / Q_MAX  98.64144 / Loss  202.21084594726562\n",
      "fps: 0.7685954926259395\n",
      "TIMESTEP 54430 / STATE explore / EPSILON 0.045725328999917075 / ACTION 0 / REWARD 0 / Q_MAX  117.32543 / Loss  32.3699951171875\n",
      "fps: 0.7769563376258573\n",
      "TIMESTEP 54431 / STATE explore / EPSILON 0.045724329999917074 / ACTION 2 / REWARD 0 / Q_MAX  75.72735 / Loss  105.46070861816406\n",
      "fps: 0.7101865095000456\n",
      "TIMESTEP 54432 / STATE explore / EPSILON 0.04572333099991707 / ACTION 1 / REWARD -1.0 / Q_MAX  28.267733 / Loss  98.93980407714844\n",
      "fps: 0.781816205983387\n",
      "TIMESTEP 54433 / STATE explore / EPSILON 0.04572233199991707 / ACTION 3 / REWARD 0 / Q_MAX  122.55217 / Loss  27.0800724029541\n",
      "fps: 0.7501446436597251\n",
      "TIMESTEP 54434 / STATE explore / EPSILON 0.04572133299991707 / ACTION 1 / REWARD 0 / Q_MAX  133.85233 / Loss  278.2346496582031\n",
      "fps: 0.8142856588169057\n",
      "TIMESTEP 54435 / STATE explore / EPSILON 0.04572033399991707 / ACTION 3 / REWARD 0 / Q_MAX  130.19908 / Loss  50.214942932128906\n",
      "fps: 0.7873565041652345\n",
      "TIMESTEP 54436 / STATE explore / EPSILON 0.045719334999917066 / ACTION 2 / REWARD 0 / Q_MAX  132.81708 / Loss  41.39124298095703\n",
      "fps: 0.7680052500358889\n",
      "TIMESTEP 54437 / STATE explore / EPSILON 0.045718335999917065 / ACTION 1 / REWARD -0.1 / Q_MAX  130.00363 / Loss  73.07392120361328\n",
      "fps: 0.7873566519681551\n",
      "TIMESTEP 54438 / STATE explore / EPSILON 0.04571733699991706 / ACTION 3 / REWARD 0 / Q_MAX  151.79227 / Loss  116.7291030883789\n",
      "fps: 0.7967672665986848\n",
      "TIMESTEP 54439 / STATE explore / EPSILON 0.04571633799991706 / ACTION 2 / REWARD 0 / Q_MAX  124.97953 / Loss  18.706329345703125\n",
      "fps: 0.7586819282003194\n",
      "TIMESTEP 54440 / STATE explore / EPSILON 0.04571533899991706 / ACTION 2 / REWARD 0 / Q_MAX  144.53836 / Loss  75.46625518798828\n",
      "fps: 0.7325586310075846\n",
      "TIMESTEP 54441 / STATE explore / EPSILON 0.04571433999991706 / ACTION 1 / REWARD 0 / Q_MAX  125.47571 / Loss  54.52012252807617\n",
      "fps: 0.7873565041652345\n",
      "TIMESTEP 54442 / STATE explore / EPSILON 0.04571334099991706 / ACTION 3 / REWARD 0 / Q_MAX  138.3173 / Loss  38.89638137817383\n",
      "fps: 0.7739496434074197\n",
      "TIMESTEP 54443 / STATE explore / EPSILON 0.045712341999917055 / ACTION 2 / REWARD 0 / Q_MAX  19.968008 / Loss  82.42074584960938\n",
      "fps: 0.7739496434074197\n",
      "TIMESTEP 54444 / STATE explore / EPSILON 0.045711342999917054 / ACTION 2 / REWARD 0 / Q_MAX  141.81808 / Loss  40.408164978027344\n",
      "fps: 0.7644820480947204\n",
      "TIMESTEP 54445 / STATE explore / EPSILON 0.04571034399991705 / ACTION 2 / REWARD 0 / Q_MAX  127.0166 / Loss  167.0732879638672\n",
      "fps: 0.8012364968291941\n",
      "TIMESTEP 54446 / STATE explore / EPSILON 0.04570934499991705 / ACTION 2 / REWARD 0 / Q_MAX  140.1025 / Loss  36.7949104309082\n",
      "fps: 0.7490208431999051\n",
      "TIMESTEP 54447 / STATE explore / EPSILON 0.04570834599991705 / ACTION 3 / REWARD 0 / Q_MAX  173.28113 / Loss  77.58155822753906\n",
      "fps: 0.7836542395909898\n",
      "TIMESTEP 54448 / STATE explore / EPSILON 0.04570734699991705 / ACTION 1 / REWARD 0 / Q_MAX  139.97305 / Loss  68.6280746459961\n",
      "fps: 0.8018786383275178\n",
      "TIMESTEP 54449 / STATE explore / EPSILON 0.045706347999917046 / ACTION 1 / REWARD 0 / Q_MAX  75.95467 / Loss  81.64479064941406\n",
      "fps: 0.7929760347265811\n",
      "TIMESTEP 54450 / STATE explore / EPSILON 0.045705348999917045 / ACTION 2 / REWARD 0 / Q_MAX  140.58621 / Loss  44.71285629272461\n",
      "fps: 0.7967671152415213\n",
      "TIMESTEP 54451 / STATE explore / EPSILON 0.04570434999991704 / ACTION 2 / REWARD 0 / Q_MAX  71.18309 / Loss  26.005144119262695\n",
      "fps: 0.8070563243692563\n",
      "TIMESTEP 54452 / STATE explore / EPSILON 0.04570335099991704 / ACTION 1 / REWARD 0 / Q_MAX  144.24843 / Loss  90.68255615234375\n",
      "fps: 0.7830406509407755\n",
      "TIMESTEP 54453 / STATE explore / EPSILON 0.04570235199991704 / ACTION 1 / REWARD 0 / Q_MAX  135.7581 / Loss  27.748476028442383\n",
      "fps: 0.7512718213958314\n",
      "TIMESTEP 54454 / STATE explore / EPSILON 0.04570135299991704 / ACTION 2 / REWARD 0 / Q_MAX  87.95879 / Loss  213.23788452148438\n",
      "fps: 0.754104709396769\n",
      "TIMESTEP 54455 / STATE explore / EPSILON 0.04570035399991704 / ACTION 3 / REWARD 0 / Q_MAX  135.40392 / Loss  37.33849334716797\n",
      "fps: 0.7685954926259395\n",
      "TIMESTEP 54456 / STATE explore / EPSILON 0.045699354999917036 / ACTION 0 / REWARD 0 / Q_MAX  133.68066 / Loss  68.50173950195312\n",
      "fps: 0.7406984099625722\n",
      "TIMESTEP 54457 / STATE explore / EPSILON 0.045698355999917034 / ACTION 2 / REWARD 0 / Q_MAX  138.317 / Loss  230.29644775390625\n",
      "fps: 0.7467831806135321\n",
      "TIMESTEP 54458 / STATE explore / EPSILON 0.04569735699991703 / ACTION 3 / REWARD 0 / Q_MAX  136.34834 / Loss  39.20820617675781\n",
      "fps: 0.7586819282003194\n",
      "TIMESTEP 54459 / STATE explore / EPSILON 0.04569635799991703 / ACTION 2 / REWARD 0 / Q_MAX  118.06361 / Loss  84.52815246582031\n",
      "fps: 0.7691866431636111\n",
      "TIMESTEP 54460 / STATE explore / EPSILON 0.04569535899991703 / ACTION 2 / REWARD 0 / Q_MAX  145.05846 / Loss  39.82258987426758\n",
      "fps: 0.7861183878956289\n",
      "TIMESTEP 54461 / STATE explore / EPSILON 0.04569435999991703 / ACTION 1 / REWARD 0 / Q_MAX  69.976425 / Loss  55.34410095214844\n",
      "----------Random Action----------\n",
      "fps: 0.8018787916329229\n",
      "TIMESTEP 54462 / STATE explore / EPSILON 0.045693360999917026 / ACTION 1 / REWARD 0 / Q_MAX  165.11993 / Loss  41.03712844848633\n",
      "fps: 0.7406984099625722\n",
      "TIMESTEP 54463 / STATE explore / EPSILON 0.045692361999917025 / ACTION 1 / REWARD 5.0 / Q_MAX  173.51816 / Loss  48.838401794433594\n",
      "----------Random Action----------\n",
      "error:  could not convert string to float:  \n",
      "score:  \n",
      "fps: 0.8090152563740535\n",
      "TIMESTEP 54464 / STATE explore / EPSILON 0.04569136299991702 / ACTION 1 / REWARD 0 / Q_MAX  102.532555 / Loss  84.33716583251953\n",
      "fps: 0.7598349281343183\n",
      "TIMESTEP 54465 / STATE explore / EPSILON 0.04569036399991702 / ACTION 1 / REWARD 0 / Q_MAX  127.19139 / Loss  110.09806823730469\n",
      "fps: 0.7751494096613504\n",
      "TIMESTEP 54466 / STATE explore / EPSILON 0.04568936499991702 / ACTION 0 / REWARD 0 / Q_MAX  137.38905 / Loss  388.3182067871094\n",
      "fps: 0.7423479894252282\n",
      "TIMESTEP 54467 / STATE explore / EPSILON 0.04568836599991702 / ACTION 1 / REWARD 0 / Q_MAX  180.86221 / Loss  93.91859436035156\n",
      "fps: 0.7967671152415213\n",
      "TIMESTEP 54468 / STATE explore / EPSILON 0.04568736699991702 / ACTION 2 / REWARD 0 / Q_MAX  39.1914 / Loss  60.752540588378906\n",
      "fps: 0.7685954926259395\n",
      "TIMESTEP 54469 / STATE explore / EPSILON 0.045686367999917016 / ACTION 0 / REWARD 0 / Q_MAX  142.76176 / Loss  51.245277404785156\n",
      "fps: 0.7691867842235344\n",
      "TIMESTEP 54470 / STATE explore / EPSILON 0.045685368999917014 / ACTION 2 / REWARD 0 / Q_MAX  150.91827 / Loss  266.00921630859375\n",
      "error:  could not convert string to float:  \n",
      "score:  \n",
      "fps: 0.7336337970423442\n",
      "TIMESTEP 54471 / STATE explore / EPSILON 0.04568436999991701 / ACTION 2 / REWARD 6.0 / Q_MAX  157.02708 / Loss  66.95706176757812\n",
      "fps: 0.7855010830289753\n",
      "TIMESTEP 54472 / STATE explore / EPSILON 0.04568337099991701 / ACTION 1 / REWARD 0 / Q_MAX  166.26617 / Loss  106.70504760742188\n",
      "fps: 0.7967671152415213\n",
      "TIMESTEP 54473 / STATE explore / EPSILON 0.04568237199991701 / ACTION 1 / REWARD 0 / Q_MAX  161.46999 / Loss  99.45307922363281\n",
      "----------Random Action----------\n",
      "fps: 0.7662396691195277\n",
      "TIMESTEP 54474 / STATE explore / EPSILON 0.04568137299991701 / ACTION 1 / REWARD 0 / Q_MAX  160.45862 / Loss  96.31343078613281\n",
      "fps: 0.7680052500358889\n",
      "TIMESTEP 54475 / STATE explore / EPSILON 0.04568037399991701 / ACTION 0 / REWARD 0 / Q_MAX  162.21036 / Loss  152.09381103515625\n",
      "fps: 0.768005109408992\n",
      "TIMESTEP 54476 / STATE explore / EPSILON 0.045679374999917005 / ACTION 1 / REWARD 0 / Q_MAX  150.44746 / Loss  101.50199890136719\n",
      "fps: 0.7691867842235344\n",
      "TIMESTEP 54477 / STATE explore / EPSILON 0.045678375999917004 / ACTION 1 / REWARD 0 / Q_MAX  159.66592 / Loss  25.451126098632812\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fps: 0.7633151902908395\n",
      "TIMESTEP 54478 / STATE explore / EPSILON 0.045677376999917 / ACTION 2 / REWARD -0.1 / Q_MAX  130.7896 / Loss  58.9147834777832\n",
      "----------Random Action----------\n",
      "fps: 0.7680053906628372\n",
      "TIMESTEP 54479 / STATE explore / EPSILON 0.045676377999917 / ACTION 0 / REWARD 0 / Q_MAX  133.19713 / Loss  439.06597900390625\n",
      "fps: 0.7751494096613504\n",
      "TIMESTEP 54480 / STATE explore / EPSILON 0.045675378999917 / ACTION 2 / REWARD 0 / Q_MAX  132.32846 / Loss  178.45159912109375\n",
      "fps: 0.7685954926259395\n",
      "TIMESTEP 54481 / STATE explore / EPSILON 0.045674379999917 / ACTION 1 / REWARD 0 / Q_MAX  135.21802 / Loss  135.55609130859375\n",
      "fps: 0.7674156324810283\n",
      "TIMESTEP 54482 / STATE explore / EPSILON 0.045673380999916996 / ACTION 2 / REWARD 0 / Q_MAX  153.2681 / Loss  44.18370056152344\n",
      "fps: 0.7873566519681551\n",
      "TIMESTEP 54483 / STATE explore / EPSILON 0.045672381999916994 / ACTION 1 / REWARD 0 / Q_MAX  156.31682 / Loss  93.14085388183594\n",
      "fps: 0.768005109408992\n",
      "TIMESTEP 54484 / STATE explore / EPSILON 0.04567138299991699 / ACTION 3 / REWARD 0 / Q_MAX  117.05014 / Loss  260.50054931640625\n",
      "fps: 0.7691866431636111\n",
      "TIMESTEP 54485 / STATE explore / EPSILON 0.04567038399991699 / ACTION 1 / REWARD 0 / Q_MAX  153.88506 / Loss  115.95057678222656\n",
      "fps: 0.7563862577788165\n",
      "TIMESTEP 54486 / STATE explore / EPSILON 0.04566938499991699 / ACTION 2 / REWARD 0 / Q_MAX  121.56364 / Loss  63.09796905517578\n",
      "fps: 0.765067164708627\n",
      "TIMESTEP 54487 / STATE explore / EPSILON 0.04566838599991699 / ACTION 2 / REWARD 0 / Q_MAX  138.85385 / Loss  25.627544403076172\n",
      "fps: 0.7656530379107693\n",
      "TIMESTEP 54488 / STATE explore / EPSILON 0.04566738699991699 / ACTION 2 / REWARD 0 / Q_MAX  134.28937 / Loss  451.30389404296875\n",
      "fps: 0.7879770259536386\n",
      "TIMESTEP 54489 / STATE explore / EPSILON 0.045666387999916985 / ACTION 3 / REWARD 0 / Q_MAX  166.83632 / Loss  61.2352294921875\n",
      "fps: 0.7650673042616669\n",
      "TIMESTEP 54490 / STATE explore / EPSILON 0.045665388999916984 / ACTION 0 / REWARD 0 / Q_MAX  143.01074 / Loss  36.9996337890625\n",
      "fps: 0.754104709396769\n",
      "TIMESTEP 54491 / STATE explore / EPSILON 0.04566438999991698 / ACTION 2 / REWARD 0 / Q_MAX  151.00398 / Loss  78.38968658447266\n",
      "fps: 0.7512718213958314\n",
      "TIMESTEP 54492 / STATE explore / EPSILON 0.04566339099991698 / ACTION 2 / REWARD 0 / Q_MAX  51.32401 / Loss  159.73086547851562\n",
      "fps: 0.7586819282003194\n",
      "TIMESTEP 54493 / STATE explore / EPSILON 0.04566239199991698 / ACTION 2 / REWARD 0 / Q_MAX  164.35036 / Loss  234.37965393066406\n",
      "fps: 0.7282904352044735\n",
      "TIMESTEP 54494 / STATE explore / EPSILON 0.04566139299991698 / ACTION 2 / REWARD 0 / Q_MAX  150.63013 / Loss  342.07696533203125\n",
      "fps: 0.7199014112065799\n",
      "TIMESTEP 54495 / STATE explore / EPSILON 0.045660393999916976 / ACTION 2 / REWARD 15.0 / Q_MAX  146.72368 / Loss  140.16192626953125\n",
      "fps: 0.7751495529167555\n",
      "TIMESTEP 54496 / STATE explore / EPSILON 0.045659394999916975 / ACTION 2 / REWARD 0 / Q_MAX  147.54333 / Loss  248.91424560546875\n",
      "fps: 0.7775605684864603\n",
      "TIMESTEP 54497 / STATE explore / EPSILON 0.04565839599991697 / ACTION 3 / REWARD 0 / Q_MAX  111.02965 / Loss  53.40928649902344\n",
      "fps: 0.7204198812544154\n",
      "TIMESTEP 54498 / STATE explore / EPSILON 0.04565739699991697 / ACTION 2 / REWARD 10.0 / Q_MAX  162.20186 / Loss  165.75621032714844\n",
      "fps: 0.7898443249448429\n",
      "TIMESTEP 54499 / STATE explore / EPSILON 0.04565639799991697 / ACTION 2 / REWARD 0 / Q_MAX  125.35175 / Loss  41.673133850097656\n",
      "fps: 0.7451137210900753\n",
      "TIMESTEP 54500 / STATE explore / EPSILON 0.04565539899991697 / ACTION 0 / REWARD 0 / Q_MAX  147.49698 / Loss  209.60696411132812\n",
      "fps: 0.7162914435676174\n",
      "TIMESTEP 54501 / STATE explore / EPSILON 0.04565439999991697 / ACTION 1 / REWARD 10.0 / Q_MAX  149.21352 / Loss  48.06189727783203\n",
      "fps: 0.7721566572789729\n",
      "TIMESTEP 54502 / STATE explore / EPSILON 0.045653400999916965 / ACTION 2 / REWARD 0 / Q_MAX  144.09293 / Loss  261.83697509765625\n",
      "fps: 0.7763531888145128\n",
      "TIMESTEP 54503 / STATE explore / EPSILON 0.045652401999916964 / ACTION 2 / REWARD 0 / Q_MAX  137.36923 / Loss  51.96999740600586\n",
      "fps: 0.7541045738143671\n",
      "TIMESTEP 54504 / STATE explore / EPSILON 0.04565140299991696 / ACTION 1 / REWARD 0 / Q_MAX  125.2863 / Loss  21.18877410888672\n",
      "fps: 0.7479004048604654\n",
      "TIMESTEP 54505 / STATE explore / EPSILON 0.04565040399991696 / ACTION 0 / REWARD 0 / Q_MAX  129.27179 / Loss  43.9160041809082\n",
      "fps: 0.7644820480947204\n",
      "TIMESTEP 54506 / STATE explore / EPSILON 0.04564940499991696 / ACTION 2 / REWARD 0 / Q_MAX  157.90367 / Loss  83.08094787597656\n",
      "fps: 0.7598349281343183\n",
      "TIMESTEP 54507 / STATE explore / EPSILON 0.04564840599991696 / ACTION 1 / REWARD 0 / Q_MAX  124.39196 / Loss  338.6348571777344\n",
      "fps: 0.7592579217137638\n",
      "TIMESTEP 54508 / STATE explore / EPSILON 0.045647406999916956 / ACTION 1 / REWARD 0 / Q_MAX  136.57755 / Loss  86.01162719726562\n",
      "fps: 0.7609914379246361\n",
      "TIMESTEP 54509 / STATE explore / EPSILON 0.045646407999916955 / ACTION 3 / REWARD 0 / Q_MAX  119.20081 / Loss  363.6301574707031\n",
      "fps: 0.7592581965974158\n",
      "TIMESTEP 54510 / STATE explore / EPSILON 0.04564540899991695 / ACTION 3 / REWARD 0 / Q_MAX  137.06712 / Loss  136.0032196044922\n",
      "fps: 0.768005109408992\n",
      "TIMESTEP 54511 / STATE explore / EPSILON 0.04564440999991695 / ACTION 0 / REWARD 0 / Q_MAX  104.05306 / Loss  51.69720458984375\n",
      "fps: 0.7330958838793168\n",
      "TIMESTEP 54512 / STATE explore / EPSILON 0.04564341099991695 / ACTION 3 / REWARD 0 / Q_MAX  146.72836 / Loss  56.72999954223633\n",
      "fps: 0.7495823892002192\n",
      "TIMESTEP 54513 / STATE explore / EPSILON 0.04564241199991695 / ACTION 3 / REWARD 0 / Q_MAX  132.89833 / Loss  94.91943359375\n",
      "error:  could not convert string to float:  \n",
      "score:  \n",
      "fps: 0.7209392225593275\n",
      "TIMESTEP 54514 / STATE explore / EPSILON 0.04564141299991695 / ACTION 3 / REWARD 3.0 / Q_MAX  37.895096 / Loss  129.5391082763672\n",
      "fps: 0.735792979633309\n",
      "TIMESTEP 54515 / STATE explore / EPSILON 0.045640413999916946 / ACTION 3 / REWARD 0 / Q_MAX  148.18446 / Loss  66.70700073242188\n",
      "fps: 0.7598347904835869\n",
      "TIMESTEP 54516 / STATE explore / EPSILON 0.045639414999916944 / ACTION 2 / REWARD -0.1 / Q_MAX  129.77039 / Loss  183.86285400390625\n",
      "fps: 0.7592579217137638\n",
      "TIMESTEP 54517 / STATE explore / EPSILON 0.04563841599991694 / ACTION 1 / REWARD -0.1 / Q_MAX  113.090996 / Loss  165.97084045410156\n",
      "fps: 0.8070563243692563\n",
      "TIMESTEP 54518 / STATE explore / EPSILON 0.04563741699991694 / ACTION 1 / REWARD 0 / Q_MAX  115.111015 / Loss  84.272216796875\n",
      "fps: 0.7775605684864603\n",
      "TIMESTEP 54519 / STATE explore / EPSILON 0.04563641799991694 / ACTION 1 / REWARD 0 / Q_MAX  109.17434 / Loss  59.54130554199219\n",
      "fps: 0.7633151902908395\n",
      "TIMESTEP 54520 / STATE explore / EPSILON 0.04563541899991694 / ACTION 3 / REWARD 0 / Q_MAX  150.69833 / Loss  69.82655334472656\n",
      "fps: 0.7581068079527846\n",
      "TIMESTEP 54521 / STATE explore / EPSILON 0.045634419999916936 / ACTION 3 / REWARD 0 / Q_MAX  128.54588 / Loss  254.4558868408203\n",
      "fps: 0.7368775382133087\n",
      "TIMESTEP 54522 / STATE explore / EPSILON 0.045633420999916935 / ACTION 2 / REWARD 0 / Q_MAX  132.9678 / Loss  62.97522735595703\n",
      "fps: 0.7501446436597251\n",
      "TIMESTEP 54523 / STATE explore / EPSILON 0.04563242199991693 / ACTION 1 / REWARD 0 / Q_MAX  93.35795 / Loss  62.5892333984375\n",
      "fps: 0.7330957557461435\n",
      "TIMESTEP 54524 / STATE explore / EPSILON 0.04563142299991693 / ACTION 3 / REWARD 0 / Q_MAX  87.970474 / Loss  121.43933868408203\n",
      "fps: 0.8012364968291941\n",
      "TIMESTEP 54525 / STATE explore / EPSILON 0.04563042399991693 / ACTION 1 / REWARD 0 / Q_MAX  117.06582 / Loss  62.70509338378906\n",
      "fps: 0.7873565041652345\n",
      "TIMESTEP 54526 / STATE explore / EPSILON 0.04562942499991693 / ACTION 0 / REWARD 0 / Q_MAX  107.25766 / Loss  264.4074401855469\n",
      "fps: 0.7592579217137638\n",
      "TIMESTEP 54527 / STATE explore / EPSILON 0.04562842599991693 / ACTION 3 / REWARD 0 / Q_MAX  14.263594 / Loss  70.16494750976562\n",
      "fps: 0.7873566519681551\n",
      "TIMESTEP 54528 / STATE explore / EPSILON 0.045627426999916926 / ACTION 3 / REWARD 0 / Q_MAX  81.37026 / Loss  67.02520751953125\n",
      "fps: 0.7775604243385098\n",
      "TIMESTEP 54529 / STATE explore / EPSILON 0.045626427999916924 / ACTION 3 / REWARD 0 / Q_MAX  78.84194 / Loss  83.43711853027344\n",
      "fps: 0.7757506887663758\n",
      "TIMESTEP 54530 / STATE explore / EPSILON 0.04562542899991692 / ACTION 2 / REWARD 0 / Q_MAX  131.42693 / Loss  37.38175964355469\n",
      "fps: 0.774548918895611\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TIMESTEP 54531 / STATE explore / EPSILON 0.04562442999991692 / ACTION 1 / REWARD 0 / Q_MAX  86.56478 / Loss  105.6810073852539\n",
      "----------Random Action----------\n",
      "fps: 0.7417973575605585\n",
      "TIMESTEP 54532 / STATE explore / EPSILON 0.04562343099991692 / ACTION 1 / REWARD 0 / Q_MAX  121.98244 / Loss  38.62260818481445\n",
      "fps: 0.7885982300505595\n",
      "TIMESTEP 54533 / STATE explore / EPSILON 0.04562243199991692 / ACTION 1 / REWARD 0 / Q_MAX  102.5513 / Loss  77.75468444824219\n",
      "fps: 0.7592579217137638\n",
      "TIMESTEP 54534 / STATE explore / EPSILON 0.04562143299991692 / ACTION 2 / REWARD 0 / Q_MAX  133.5553 / Loss  252.54299926757812\n",
      "fps: 0.781816205983387\n",
      "TIMESTEP 54535 / STATE explore / EPSILON 0.045620433999916915 / ACTION 2 / REWARD 0 / Q_MAX  114.20891 / Loss  172.63116455078125\n",
      "fps: 0.7484601379286716\n",
      "TIMESTEP 54536 / STATE explore / EPSILON 0.045619434999916914 / ACTION 3 / REWARD 0 / Q_MAX  121.53833 / Loss  64.4264907836914\n",
      "fps: 0.8044590730901859\n",
      "TIMESTEP 54537 / STATE explore / EPSILON 0.04561843599991691 / ACTION 1 / REWARD 0 / Q_MAX  119.85597 / Loss  203.6322784423828\n",
      "fps: 0.7763530451138745\n",
      "TIMESTEP 54538 / STATE explore / EPSILON 0.04561743699991691 / ACTION 2 / REWARD 0 / Q_MAX  117.38835 / Loss  56.22722625732422\n",
      "fps: 0.806405401793997\n",
      "TIMESTEP 54539 / STATE explore / EPSILON 0.04561643799991691 / ACTION 1 / REWARD 0 / Q_MAX  24.819342 / Loss  75.5188980102539\n",
      "fps: 0.7685956334691258\n",
      "TIMESTEP 54540 / STATE explore / EPSILON 0.04561543899991691 / ACTION 0 / REWARD 0 / Q_MAX  120.59466 / Loss  35.84825134277344\n",
      "fps: 0.7824278764405731\n",
      "TIMESTEP 54541 / STATE explore / EPSILON 0.045614439999916906 / ACTION 1 / REWARD 0 / Q_MAX  130.02571 / Loss  397.0162353515625\n",
      "fps: 0.7781655955142024\n",
      "TIMESTEP 54542 / STATE explore / EPSILON 0.045613440999916904 / ACTION 2 / REWARD 0 / Q_MAX  100.14149 / Loss  235.76986694335938\n",
      "fps: 0.7598349281343183\n",
      "TIMESTEP 54543 / STATE explore / EPSILON 0.0456124419999169 / ACTION 0 / REWARD 0 / Q_MAX  85.97271 / Loss  210.58230590820312\n",
      "error:  could not convert string to float:  \n",
      "score:  \n",
      "fps: 0.7267025463759074\n",
      "TIMESTEP 54544 / STATE explore / EPSILON 0.0456114429999169 / ACTION 1 / REWARD 21.0 / Q_MAX  14.326459 / Loss  111.12571716308594\n",
      "fps: 0.7739496434074197\n",
      "TIMESTEP 54545 / STATE explore / EPSILON 0.0456104439999169 / ACTION 2 / REWARD 0 / Q_MAX  113.34173 / Loss  26.594640731811523\n",
      "fps: 0.7456694218049652\n",
      "TIMESTEP 54546 / STATE explore / EPSILON 0.0456094449999169 / ACTION 1 / REWARD 0 / Q_MAX  47.27973 / Loss  77.71240234375\n",
      "fps: 0.7423479894252282\n",
      "TIMESTEP 54547 / STATE explore / EPSILON 0.0456084459999169 / ACTION 3 / REWARD 0 / Q_MAX  139.15727 / Loss  76.97676086425781\n",
      "fps: 0.7668273401484729\n",
      "TIMESTEP 54548 / STATE explore / EPSILON 0.045607446999916895 / ACTION 1 / REWARD 0 / Q_MAX  131.841 / Loss  50.71299362182617\n",
      "fps: 0.7691867842235344\n",
      "TIMESTEP 54549 / STATE explore / EPSILON 0.045606447999916894 / ACTION 3 / REWARD 0 / Q_MAX  123.66629 / Loss  93.24708557128906\n",
      "fps: 0.7581068079527846\n",
      "TIMESTEP 54550 / STATE explore / EPSILON 0.04560544899991689 / ACTION 2 / REWARD 0 / Q_MAX  126.22309 / Loss  37.529640197753906\n",
      "fps: 0.7685956334691258\n",
      "TIMESTEP 54551 / STATE explore / EPSILON 0.04560444999991689 / ACTION 1 / REWARD 0 / Q_MAX  146.3662 / Loss  89.3773193359375\n",
      "fps: 0.7781655955142024\n",
      "TIMESTEP 54552 / STATE explore / EPSILON 0.04560345099991689 / ACTION 1 / REWARD 0 / Q_MAX  45.75056 / Loss  81.52317810058594\n",
      "fps: 0.7581068079527846\n",
      "TIMESTEP 54553 / STATE explore / EPSILON 0.04560245199991689 / ACTION 1 / REWARD 0 / Q_MAX  148.47069 / Loss  49.69209289550781\n",
      "fps: 0.7818160602532572\n",
      "TIMESTEP 54554 / STATE explore / EPSILON 0.045601452999916886 / ACTION 0 / REWARD 0 / Q_MAX  134.72255 / Loss  250.57521057128906\n",
      "fps: 0.7518368836558493\n",
      "TIMESTEP 54555 / STATE explore / EPSILON 0.045600453999916885 / ACTION 2 / REWARD 0 / Q_MAX  126.95318 / Loss  51.102718353271484\n",
      "fps: 0.7507078765992999\n",
      "TIMESTEP 54556 / STATE explore / EPSILON 0.04559945499991688 / ACTION 2 / REWARD 0 / Q_MAX  118.505104 / Loss  44.72541427612305\n",
      "fps: 0.7598350657850996\n",
      "TIMESTEP 54557 / STATE explore / EPSILON 0.04559845599991688 / ACTION 1 / REWARD 0 / Q_MAX  150.57121 / Loss  240.75234985351562\n",
      "fps: 0.7501446436597251\n",
      "TIMESTEP 54558 / STATE explore / EPSILON 0.04559745699991688 / ACTION 1 / REWARD 0 / Q_MAX  137.05363 / Loss  70.6932601928711\n",
      "fps: 0.7473415083551321\n",
      "TIMESTEP 54559 / STATE explore / EPSILON 0.04559645799991688 / ACTION 2 / REWARD 0 / Q_MAX  131.66614 / Loss  91.8737564086914\n",
      "fps: 0.7948671299833837\n",
      "TIMESTEP 54560 / STATE explore / EPSILON 0.04559545899991688 / ACTION 1 / REWARD 0 / Q_MAX  49.86138 / Loss  89.11043548583984\n",
      "fps: 0.7763530451138745\n",
      "TIMESTEP 54561 / STATE explore / EPSILON 0.045594459999916875 / ACTION 3 / REWARD 0 / Q_MAX  25.941908 / Loss  97.85961151123047\n",
      "fps: 0.7848844531348098\n",
      "TIMESTEP 54562 / STATE explore / EPSILON 0.045593460999916874 / ACTION 3 / REWARD 0 / Q_MAX  144.81114 / Loss  75.16287231445312\n",
      "fps: 0.7974024132127265\n",
      "TIMESTEP 54563 / STATE explore / EPSILON 0.04559246199991687 / ACTION 3 / REWARD 0 / Q_MAX  145.3536 / Loss  73.90771484375\n",
      "fps: 0.7592581965974158\n",
      "TIMESTEP 54564 / STATE explore / EPSILON 0.04559146299991687 / ACTION 3 / REWARD 0 / Q_MAX  120.578 / Loss  100.14204406738281\n",
      "fps: 0.7406984099625722\n",
      "TIMESTEP 54565 / STATE explore / EPSILON 0.04559046399991687 / ACTION 2 / REWARD 0 / Q_MAX  134.72737 / Loss  131.41236877441406\n",
      "fps: 0.7954995513548401\n",
      "TIMESTEP 54566 / STATE explore / EPSILON 0.04558946499991687 / ACTION 0 / REWARD 0 / Q_MAX  120.55881 / Loss  50.79014587402344\n",
      "----------Random Action----------\n",
      "fps: 0.7721566572789729\n",
      "TIMESTEP 54567 / STATE explore / EPSILON 0.045588465999916866 / ACTION 0 / REWARD -0.1 / Q_MAX  117.74702 / Loss  190.57080078125\n",
      "fps: 0.7745490619291485\n",
      "TIMESTEP 54568 / STATE explore / EPSILON 0.045587466999916865 / ACTION 1 / REWARD -0.1 / Q_MAX  138.28893 / Loss  119.56161499023438\n",
      "fps: 0.7787715648293334\n",
      "TIMESTEP 54569 / STATE explore / EPSILON 0.04558646799991686 / ACTION 0 / REWARD 0 / Q_MAX  118.13574 / Loss  37.29703903198242\n",
      "fps: 0.7680052500358889\n",
      "TIMESTEP 54570 / STATE explore / EPSILON 0.04558546899991686 / ACTION 3 / REWARD 0 / Q_MAX  69.49931 / Loss  55.495689392089844\n",
      "fps: 0.7781654511418389\n",
      "TIMESTEP 54571 / STATE explore / EPSILON 0.04558446999991686 / ACTION 1 / REWARD 0 / Q_MAX  138.693 / Loss  53.13439178466797\n",
      "fps: 0.7685954926259395\n",
      "TIMESTEP 54572 / STATE explore / EPSILON 0.04558347099991686 / ACTION 0 / REWARD 0 / Q_MAX  151.00652 / Loss  226.4590606689453\n",
      "fps: 0.7763530451138745\n",
      "TIMESTEP 54573 / STATE explore / EPSILON 0.04558247199991686 / ACTION 3 / REWARD 0 / Q_MAX  106.29918 / Loss  60.581573486328125\n",
      "fps: 0.7787718540240963\n",
      "TIMESTEP 54574 / STATE explore / EPSILON 0.045581472999916856 / ACTION 3 / REWARD 0 / Q_MAX  123.7049 / Loss  135.25341796875\n",
      "fps: 0.7812053456350289\n",
      "TIMESTEP 54575 / STATE explore / EPSILON 0.045580473999916854 / ACTION 3 / REWARD 0 / Q_MAX  119.19616 / Loss  28.337581634521484\n",
      "----------Random Action----------\n",
      "fps: 0.7739496434074197\n",
      "TIMESTEP 54576 / STATE explore / EPSILON 0.04557947499991685 / ACTION 2 / REWARD 0 / Q_MAX  140.10439 / Loss  61.73999786376953\n",
      "fps: 0.7775604243385098\n",
      "TIMESTEP 54577 / STATE explore / EPSILON 0.04557847599991685 / ACTION 1 / REWARD 0 / Q_MAX  136.04128 / Loss  209.9536895751953\n",
      "fps: 0.7598350657850996\n",
      "TIMESTEP 54578 / STATE explore / EPSILON 0.04557747699991685 / ACTION 2 / REWARD 0 / Q_MAX  164.42688 / Loss  263.3526916503906\n",
      "fps: 0.7417973575605585\n",
      "TIMESTEP 54579 / STATE explore / EPSILON 0.04557647799991685 / ACTION 3 / REWARD 0 / Q_MAX  144.65408 / Loss  63.57533264160156\n",
      "fps: 0.7769563376258573\n",
      "TIMESTEP 54580 / STATE explore / EPSILON 0.045575478999916846 / ACTION 1 / REWARD 0 / Q_MAX  139.37163 / Loss  109.30450439453125\n",
      "fps: 0.7401500909944521\n",
      "TIMESTEP 54581 / STATE explore / EPSILON 0.045574479999916845 / ACTION 1 / REWARD 0 / Q_MAX  96.94598 / Loss  42.78135681152344\n",
      "fps: 0.7674157728921336\n",
      "TIMESTEP 54582 / STATE explore / EPSILON 0.04557348099991684 / ACTION 2 / REWARD 0 / Q_MAX  63.544876 / Loss  50.30461883544922\n",
      "fps: 0.7592579217137638\n",
      "TIMESTEP 54583 / STATE explore / EPSILON 0.04557248199991684 / ACTION 1 / REWARD 0 / Q_MAX  81.57851 / Loss  285.5773010253906\n",
      "fps: 0.781816205983387\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TIMESTEP 54584 / STATE explore / EPSILON 0.04557148299991684 / ACTION 1 / REWARD 0 / Q_MAX  129.49611 / Loss  230.73068237304688\n",
      "fps: 0.7745490619291485\n",
      "TIMESTEP 54585 / STATE explore / EPSILON 0.04557048399991684 / ACTION 0 / REWARD 0 / Q_MAX  150.4885 / Loss  141.33389282226562\n",
      "fps: 0.7967672665986848\n",
      "TIMESTEP 54586 / STATE explore / EPSILON 0.04556948499991684 / ACTION 3 / REWARD 0 / Q_MAX  -11.406457 / Loss  50.84705352783203\n",
      "fps: 0.7685954926259395\n",
      "TIMESTEP 54587 / STATE explore / EPSILON 0.045568485999916836 / ACTION 3 / REWARD 0 / Q_MAX  92.22284 / Loss  22.42959976196289\n",
      "fps: 0.7691866431636111\n",
      "TIMESTEP 54588 / STATE explore / EPSILON 0.045567486999916834 / ACTION 1 / REWARD -0.1 / Q_MAX  163.38597 / Loss  53.2658805847168\n",
      "fps: 0.7910940656590661\n",
      "TIMESTEP 54589 / STATE explore / EPSILON 0.04556648799991683 / ACTION 2 / REWARD -0.1 / Q_MAX  121.10301 / Loss  85.283935546875\n",
      "----------Random Action----------\n",
      "fps: 0.7721566572789729\n",
      "TIMESTEP 54590 / STATE explore / EPSILON 0.04556548899991683 / ACTION 1 / REWARD -0.1 / Q_MAX  72.564156 / Loss  53.941471099853516\n",
      "fps: 0.7680052500358889\n",
      "TIMESTEP 54591 / STATE explore / EPSILON 0.04556448999991683 / ACTION 1 / REWARD 0 / Q_MAX  117.65128 / Loss  195.9746856689453\n",
      "fps: 0.754104709396769\n",
      "TIMESTEP 54592 / STATE explore / EPSILON 0.04556349099991683 / ACTION 1 / REWARD 0 / Q_MAX  127.104195 / Loss  57.52050018310547\n",
      "fps: 0.7873565041652345\n",
      "TIMESTEP 54593 / STATE explore / EPSILON 0.045562491999916827 / ACTION 0 / REWARD 0 / Q_MAX  142.82605 / Loss  139.3106231689453\n",
      "fps: 0.7598349281343183\n",
      "TIMESTEP 54594 / STATE explore / EPSILON 0.045561492999916825 / ACTION 3 / REWARD 0 / Q_MAX  155.46294 / Loss  69.7032470703125\n",
      "fps: 0.7775604243385098\n",
      "TIMESTEP 54595 / STATE explore / EPSILON 0.045560493999916823 / ACTION 1 / REWARD 0 / Q_MAX  144.9122 / Loss  50.41820526123047\n",
      "fps: 0.7680052500358889\n",
      "TIMESTEP 54596 / STATE explore / EPSILON 0.04555949499991682 / ACTION 1 / REWARD 0 / Q_MAX  128.30045 / Loss  415.8555908203125\n",
      "----------Random Action----------\n",
      "fps: 0.7769563376258573\n",
      "TIMESTEP 54597 / STATE explore / EPSILON 0.04555849599991682 / ACTION 1 / REWARD 0 / Q_MAX  17.298143 / Loss  354.01141357421875\n",
      "error:  could not convert string to float: '\\\\... M' \n",
      "score:  \\... M\n",
      "fps: 0.7101867500000423\n",
      "TIMESTEP 54598 / STATE explore / EPSILON 0.04555749699991682 / ACTION 2 / REWARD 39.0 / Q_MAX  135.72441 / Loss  27.502138137817383\n",
      "fps: 0.7830405047538166\n",
      "TIMESTEP 54599 / STATE explore / EPSILON 0.04555649799991682 / ACTION 1 / REWARD 0 / Q_MAX  -29.238821 / Loss  64.14936065673828\n",
      "fps: 0.7910940656590661\n",
      "TIMESTEP 54600 / STATE explore / EPSILON 0.045555498999916816 / ACTION 0 / REWARD 0 / Q_MAX  125.52382 / Loss  121.73554992675781\n",
      "fps: 0.7775605684864603\n",
      "TIMESTEP 54601 / STATE explore / EPSILON 0.045554499999916814 / ACTION 1 / REWARD 0 / Q_MAX  122.90925 / Loss  38.551326751708984\n",
      "fps: 0.7781657398866194\n",
      "TIMESTEP 54602 / STATE explore / EPSILON 0.04555350099991681 / ACTION 3 / REWARD 0 / Q_MAX  155.70322 / Loss  39.31156921386719\n",
      "fps: 0.7473412420324569\n",
      "TIMESTEP 54603 / STATE explore / EPSILON 0.04555250199991681 / ACTION 1 / REWARD 0 / Q_MAX  126.86642 / Loss  191.91281127929688\n",
      "fps: 0.7775604243385098\n",
      "TIMESTEP 54604 / STATE explore / EPSILON 0.04555150299991681 / ACTION 3 / REWARD 0 / Q_MAX  130.46686 / Loss  61.73776626586914\n",
      "fps: 0.7961326776439218\n",
      "TIMESTEP 54605 / STATE explore / EPSILON 0.04555050399991681 / ACTION 1 / REWARD -0.1 / Q_MAX  96.65282 / Loss  259.14825439453125\n",
      "fps: 0.773949500595182\n",
      "TIMESTEP 54606 / STATE explore / EPSILON 0.04554950499991681 / ACTION 2 / REWARD -0.1 / Q_MAX  123.93583 / Loss  114.11827087402344\n",
      "fps: 0.759258059155565\n",
      "TIMESTEP 54607 / STATE explore / EPSILON 0.045548505999916805 / ACTION 2 / REWARD -0.1 / Q_MAX  125.02137 / Loss  33.90425109863281\n",
      "fps: 0.7929760347265811\n",
      "TIMESTEP 54608 / STATE explore / EPSILON 0.045547506999916804 / ACTION 3 / REWARD 0 / Q_MAX  144.1856 / Loss  85.4232177734375\n",
      "fps: 0.7775605684864603\n",
      "TIMESTEP 54609 / STATE explore / EPSILON 0.0455465079999168 / ACTION 3 / REWARD 0 / Q_MAX  145.41685 / Loss  268.6833190917969\n",
      "fps: 0.8156138478626788\n",
      "TIMESTEP 54610 / STATE explore / EPSILON 0.0455455089999168 / ACTION 2 / REWARD 0 / Q_MAX  129.41083 / Loss  66.03340148925781\n",
      "fps: 0.7812053456350289\n",
      "TIMESTEP 54611 / STATE explore / EPSILON 0.0455445099999168 / ACTION 0 / REWARD 0 / Q_MAX  123.52875 / Loss  36.093666076660156\n",
      "fps: 0.7680053906628372\n",
      "TIMESTEP 54612 / STATE explore / EPSILON 0.0455435109999168 / ACTION 2 / REWARD 0 / Q_MAX  44.384525 / Loss  36.47819900512695\n",
      "fps: 0.7781655955142024\n",
      "TIMESTEP 54613 / STATE explore / EPSILON 0.045542511999916796 / ACTION 1 / REWARD 0 / Q_MAX  124.799194 / Loss  114.13240051269531\n",
      "fps: 0.7563863941829193\n",
      "TIMESTEP 54614 / STATE explore / EPSILON 0.045541512999916794 / ACTION 1 / REWARD 0 / Q_MAX  113.20784 / Loss  62.55182647705078\n",
      "fps: 0.7769564815499177\n",
      "TIMESTEP 54615 / STATE explore / EPSILON 0.04554051399991679 / ACTION 2 / REWARD 0 / Q_MAX  145.17422 / Loss  100.92825317382812\n",
      "fps: 0.7518366141195939\n",
      "TIMESTEP 54616 / STATE explore / EPSILON 0.04553951499991679 / ACTION 3 / REWARD 0 / Q_MAX  100.24455 / Loss  50.247833251953125\n",
      "fps: 0.7656530379107693\n",
      "TIMESTEP 54617 / STATE explore / EPSILON 0.04553851599991679 / ACTION 2 / REWARD 0 / Q_MAX  35.015526 / Loss  54.16575622558594\n",
      "fps: 0.7775605684864603\n",
      "TIMESTEP 54618 / STATE explore / EPSILON 0.04553751699991679 / ACTION 2 / REWARD 0 / Q_MAX  146.73245 / Loss  41.846405029296875\n",
      "fps: 0.7417972263675828\n",
      "TIMESTEP 54619 / STATE explore / EPSILON 0.04553651799991679 / ACTION 2 / REWARD 0 / Q_MAX  121.35664 / Loss  219.00128173828125\n",
      "fps: 0.7417973575605585\n",
      "TIMESTEP 54620 / STATE explore / EPSILON 0.045535518999916785 / ACTION 3 / REWARD 0 / Q_MAX  130.71014 / Loss  140.83706665039062\n",
      "fps: 0.7592579217137638\n",
      "TIMESTEP 54621 / STATE explore / EPSILON 0.045534519999916784 / ACTION 1 / REWARD 0 / Q_MAX  16.92778 / Loss  61.9219856262207\n",
      "fps: 0.7368775382133087\n",
      "TIMESTEP 54622 / STATE explore / EPSILON 0.04553352099991678 / ACTION 2 / REWARD 0 / Q_MAX  -6.3642335 / Loss  427.2845458984375\n",
      "fps: 0.754104709396769\n",
      "TIMESTEP 54623 / STATE explore / EPSILON 0.04553252199991678 / ACTION 2 / REWARD 0 / Q_MAX  129.42871 / Loss  61.01378631591797\n",
      "fps: 0.7885983783200424\n",
      "TIMESTEP 54624 / STATE explore / EPSILON 0.04553152299991678 / ACTION 2 / REWARD 0 / Q_MAX  134.09056 / Loss  220.0909881591797\n",
      "fps: 0.7781654511418389\n",
      "TIMESTEP 54625 / STATE explore / EPSILON 0.04553052399991678 / ACTION 1 / REWARD 0 / Q_MAX  28.563927 / Loss  181.93032836914062\n",
      "fps: 0.7541048449792196\n",
      "TIMESTEP 54626 / STATE explore / EPSILON 0.045529524999916776 / ACTION 1 / REWARD 0 / Q_MAX  141.1178 / Loss  65.20722198486328\n",
      "fps: 0.7917201960847234\n",
      "TIMESTEP 54627 / STATE explore / EPSILON 0.045528525999916775 / ACTION 3 / REWARD -0.1 / Q_MAX  127.08247 / Loss  86.8533935546875\n",
      "fps: 0.8070564796608223\n",
      "TIMESTEP 54628 / STATE explore / EPSILON 0.04552752699991677 / ACTION 2 / REWARD 0 / Q_MAX  122.91778 / Loss  165.37026977539062\n",
      "fps: 0.7775607126344644\n",
      "TIMESTEP 54629 / STATE explore / EPSILON 0.04552652799991677 / ACTION 1 / REWARD 0 / Q_MAX  124.76559 / Loss  76.86962890625\n",
      "fps: 0.7818160602532572\n",
      "TIMESTEP 54630 / STATE explore / EPSILON 0.04552552899991677 / ACTION 2 / REWARD 0 / Q_MAX  131.41101 / Loss  377.89361572265625\n",
      "fps: 0.7879768779177125\n",
      "TIMESTEP 54631 / STATE explore / EPSILON 0.04552452999991677 / ACTION 1 / REWARD 0 / Q_MAX  107.90343 / Loss  85.4969482421875\n",
      "fps: 0.8077081430876247\n",
      "TIMESTEP 54632 / STATE explore / EPSILON 0.04552353099991677 / ACTION 2 / REWARD 0 / Q_MAX  132.52954 / Loss  83.33824157714844\n",
      "fps: 0.7910939164496377\n",
      "TIMESTEP 54633 / STATE explore / EPSILON 0.045522531999916765 / ACTION 2 / REWARD 0 / Q_MAX  149.90477 / Loss  60.04738998413086\n",
      "fps: 0.8038123316648824\n",
      "TIMESTEP 54634 / STATE explore / EPSILON 0.045521532999916764 / ACTION 2 / REWARD 0 / Q_MAX  127.1032 / Loss  139.9584503173828\n",
      "fps: 0.7751494096613504\n",
      "TIMESTEP 54635 / STATE explore / EPSILON 0.04552053399991676 / ACTION 2 / REWARD 0 / Q_MAX  128.19138 / Loss  144.5169677734375\n",
      "fps: 0.8044592273838481\n",
      "TIMESTEP 54636 / STATE explore / EPSILON 0.04551953499991676 / ACTION 2 / REWARD 0 / Q_MAX  151.82542 / Loss  94.17329406738281\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fps: 0.8044590730901859\n",
      "TIMESTEP 54637 / STATE explore / EPSILON 0.04551853599991676 / ACTION 2 / REWARD 0 / Q_MAX  137.17976 / Loss  167.5292205810547\n",
      "fps: 0.8064052467528872\n",
      "TIMESTEP 54638 / STATE explore / EPSILON 0.04551753699991676 / ACTION 2 / REWARD 0 / Q_MAX  118.060684 / Loss  66.88111877441406\n",
      "fps: 0.793605449256167\n",
      "TIMESTEP 54639 / STATE explore / EPSILON 0.045516537999916756 / ACTION 1 / REWARD 0 / Q_MAX  110.90995 / Loss  246.84902954101562\n",
      "----------Random Action----------\n",
      "fps: 0.8044592273838481\n",
      "TIMESTEP 54640 / STATE explore / EPSILON 0.045515538999916755 / ACTION 3 / REWARD 0 / Q_MAX  84.44392 / Loss  79.57928466796875\n",
      "fps: 0.8012361907094285\n",
      "TIMESTEP 54641 / STATE explore / EPSILON 0.04551453999991675 / ACTION 3 / REWARD 0 / Q_MAX  164.93541 / Loss  222.17994689941406\n",
      "----------Random Action----------\n",
      "fps: 0.7942357133619053\n",
      "TIMESTEP 54642 / STATE explore / EPSILON 0.04551354099991675 / ACTION 0 / REWARD 0 / Q_MAX  89.99414 / Loss  69.11834716796875\n",
      "fps: 0.8109837211748473\n",
      "TIMESTEP 54643 / STATE explore / EPSILON 0.04551254199991675 / ACTION 3 / REWARD 0 / Q_MAX  17.162252 / Loss  38.03304672241211\n",
      "fps: 0.8070563243692563\n",
      "TIMESTEP 54644 / STATE explore / EPSILON 0.04551154299991675 / ACTION 0 / REWARD 0 / Q_MAX  134.67728 / Loss  248.96481323242188\n",
      "fps: 0.7434518409645952\n",
      "TIMESTEP 54645 / STATE explore / EPSILON 0.04551054399991675 / ACTION 3 / REWARD 35.0 / Q_MAX  149.67148 / Loss  70.55864715576172\n",
      "fps: 0.7993148572981457\n",
      "TIMESTEP 54646 / STATE explore / EPSILON 0.045509544999916746 / ACTION 1 / REWARD 0 / Q_MAX  138.7781 / Loss  45.74971008300781\n",
      "----------Random Action----------\n",
      "fps: 0.7917203455304925\n",
      "TIMESTEP 54647 / STATE explore / EPSILON 0.045508545999916744 / ACTION 1 / REWARD 0 / Q_MAX  91.67237 / Loss  76.04034423828125\n",
      "fps: 0.8038126397562904\n",
      "TIMESTEP 54648 / STATE explore / EPSILON 0.04550754699991674 / ACTION 1 / REWARD 0 / Q_MAX  123.731064 / Loss  68.54035949707031\n",
      "fps: 0.8012361907094285\n",
      "TIMESTEP 54649 / STATE explore / EPSILON 0.04550654799991674 / ACTION 1 / REWARD 0 / Q_MAX  135.40886 / Loss  142.6062469482422\n",
      "error:  could not convert string to float:  \n",
      "score:  \n",
      "fps: 0.7563863941829193\n",
      "TIMESTEP 54650 / STATE explore / EPSILON 0.04550554899991674 / ACTION 3 / REWARD -2.0 / Q_MAX  127.93281 / Loss  253.95156860351562\n",
      "fps: 0.7836540931748377\n",
      "TIMESTEP 54651 / STATE explore / EPSILON 0.04550454999991674 / ACTION 2 / REWARD 0 / Q_MAX  137.21663 / Loss  94.9205551147461\n",
      "fps: 0.8057556831531081\n",
      "TIMESTEP 54652 / STATE explore / EPSILON 0.045503550999916736 / ACTION 2 / REWARD 0 / Q_MAX  134.57187 / Loss  283.32958984375\n",
      "fps: 0.8044592273838481\n",
      "TIMESTEP 54653 / STATE explore / EPSILON 0.045502551999916735 / ACTION 1 / REWARD 0 / Q_MAX  141.3929 / Loss  225.5999755859375\n",
      "fps: 0.8162798862506705\n",
      "TIMESTEP 54654 / STATE explore / EPSILON 0.04550155299991673 / ACTION 3 / REWARD 0 / Q_MAX  54.71317 / Loss  103.0068359375\n",
      "fps: 0.7936055994145076\n",
      "TIMESTEP 54655 / STATE explore / EPSILON 0.04550055399991673 / ACTION 2 / REWARD 0 / Q_MAX  126.74566 / Loss  51.17228698730469\n",
      "fps: 0.7879770259536386\n",
      "TIMESTEP 54656 / STATE explore / EPSILON 0.04549955499991673 / ACTION 1 / REWARD 0 / Q_MAX  127.84881 / Loss  226.66400146484375\n",
      "fps: 0.793605449256167\n",
      "TIMESTEP 54657 / STATE explore / EPSILON 0.04549855599991673 / ACTION 0 / REWARD 0 / Q_MAX  113.62023 / Loss  46.447044372558594\n",
      "fps: 0.7873563563623694\n",
      "TIMESTEP 54658 / STATE explore / EPSILON 0.04549755699991673 / ACTION 3 / REWARD 0 / Q_MAX  117.94569 / Loss  57.89227294921875\n",
      "fps: 0.8116419018367631\n",
      "TIMESTEP 54659 / STATE explore / EPSILON 0.045496557999916726 / ACTION 3 / REWARD 0 / Q_MAX  159.35303 / Loss  52.31824493408203\n",
      "fps: 0.8149493705195848\n",
      "TIMESTEP 54660 / STATE explore / EPSILON 0.045495558999916724 / ACTION 2 / REWARD 0 / Q_MAX  138.03717 / Loss  52.550865173339844\n",
      "fps: 0.8169466950726921\n",
      "TIMESTEP 54661 / STATE explore / EPSILON 0.04549455999991672 / ACTION 2 / REWARD 0 / Q_MAX  115.32624 / Loss  39.50981521606445\n",
      "fps: 0.8142858169030299\n",
      "TIMESTEP 54662 / STATE explore / EPSILON 0.04549356099991672 / ACTION 2 / REWARD 0 / Q_MAX  86.253456 / Loss  69.19269561767578\n",
      "fps: 0.8129616309534123\n",
      "TIMESTEP 54663 / STATE explore / EPSILON 0.04549256199991672 / ACTION 3 / REWARD 0 / Q_MAX  152.8795 / Loss  28.978179931640625\n",
      "fps: 0.8162798862506705\n",
      "TIMESTEP 54664 / STATE explore / EPSILON 0.04549156299991672 / ACTION 3 / REWARD 0 / Q_MAX  137.43689 / Loss  86.58158874511719\n",
      "fps: 0.8070563243692563\n",
      "TIMESTEP 54665 / STATE explore / EPSILON 0.04549056399991672 / ACTION 2 / REWARD 0 / Q_MAX  128.3068 / Loss  46.040287017822266\n",
      "fps: 0.8044590730901859\n",
      "TIMESTEP 54666 / STATE explore / EPSILON 0.045489564999916715 / ACTION 2 / REWARD 0 / Q_MAX  152.93867 / Loss  117.69742584228516\n",
      "fps: 0.7818160602532572\n",
      "TIMESTEP 54667 / STATE explore / EPSILON 0.045488565999916714 / ACTION 2 / REWARD 0 / Q_MAX  124.88218 / Loss  93.81283569335938\n",
      "fps: 0.7842687906083047\n",
      "TIMESTEP 54668 / STATE explore / EPSILON 0.04548756699991671 / ACTION 3 / REWARD 0 / Q_MAX  150.14377 / Loss  190.82220458984375\n",
      "fps: 0.7781657398866194\n",
      "TIMESTEP 54669 / STATE explore / EPSILON 0.04548656799991671 / ACTION 1 / REWARD 0 / Q_MAX  153.58286 / Loss  64.86869049072266\n",
      "fps: 0.797402564811354\n",
      "TIMESTEP 54670 / STATE explore / EPSILON 0.04548556899991671 / ACTION 1 / REWARD 0 / Q_MAX  142.91142 / Loss  42.20148468017578\n",
      "fps: 0.7727531582660263\n",
      "TIMESTEP 54671 / STATE explore / EPSILON 0.04548456999991671 / ACTION 2 / REWARD 0 / Q_MAX  139.81851 / Loss  107.51918029785156\n",
      "fps: 0.7739496434074197\n",
      "TIMESTEP 54672 / STATE explore / EPSILON 0.045483570999916706 / ACTION 0 / REWARD 0 / Q_MAX  140.45428 / Loss  261.1657409667969\n",
      "fps: 0.7948671299833837\n",
      "TIMESTEP 54673 / STATE explore / EPSILON 0.045482571999916704 / ACTION 3 / REWARD 0 / Q_MAX  147.74359 / Loss  119.67305755615234\n",
      "fps: 0.7873565041652345\n",
      "TIMESTEP 54674 / STATE explore / EPSILON 0.0454815729999167 / ACTION 1 / REWARD 0 / Q_MAX  145.07176 / Loss  53.786163330078125\n",
      "fps: 0.806405401793997\n",
      "TIMESTEP 54675 / STATE explore / EPSILON 0.0454805739999167 / ACTION 3 / REWARD 0 / Q_MAX  128.74506 / Loss  71.84839630126953\n",
      "fps: 0.7936052990978834\n",
      "TIMESTEP 54676 / STATE explore / EPSILON 0.0454795749999167 / ACTION 3 / REWARD 0 / Q_MAX  147.61382 / Loss  110.37265014648438\n",
      "fps: 0.7751494096613504\n",
      "TIMESTEP 54677 / STATE explore / EPSILON 0.0454785759999167 / ACTION 1 / REWARD 0 / Q_MAX  153.15854 / Loss  121.2744140625\n",
      "fps: 0.8025222705792627\n",
      "TIMESTEP 54678 / STATE explore / EPSILON 0.0454775769999167 / ACTION 0 / REWARD 0 / Q_MAX  123.697754 / Loss  46.03740310668945\n",
      "fps: 0.8051070106224577\n",
      "TIMESTEP 54679 / STATE explore / EPSILON 0.045476577999916695 / ACTION 2 / REWARD 0 / Q_MAX  118.563416 / Loss  321.2677307128906\n",
      "----------Random Action----------\n",
      "fps: 0.8129616309534123\n",
      "TIMESTEP 54680 / STATE explore / EPSILON 0.045475578999916694 / ACTION 2 / REWARD 0 / Q_MAX  115.79193 / Loss  67.45417022705078\n",
      "----------Random Action----------\n",
      "fps: 0.7581066709274692\n",
      "TIMESTEP 54681 / STATE explore / EPSILON 0.04547457999991669 / ACTION 3 / REWARD 0 / Q_MAX  137.86436 / Loss  31.98758888244629\n",
      "fps: 0.7948669793472747\n",
      "TIMESTEP 54682 / STATE explore / EPSILON 0.04547358099991669 / ACTION 2 / REWARD 0 / Q_MAX  148.28142 / Loss  112.63880920410156\n",
      "fps: 0.7848844531348098\n",
      "TIMESTEP 54683 / STATE explore / EPSILON 0.04547258199991669 / ACTION 1 / REWARD 0 / Q_MAX  156.63316 / Loss  287.5787658691406\n",
      "fps: 0.8038124857105569\n",
      "TIMESTEP 54684 / STATE explore / EPSILON 0.04547158299991669 / ACTION 3 / REWARD 0 / Q_MAX  142.19942 / Loss  53.021053314208984\n",
      "fps: 0.7745490619291485\n",
      "TIMESTEP 54685 / STATE explore / EPSILON 0.045470583999916686 / ACTION 2 / REWARD 0 / Q_MAX  152.19287 / Loss  53.071044921875\n",
      "fps: 0.7861185352340754\n",
      "TIMESTEP 54686 / STATE explore / EPSILON 0.045469584999916685 / ACTION 1 / REWARD 0 / Q_MAX  149.99268 / Loss  393.8594055175781\n",
      "fps: 0.7781655955142024\n",
      "TIMESTEP 54687 / STATE explore / EPSILON 0.04546858599991668 / ACTION 0 / REWARD 0 / Q_MAX  70.6773 / Loss  148.36181640625\n",
      "fps: 0.7842690839002386\n",
      "TIMESTEP 54688 / STATE explore / EPSILON 0.04546758699991668 / ACTION 3 / REWARD 0 / Q_MAX  147.65369 / Loss  161.92111206054688\n",
      "fps: 0.8012361907094285\n",
      "TIMESTEP 54689 / STATE explore / EPSILON 0.04546658799991668 / ACTION 3 / REWARD 0 / Q_MAX  156.27953 / Loss  155.09222412109375\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fps: 0.8057558379445455\n",
      "TIMESTEP 54690 / STATE explore / EPSILON 0.04546558899991668 / ACTION 3 / REWARD 0 / Q_MAX  123.71778 / Loss  35.754493713378906\n",
      "fps: 0.8038126397562904\n",
      "TIMESTEP 54691 / STATE explore / EPSILON 0.04546458999991668 / ACTION 1 / REWARD 0 / Q_MAX  39.512413 / Loss  53.94624710083008\n",
      "fps: 0.8142858169030299\n",
      "TIMESTEP 54692 / STATE explore / EPSILON 0.045463590999916675 / ACTION 3 / REWARD 0 / Q_MAX  139.51962 / Loss  203.2947235107422\n",
      "fps: 0.8044593816775695\n",
      "TIMESTEP 54693 / STATE explore / EPSILON 0.045462591999916674 / ACTION 1 / REWARD 0 / Q_MAX  149.91716 / Loss  62.13154983520508\n",
      "fps: 0.7942357133619053\n",
      "TIMESTEP 54694 / STATE explore / EPSILON 0.04546159299991667 / ACTION 1 / REWARD 0 / Q_MAX  147.80908 / Loss  20.038026809692383\n",
      "----------Random Action----------\n",
      "fps: 0.7818160602532572\n",
      "TIMESTEP 54695 / STATE explore / EPSILON 0.04546059399991667 / ACTION 0 / REWARD 0 / Q_MAX  133.02795 / Loss  135.30047607421875\n",
      "fps: 0.8038126397562904\n",
      "TIMESTEP 54696 / STATE explore / EPSILON 0.04545959499991667 / ACTION 3 / REWARD 0 / Q_MAX  140.76114 / Loss  52.06122589111328\n",
      "fps: 0.7769561937018501\n",
      "TIMESTEP 54697 / STATE explore / EPSILON 0.04545859599991667 / ACTION 3 / REWARD 0 / Q_MAX  131.19548 / Loss  30.392921447753906\n",
      "fps: 0.8051070106224577\n",
      "TIMESTEP 54698 / STATE explore / EPSILON 0.045457596999916666 / ACTION 1 / REWARD -0.1 / Q_MAX  128.0077 / Loss  144.44369506835938\n",
      "fps: 0.8109835643682666\n",
      "TIMESTEP 54699 / STATE explore / EPSILON 0.045456597999916665 / ACTION 2 / REWARD 0 / Q_MAX  145.47029 / Loss  131.4498291015625\n",
      "fps: 0.8038126397562904\n",
      "TIMESTEP 54700 / STATE explore / EPSILON 0.04545559899991666 / ACTION 1 / REWARD 0 / Q_MAX  163.5432 / Loss  178.56617736816406\n",
      "fps: 0.8044592273838481\n",
      "TIMESTEP 54701 / STATE explore / EPSILON 0.04545459999991666 / ACTION 1 / REWARD 0 / Q_MAX  150.25299 / Loss  66.23554992675781\n",
      "fps: 0.7830406509407755\n",
      "TIMESTEP 54702 / STATE explore / EPSILON 0.04545360099991666 / ACTION 2 / REWARD 0 / Q_MAX  150.04788 / Loss  152.42703247070312\n",
      "fps: 0.7818159145231819\n",
      "TIMESTEP 54703 / STATE explore / EPSILON 0.04545260199991666 / ACTION 1 / REWARD 0 / Q_MAX  151.50769 / Loss  63.53900909423828\n",
      "fps: 0.7967669638844153\n",
      "TIMESTEP 54704 / STATE explore / EPSILON 0.04545160299991666 / ACTION 2 / REWARD 0 / Q_MAX  145.5751 / Loss  50.337364196777344\n",
      "fps: 0.7936052990978834\n",
      "TIMESTEP 54705 / STATE explore / EPSILON 0.045450603999916656 / ACTION 2 / REWARD 0 / Q_MAX  158.38881 / Loss  61.05018615722656\n",
      "fps: 0.8012361907094285\n",
      "TIMESTEP 54706 / STATE explore / EPSILON 0.045449604999916654 / ACTION 3 / REWARD 0 / Q_MAX  153.25584 / Loss  14.326254844665527\n",
      "fps: 0.8070561690777502\n",
      "TIMESTEP 54707 / STATE explore / EPSILON 0.04544860599991665 / ACTION 3 / REWARD 0 / Q_MAX  151.05377 / Loss  149.5505828857422\n",
      "fps: 0.8038124857105569\n",
      "TIMESTEP 54708 / STATE explore / EPSILON 0.04544760699991665 / ACTION 1 / REWARD 0 / Q_MAX  141.90782 / Loss  58.351322174072266\n",
      "fps: 0.8064055568351665\n",
      "TIMESTEP 54709 / STATE explore / EPSILON 0.04544660799991665 / ACTION 1 / REWARD 0 / Q_MAX  137.73341 / Loss  27.043472290039062\n",
      "fps: 0.7974024132127265\n",
      "TIMESTEP 54710 / STATE explore / EPSILON 0.04544560899991665 / ACTION 2 / REWARD 0 / Q_MAX  180.51437 / Loss  97.56098937988281\n",
      "error:  could not convert string to float:  \n",
      "score:  \n",
      "fps: 0.748460271489014\n",
      "TIMESTEP 54711 / STATE explore / EPSILON 0.045444609999916646 / ACTION 1 / REWARD 5.0 / Q_MAX  142.49232 / Loss  84.30255889892578\n",
      "fps: 0.7842689372542442\n",
      "TIMESTEP 54712 / STATE explore / EPSILON 0.045443610999916645 / ACTION 0 / REWARD 0 / Q_MAX  135.39403 / Loss  50.88132095336914\n",
      "fps: 0.8070564796608223\n",
      "TIMESTEP 54713 / STATE explore / EPSILON 0.04544261199991664 / ACTION 0 / REWARD 0 / Q_MAX  151.93663 / Loss  47.00938415527344\n",
      "fps: 0.8038124857105569\n",
      "TIMESTEP 54714 / STATE explore / EPSILON 0.04544161299991664 / ACTION 1 / REWARD 0 / Q_MAX  177.06017 / Loss  102.10581970214844\n",
      "----------Random Action----------\n",
      "fps: 0.781816205983387\n",
      "TIMESTEP 54715 / STATE explore / EPSILON 0.04544061399991664 / ACTION 3 / REWARD 0 / Q_MAX  145.39314 / Loss  355.72705078125\n",
      "fps: 0.8070564796608223\n",
      "TIMESTEP 54716 / STATE explore / EPSILON 0.04543961499991664 / ACTION 3 / REWARD 0 / Q_MAX  159.77103 / Loss  141.45103454589844\n",
      "fps: 0.7917201960847234\n",
      "TIMESTEP 54717 / STATE explore / EPSILON 0.04543861599991664 / ACTION 0 / REWARD 0 / Q_MAX  167.99977 / Loss  77.25635528564453\n",
      "----------Random Action----------\n",
      "fps: 0.7867371064841083\n",
      "TIMESTEP 54718 / STATE explore / EPSILON 0.045437616999916636 / ACTION 2 / REWARD 0 / Q_MAX  82.18105 / Loss  53.781124114990234\n",
      "fps: 0.7775604243385098\n",
      "TIMESTEP 54719 / STATE explore / EPSILON 0.045436617999916634 / ACTION 1 / REWARD 0 / Q_MAX  125.176994 / Loss  106.33090209960938\n",
      "fps: 0.806405401793997\n",
      "TIMESTEP 54720 / STATE explore / EPSILON 0.04543561899991663 / ACTION 2 / REWARD 0 / Q_MAX  130.12302 / Loss  77.68724060058594\n",
      "fps: 0.7961326776439218\n",
      "TIMESTEP 54721 / STATE explore / EPSILON 0.04543461999991663 / ACTION 2 / REWARD 0 / Q_MAX  147.91312 / Loss  39.79928970336914\n",
      "fps: 0.7873565041652345\n",
      "TIMESTEP 54722 / STATE explore / EPSILON 0.04543362099991663 / ACTION 3 / REWARD 0 / Q_MAX  23.125664 / Loss  33.6078987121582\n",
      "fps: 0.8142856588169057\n",
      "TIMESTEP 54723 / STATE explore / EPSILON 0.04543262199991663 / ACTION 2 / REWARD 0 / Q_MAX  152.98253 / Loss  43.2866096496582\n",
      "fps: 0.7775604243385098\n",
      "TIMESTEP 54724 / STATE explore / EPSILON 0.04543162299991663 / ACTION 2 / REWARD 0 / Q_MAX  103.72892 / Loss  52.532840728759766\n",
      "fps: 0.8038124857105569\n",
      "TIMESTEP 54725 / STATE explore / EPSILON 0.045430623999916625 / ACTION 3 / REWARD 0 / Q_MAX  172.75188 / Loss  48.24341583251953\n",
      "fps: 0.773949500595182\n",
      "TIMESTEP 54726 / STATE explore / EPSILON 0.045429624999916624 / ACTION 3 / REWARD 0 / Q_MAX  142.71411 / Loss  158.96624755859375\n",
      "fps: 0.7680053906628372\n",
      "TIMESTEP 54727 / STATE explore / EPSILON 0.04542862599991662 / ACTION 0 / REWARD 0 / Q_MAX  87.3496 / Loss  260.65399169921875\n",
      "fps: 0.7775604243385098\n",
      "TIMESTEP 54728 / STATE explore / EPSILON 0.04542762699991662 / ACTION 2 / REWARD 0 / Q_MAX  165.42351 / Loss  55.882957458496094\n",
      "fps: 0.7873565041652345\n",
      "TIMESTEP 54729 / STATE explore / EPSILON 0.04542662799991662 / ACTION 3 / REWARD 0 / Q_MAX  93.401505 / Loss  461.3381042480469\n",
      "fps: 0.7168049700344602\n",
      "TIMESTEP 54730 / STATE explore / EPSILON 0.04542562899991662 / ACTION 3 / REWARD 0.0 / Q_MAX  144.7959 / Loss  265.9740295410156\n",
      "fps: 0.7855010830289753\n",
      "TIMESTEP 54731 / STATE explore / EPSILON 0.045424629999916616 / ACTION 1 / REWARD 0 / Q_MAX  -9.054618 / Loss  57.24909973144531\n",
      "fps: 0.759258059155565\n",
      "TIMESTEP 54732 / STATE explore / EPSILON 0.045423630999916614 / ACTION 0 / REWARD 0 / Q_MAX  151.70319 / Loss  80.33772277832031\n",
      "fps: 0.7751494096613504\n",
      "TIMESTEP 54733 / STATE explore / EPSILON 0.04542263199991661 / ACTION 3 / REWARD 0 / Q_MAX  54.747875 / Loss  294.23455810546875\n",
      "fps: 0.7473415083551321\n",
      "TIMESTEP 54734 / STATE explore / EPSILON 0.04542163299991661 / ACTION 1 / REWARD 0 / Q_MAX  143.02599 / Loss  147.24786376953125\n",
      "fps: 0.7967672665986848\n",
      "TIMESTEP 54735 / STATE explore / EPSILON 0.04542063399991661 / ACTION 3 / REWARD 0 / Q_MAX  118.18644 / Loss  91.88580322265625\n",
      "fps: 0.7830405047538166\n",
      "TIMESTEP 54736 / STATE explore / EPSILON 0.04541963499991661 / ACTION 3 / REWARD 0 / Q_MAX  44.528473 / Loss  185.90243530273438\n",
      "fps: 0.8064055568351665\n",
      "TIMESTEP 54737 / STATE explore / EPSILON 0.04541863599991661 / ACTION 2 / REWARD 0 / Q_MAX  149.96834 / Loss  57.729957580566406\n",
      "error:  could not convert string to float:  \n",
      "score:  \n",
      "fps: 0.7490208431999051\n",
      "TIMESTEP 54738 / STATE explore / EPSILON 0.045417636999916605 / ACTION 0 / REWARD 0.0 / Q_MAX  142.67442 / Loss  247.95986938476562\n",
      "fps: 0.7999542262520074\n",
      "TIMESTEP 54739 / STATE explore / EPSILON 0.045416637999916604 / ACTION 1 / REWARD 0 / Q_MAX  134.60542 / Loss  51.46759033203125\n",
      "fps: 0.7954994004789341\n",
      "TIMESTEP 54740 / STATE explore / EPSILON 0.0454156389999166 / ACTION 1 / REWARD 0 / Q_MAX  121.76636 / Loss  61.4824104309082\n",
      "fps: 0.7855010830289753\n",
      "TIMESTEP 54741 / STATE explore / EPSILON 0.0454146399999166 / ACTION 1 / REWARD 0 / Q_MAX  134.78188 / Loss  499.624267578125\n",
      "fps: 0.8044592273838481\n",
      "TIMESTEP 54742 / STATE explore / EPSILON 0.0454136409999166 / ACTION 1 / REWARD 0 / Q_MAX  12.712404 / Loss  221.50865173339844\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fps: 0.7581069449781496\n",
      "TIMESTEP 54743 / STATE explore / EPSILON 0.0454126419999166 / ACTION 2 / REWARD 0 / Q_MAX  136.68216 / Loss  98.01881408691406\n",
      "fps: 0.7495822552391002\n",
      "TIMESTEP 54744 / STATE explore / EPSILON 0.045411642999916596 / ACTION 2 / REWARD 0 / Q_MAX  8.401579 / Loss  61.450279235839844\n",
      "fps: 0.7942357133619053\n",
      "TIMESTEP 54745 / STATE explore / EPSILON 0.045410643999916595 / ACTION 3 / REWARD 0 / Q_MAX  134.95001 / Loss  57.80274200439453\n",
      "fps: 0.7830405047538166\n",
      "TIMESTEP 54746 / STATE explore / EPSILON 0.04540964499991659 / ACTION 1 / REWARD 0 / Q_MAX  5.956073 / Loss  66.1451416015625\n",
      "fps: 0.7818160602532572\n",
      "TIMESTEP 54747 / STATE explore / EPSILON 0.04540864599991659 / ACTION 1 / REWARD 0 / Q_MAX  139.91927 / Loss  125.8819351196289\n",
      "error:  could not convert string to float: '\\\\... Me' \n",
      "score:  \\... Me\n",
      "fps: 0.7183497308096434\n",
      "TIMESTEP 54748 / STATE explore / EPSILON 0.04540764699991659 / ACTION 1 / REWARD 2.0 / Q_MAX  80.58055 / Loss  54.55659866333008\n",
      "error:  could not convert string to float:  \n",
      "score:  \n",
      "fps: 0.8031667830925157\n",
      "TIMESTEP 54749 / STATE explore / EPSILON 0.04540664799991659 / ACTION 1 / REWARD 0 / Q_MAX  142.68248 / Loss  95.00540161132812\n",
      "fps: 0.797402564811354\n",
      "TIMESTEP 54750 / STATE explore / EPSILON 0.04540564899991659 / ACTION 3 / REWARD -0.1 / Q_MAX  96.191765 / Loss  95.33033752441406\n",
      "fps: 0.8077082986301344\n",
      "TIMESTEP 54751 / STATE explore / EPSILON 0.045404649999916585 / ACTION 1 / REWARD -0.1 / Q_MAX  171.06378 / Loss  73.23165893554688\n",
      "fps: 0.8012361907094285\n",
      "TIMESTEP 54752 / STATE explore / EPSILON 0.045403650999916584 / ACTION 2 / REWARD -0.1 / Q_MAX  132.90045 / Loss  75.30009460449219\n",
      "fps: 0.7769563376258573\n",
      "TIMESTEP 54753 / STATE explore / EPSILON 0.04540265199991658 / ACTION 3 / REWARD -0.1 / Q_MAX  131.51341 / Loss  231.9080810546875\n",
      "fps: 0.7867371064841083\n",
      "TIMESTEP 54754 / STATE explore / EPSILON 0.04540165299991658 / ACTION 1 / REWARD -0.1 / Q_MAX  144.88998 / Loss  79.03207397460938\n",
      "fps: 0.7586819282003194\n",
      "TIMESTEP 54755 / STATE explore / EPSILON 0.04540065399991658 / ACTION 1 / REWARD 0 / Q_MAX  121.37128 / Loss  57.70475769042969\n",
      "fps: 0.797402564811354\n",
      "TIMESTEP 54756 / STATE explore / EPSILON 0.04539965499991658 / ACTION 1 / REWARD 0 / Q_MAX  134.26535 / Loss  52.2965087890625\n",
      "fps: 0.7680052500358889\n",
      "TIMESTEP 54757 / STATE explore / EPSILON 0.045398655999916576 / ACTION 1 / REWARD 0 / Q_MAX  147.72482 / Loss  142.69342041015625\n",
      "fps: 0.7633150513762607\n",
      "TIMESTEP 54758 / STATE explore / EPSILON 0.045397656999916575 / ACTION 0 / REWARD 0 / Q_MAX  126.812485 / Loss  65.1463394165039\n",
      "fps: 0.7848843062585963\n",
      "TIMESTEP 54759 / STATE explore / EPSILON 0.04539665799991657 / ACTION 1 / REWARD 0 / Q_MAX  102.30422 / Loss  202.2288055419922\n",
      "fps: 0.7721566572789729\n",
      "TIMESTEP 54760 / STATE explore / EPSILON 0.04539565899991657 / ACTION 1 / REWARD 0 / Q_MAX  83.95547 / Loss  118.50421905517578\n",
      "fps: 0.7910940656590661\n",
      "TIMESTEP 54761 / STATE explore / EPSILON 0.04539465999991657 / ACTION 3 / REWARD 0 / Q_MAX  140.51271 / Loss  44.03609085083008\n",
      "fps: 0.8070563243692563\n",
      "TIMESTEP 54762 / STATE explore / EPSILON 0.04539366099991657 / ACTION 1 / REWARD 0 / Q_MAX  135.0058 / Loss  265.6217041015625\n",
      "fps: 0.7830406509407755\n",
      "TIMESTEP 54763 / STATE explore / EPSILON 0.04539266199991657 / ACTION 3 / REWARD 0 / Q_MAX  161.65895 / Loss  59.64594268798828\n",
      "fps: 0.7818160602532572\n",
      "TIMESTEP 54764 / STATE explore / EPSILON 0.045391662999916566 / ACTION 3 / REWARD 0 / Q_MAX  122.55001 / Loss  71.96372985839844\n",
      "fps: 0.8038126397562904\n",
      "TIMESTEP 54765 / STATE explore / EPSILON 0.045390663999916564 / ACTION 2 / REWARD 0 / Q_MAX  135.6033 / Loss  24.27615737915039\n",
      "fps: 0.7769561937018501\n",
      "TIMESTEP 54766 / STATE explore / EPSILON 0.04538966499991656 / ACTION 1 / REWARD -0.1 / Q_MAX  140.37517 / Loss  258.2754821777344\n",
      "fps: 0.8109838779814887\n",
      "TIMESTEP 54767 / STATE explore / EPSILON 0.04538866599991656 / ACTION 0 / REWARD 0 / Q_MAX  127.832275 / Loss  56.11345672607422\n",
      "fps: 0.8051070106224577\n",
      "TIMESTEP 54768 / STATE explore / EPSILON 0.04538766699991656 / ACTION 0 / REWARD 0 / Q_MAX  137.16881 / Loss  21.93338966369629\n",
      "fps: 0.7929761846468326\n",
      "TIMESTEP 54769 / STATE explore / EPSILON 0.04538666799991656 / ACTION 2 / REWARD 0 / Q_MAX  139.31 / Loss  37.248199462890625\n",
      "fps: 0.8031669368908586\n",
      "TIMESTEP 54770 / STATE explore / EPSILON 0.045385668999916556 / ACTION 1 / REWARD 0 / Q_MAX  139.23125 / Loss  85.26896667480469\n",
      "fps: 0.8038126397562904\n",
      "TIMESTEP 54771 / STATE explore / EPSILON 0.045384669999916555 / ACTION 1 / REWARD 0 / Q_MAX  129.76106 / Loss  81.61280059814453\n",
      "fps: 0.8012361907094285\n",
      "TIMESTEP 54772 / STATE explore / EPSILON 0.04538367099991655 / ACTION 2 / REWARD 0 / Q_MAX  147.80956 / Loss  152.9677734375\n",
      "fps: 0.7873566519681551\n",
      "TIMESTEP 54773 / STATE explore / EPSILON 0.04538267199991655 / ACTION 2 / REWARD 0 / Q_MAX  159.92184 / Loss  65.48360443115234\n",
      "fps: 0.7662396691195277\n",
      "TIMESTEP 54774 / STATE explore / EPSILON 0.04538167299991655 / ACTION 2 / REWARD 0 / Q_MAX  124.5921 / Loss  264.6249084472656\n",
      "fps: 0.8057556831531081\n",
      "TIMESTEP 54775 / STATE explore / EPSILON 0.04538067399991655 / ACTION 0 / REWARD 0 / Q_MAX  164.41565 / Loss  55.60335159301758\n",
      "fps: 0.8044590730901859\n",
      "TIMESTEP 54776 / STATE explore / EPSILON 0.04537967499991655 / ACTION 1 / REWARD 0 / Q_MAX  131.97351 / Loss  138.70156860351562\n",
      "fps: 0.793605449256167\n",
      "TIMESTEP 54777 / STATE explore / EPSILON 0.045378675999916546 / ACTION 0 / REWARD 0 / Q_MAX  139.06702 / Loss  80.61408233642578\n",
      "fps: 0.8038124857105569\n",
      "TIMESTEP 54778 / STATE explore / EPSILON 0.045377676999916544 / ACTION 3 / REWARD 0 / Q_MAX  94.57664 / Loss  87.65172576904297\n",
      "fps: 0.7942357133619053\n",
      "TIMESTEP 54779 / STATE explore / EPSILON 0.04537667799991654 / ACTION 3 / REWARD -0.1 / Q_MAX  144.50485 / Loss  48.11616897583008\n",
      "fps: 0.7812053456350289\n",
      "TIMESTEP 54780 / STATE explore / EPSILON 0.04537567899991654 / ACTION 1 / REWARD 0 / Q_MAX  143.9767 / Loss  32.07466506958008\n",
      "fps: 0.7942357133619053\n",
      "TIMESTEP 54781 / STATE explore / EPSILON 0.04537467999991654 / ACTION 0 / REWARD 0 / Q_MAX  169.36008 / Loss  256.18072509765625\n",
      "fps: 0.7961329798764166\n",
      "TIMESTEP 54782 / STATE explore / EPSILON 0.04537368099991654 / ACTION 3 / REWARD 0 / Q_MAX  116.26259 / Loss  154.86032104492188\n",
      "fps: 0.7818160602532572\n",
      "TIMESTEP 54783 / STATE explore / EPSILON 0.045372681999916536 / ACTION 1 / REWARD 0 / Q_MAX  161.35997 / Loss  105.41670227050781\n",
      "fps: 0.8012361907094285\n",
      "TIMESTEP 54784 / STATE explore / EPSILON 0.045371682999916535 / ACTION 3 / REWARD 0 / Q_MAX  150.60889 / Loss  116.61653900146484\n",
      "fps: 0.7855012301361156\n",
      "TIMESTEP 54785 / STATE explore / EPSILON 0.04537068399991653 / ACTION 0 / REWARD 0 / Q_MAX  141.58516 / Loss  38.61929702758789\n",
      "fps: 0.8038126397562904\n",
      "TIMESTEP 54786 / STATE explore / EPSILON 0.04536968499991653 / ACTION 0 / REWARD 0 / Q_MAX  90.43889 / Loss  49.441795349121094\n",
      "fps: 0.8142856588169057\n",
      "TIMESTEP 54787 / STATE explore / EPSILON 0.04536868599991653 / ACTION 1 / REWARD 0 / Q_MAX  128.30481 / Loss  123.79779052734375\n",
      "fps: 0.8012363437692821\n",
      "TIMESTEP 54788 / STATE explore / EPSILON 0.04536768699991653 / ACTION 1 / REWARD 0 / Q_MAX  20.753078 / Loss  73.06471252441406\n",
      "fps: 0.8116419018367631\n",
      "TIMESTEP 54789 / STATE explore / EPSILON 0.04536668799991653 / ACTION 1 / REWARD 0 / Q_MAX  134.53217 / Loss  198.60150146484375\n",
      "fps: 0.8038126397562904\n",
      "TIMESTEP 54790 / STATE explore / EPSILON 0.045365688999916526 / ACTION 2 / REWARD 0 / Q_MAX  132.34361 / Loss  99.46087646484375\n",
      "----------Random Action----------\n",
      "fps: 0.7910939164496377\n",
      "TIMESTEP 54791 / STATE explore / EPSILON 0.045364689999916524 / ACTION 2 / REWARD 0 / Q_MAX  139.28407 / Loss  74.33255767822266\n",
      "fps: 0.8012361907094285\n",
      "TIMESTEP 54792 / STATE explore / EPSILON 0.04536369099991652 / ACTION 0 / REWARD 0 / Q_MAX  146.56062 / Loss  137.44659423828125\n",
      "fps: 0.8156140064649351\n",
      "TIMESTEP 54793 / STATE explore / EPSILON 0.04536269199991652 / ACTION 1 / REWARD 0 / Q_MAX  142.30505 / Loss  49.15087127685547\n",
      "fps: 0.8051068560802085\n",
      "TIMESTEP 54794 / STATE explore / EPSILON 0.04536169299991652 / ACTION 2 / REWARD 0 / Q_MAX  116.19289 / Loss  52.47096633911133\n",
      "fps: 0.8077079875451751\n",
      "TIMESTEP 54795 / STATE explore / EPSILON 0.04536069399991652 / ACTION 3 / REWARD 0 / Q_MAX  131.67664 / Loss  368.50347900390625\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fps: 0.8044592273838481\n",
      "TIMESTEP 54796 / STATE explore / EPSILON 0.04535969499991652 / ACTION 0 / REWARD 0 / Q_MAX  113.665344 / Loss  56.67031478881836\n",
      "fps: 0.8077082986301344\n",
      "TIMESTEP 54797 / STATE explore / EPSILON 0.045358695999916515 / ACTION 1 / REWARD 0 / Q_MAX  134.6278 / Loss  45.565616607666016\n",
      "fps: 0.8012361907094285\n",
      "TIMESTEP 54798 / STATE explore / EPSILON 0.045357696999916514 / ACTION 3 / REWARD 0 / Q_MAX  140.19362 / Loss  26.787315368652344\n",
      "fps: 0.7745490619291485\n",
      "TIMESTEP 54799 / STATE explore / EPSILON 0.04535669799991651 / ACTION 1 / REWARD 0 / Q_MAX  155.41223 / Loss  280.5610046386719\n",
      "fps: 0.7954994004789341\n",
      "TIMESTEP 54800 / STATE explore / EPSILON 0.04535569899991651 / ACTION 1 / REWARD 0 / Q_MAX  157.18658 / Loss  363.8948974609375\n",
      "fps: 0.7848844531348098\n",
      "TIMESTEP 54801 / STATE explore / EPSILON 0.04535469999991651 / ACTION 0 / REWARD 0 / Q_MAX  17.8597 / Loss  54.37299346923828\n",
      "fps: 0.8136230273156945\n",
      "TIMESTEP 54802 / STATE explore / EPSILON 0.04535370099991651 / ACTION 0 / REWARD 0 / Q_MAX  138.77383 / Loss  66.98519897460938\n",
      "fps: 0.7936052990978834\n",
      "TIMESTEP 54803 / STATE explore / EPSILON 0.045352701999916506 / ACTION 2 / REWARD 0 / Q_MAX  115.59026 / Loss  75.77613830566406\n",
      "fps: 0.8057556831531081\n",
      "TIMESTEP 54804 / STATE explore / EPSILON 0.045351702999916504 / ACTION 0 / REWARD 0 / Q_MAX  121.920235 / Loss  306.5647277832031\n",
      "fps: 0.7910939164496377\n",
      "TIMESTEP 54805 / STATE explore / EPSILON 0.0453507039999165 / ACTION 2 / REWARD 0 / Q_MAX  -4.194455 / Loss  86.82530975341797\n",
      "fps: 0.7512718213958314\n",
      "TIMESTEP 54806 / STATE explore / EPSILON 0.0453497049999165 / ACTION 0 / REWARD 25.0 / Q_MAX  34.545025 / Loss  147.8421173095703\n",
      "fps: 0.8083611713333582\n",
      "TIMESTEP 54807 / STATE explore / EPSILON 0.0453487059999165 / ACTION 2 / REWARD 0 / Q_MAX  122.55274 / Loss  133.9217987060547\n",
      "fps: 0.8136230273156945\n",
      "TIMESTEP 54808 / STATE explore / EPSILON 0.0453477069999165 / ACTION 2 / REWARD 0 / Q_MAX  137.6041 / Loss  66.5751724243164\n",
      "fps: 0.7507078765992999\n",
      "TIMESTEP 54809 / STATE explore / EPSILON 0.0453467079999165 / ACTION 1 / REWARD 10.0 / Q_MAX  136.65376 / Loss  46.79472732543945\n",
      "fps: 0.8103264505656514\n",
      "TIMESTEP 54810 / STATE explore / EPSILON 0.045345708999916495 / ACTION 2 / REWARD 0 / Q_MAX  130.35957 / Loss  62.1760139465332\n",
      "fps: 0.7917203455304925\n",
      "TIMESTEP 54811 / STATE explore / EPSILON 0.045344709999916494 / ACTION 1 / REWARD 0 / Q_MAX  125.75013 / Loss  64.07841491699219\n",
      "fps: 0.7775604243385098\n",
      "TIMESTEP 54812 / STATE explore / EPSILON 0.04534371099991649 / ACTION 0 / REWARD 0 / Q_MAX  119.78852 / Loss  49.00825881958008\n",
      "fps: 0.8025222705792627\n",
      "TIMESTEP 54813 / STATE explore / EPSILON 0.04534271199991649 / ACTION 2 / REWARD 0 / Q_MAX  143.23929 / Loss  27.696552276611328\n",
      "fps: 0.7929760347265811\n",
      "TIMESTEP 54814 / STATE explore / EPSILON 0.04534171299991649 / ACTION 2 / REWARD 0 / Q_MAX  110.8884 / Loss  56.79779052734375\n",
      "fps: 0.7836542395909898\n",
      "TIMESTEP 54815 / STATE explore / EPSILON 0.04534071399991649 / ACTION 0 / REWARD 0 / Q_MAX  105.855965 / Loss  29.566625595092773\n",
      "fps: 0.8018786383275178\n",
      "TIMESTEP 54816 / STATE explore / EPSILON 0.045339714999916486 / ACTION 2 / REWARD 0 / Q_MAX  50.5349 / Loss  147.26373291015625\n",
      "fps: 0.7936052990978834\n",
      "TIMESTEP 54817 / STATE explore / EPSILON 0.045338715999916485 / ACTION 1 / REWARD 0 / Q_MAX  137.89725 / Loss  26.923635482788086\n",
      "fps: 0.8038124857105569\n",
      "TIMESTEP 54818 / STATE explore / EPSILON 0.04533771699991648 / ACTION 0 / REWARD 0 / Q_MAX  106.269356 / Loss  267.13134765625\n",
      "fps: 0.8142856588169057\n",
      "TIMESTEP 54819 / STATE explore / EPSILON 0.04533671799991648 / ACTION 2 / REWARD 0 / Q_MAX  43.295963 / Loss  74.02239990234375\n",
      "error:  could not convert string to float: ', ToS' \n",
      "score:  , ToS\n",
      "fps: 0.7627328880804256\n",
      "TIMESTEP 54820 / STATE explore / EPSILON 0.04533571899991648 / ACTION 2 / REWARD 6.0 / Q_MAX  118.85654 / Loss  35.944618225097656\n",
      "error:  could not convert string to float:  \n",
      "score:  \n",
      "fps: 0.8488478343097884\n",
      "TIMESTEP 54821 / STATE explore / EPSILON 0.04533471999991648 / ACTION 1 / REWARD 0 / Q_MAX  139.43138 / Loss  82.14849090576172\n",
      "fps: 0.8025222705792627\n",
      "TIMESTEP 54822 / STATE explore / EPSILON 0.04533372099991648 / ACTION 1 / REWARD 0 / Q_MAX  103.770775 / Loss  242.71458435058594\n",
      "fps: 0.8136231851446356\n",
      "TIMESTEP 54823 / STATE explore / EPSILON 0.045332721999916475 / ACTION 2 / REWARD 0 / Q_MAX  25.315952 / Loss  53.752220153808594\n",
      "fps: 0.8025224241308694\n",
      "TIMESTEP 54824 / STATE explore / EPSILON 0.045331722999916474 / ACTION 2 / REWARD 0 / Q_MAX  18.105295 / Loss  78.26899719238281\n",
      "fps: 0.8044592273838481\n",
      "TIMESTEP 54825 / STATE explore / EPSILON 0.04533072399991647 / ACTION 1 / REWARD 0 / Q_MAX  126.134384 / Loss  42.58373260498047\n",
      "error:  could not convert string to float: '-a' \n",
      "score:  -a\n",
      "fps: 0.7272312408658383\n",
      "TIMESTEP 54826 / STATE explore / EPSILON 0.04532972499991647 / ACTION 2 / REWARD 8.0 / Q_MAX  134.39014 / Loss  42.36474609375\n",
      "----------Random Action----------\n",
      "fps: 0.8083613271274802\n",
      "TIMESTEP 54827 / STATE explore / EPSILON 0.04532872599991647 / ACTION 2 / REWARD 0 / Q_MAX  44.435688 / Loss  116.78706359863281\n",
      "fps: 0.8142856588169057\n",
      "TIMESTEP 54828 / STATE explore / EPSILON 0.04532772699991647 / ACTION 3 / REWARD 0 / Q_MAX  60.89148 / Loss  44.386444091796875\n",
      "fps: 0.8044590730901859\n",
      "TIMESTEP 54829 / STATE explore / EPSILON 0.045326727999916466 / ACTION 3 / REWARD 0 / Q_MAX  117.77239 / Loss  45.76911163330078\n",
      "fps: 0.7842687906083047\n",
      "TIMESTEP 54830 / STATE explore / EPSILON 0.045325728999916465 / ACTION 3 / REWARD 0 / Q_MAX  110.60407 / Loss  52.722686767578125\n",
      "fps: 0.8012361907094285\n",
      "TIMESTEP 54831 / STATE explore / EPSILON 0.04532472999991646 / ACTION 0 / REWARD 0 / Q_MAX  111.48798 / Loss  101.16836547851562\n",
      "fps: 0.814285500730843\n",
      "TIMESTEP 54832 / STATE explore / EPSILON 0.04532373099991646 / ACTION 3 / REWARD 0 / Q_MAX  145.41539 / Loss  114.56047821044922\n",
      "fps: 0.7633151902908395\n",
      "TIMESTEP 54833 / STATE explore / EPSILON 0.04532273199991646 / ACTION 0 / REWARD 0 / Q_MAX  123.05923 / Loss  247.38470458984375\n",
      "fps: 0.7923477674754426\n",
      "TIMESTEP 54834 / STATE explore / EPSILON 0.04532173299991646 / ACTION 0 / REWARD 0 / Q_MAX  140.91135 / Loss  54.86698913574219\n",
      "fps: 0.7558146114288041\n",
      "TIMESTEP 54835 / STATE explore / EPSILON 0.04532073399991646 / ACTION 1 / REWARD 1.0 / Q_MAX  135.85712 / Loss  234.0355987548828\n",
      "fps: 0.7999542262520074\n",
      "TIMESTEP 54836 / STATE explore / EPSILON 0.045319734999916456 / ACTION 1 / REWARD 0 / Q_MAX  86.73581 / Loss  74.83550262451172\n",
      "fps: 0.7501445094975648\n",
      "TIMESTEP 54837 / STATE explore / EPSILON 0.045318735999916454 / ACTION 3 / REWARD 0 / Q_MAX  133.30571 / Loss  86.50159454345703\n",
      "fps: 0.798039028776602\n",
      "TIMESTEP 54838 / STATE explore / EPSILON 0.04531773699991645 / ACTION 2 / REWARD 0 / Q_MAX  86.22309 / Loss  39.530006408691406\n",
      "fps: 0.8156140064649351\n",
      "TIMESTEP 54839 / STATE explore / EPSILON 0.04531673799991645 / ACTION 1 / REWARD 0 / Q_MAX  120.63538 / Loss  63.82599639892578\n",
      "fps: 0.7923477674754426\n",
      "TIMESTEP 54840 / STATE explore / EPSILON 0.04531573899991645 / ACTION 3 / REWARD -0.1 / Q_MAX  146.051 / Loss  56.27145767211914\n",
      "fps: 0.8025222705792627\n",
      "TIMESTEP 54841 / STATE explore / EPSILON 0.04531473999991645 / ACTION 0 / REWARD 0 / Q_MAX  52.026413 / Loss  68.69895935058594\n",
      "fps: 0.7961328287601405\n",
      "TIMESTEP 54842 / STATE explore / EPSILON 0.045313740999916446 / ACTION 0 / REWARD 0 / Q_MAX  134.53102 / Loss  308.1412048339844\n",
      "fps: 0.7910940656590661\n",
      "TIMESTEP 54843 / STATE explore / EPSILON 0.045312741999916445 / ACTION 1 / REWARD 0 / Q_MAX  122.11475 / Loss  106.58523559570312\n",
      "fps: 0.7830405047538166\n",
      "TIMESTEP 54844 / STATE explore / EPSILON 0.04531174299991644 / ACTION 1 / REWARD 0 / Q_MAX  138.65932 / Loss  87.16300964355469\n",
      "fps: 0.7954995513548401\n",
      "TIMESTEP 54845 / STATE explore / EPSILON 0.04531074399991644 / ACTION 2 / REWARD 0 / Q_MAX  109.32774 / Loss  130.5294189453125\n",
      "fps: 0.8142858169030299\n",
      "TIMESTEP 54846 / STATE explore / EPSILON 0.04530974499991644 / ACTION 2 / REWARD 0 / Q_MAX  129.75304 / Loss  87.97750091552734\n",
      "fps: 0.8064055568351665\n",
      "TIMESTEP 54847 / STATE explore / EPSILON 0.04530874599991644 / ACTION 2 / REWARD 0 / Q_MAX  153.97015 / Loss  57.36882400512695\n",
      "fps: 0.7861186825725772\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TIMESTEP 54848 / STATE explore / EPSILON 0.04530774699991644 / ACTION 0 / REWARD 0 / Q_MAX  120.71946 / Loss  141.75942993164062\n",
      "fps: 0.7948672806195498\n",
      "TIMESTEP 54849 / STATE explore / EPSILON 0.045306747999916436 / ACTION 0 / REWARD 0 / Q_MAX  117.02642 / Loss  100.18280029296875\n",
      "fps: 0.8025224241308694\n",
      "TIMESTEP 54850 / STATE explore / EPSILON 0.045305748999916434 / ACTION 0 / REWARD 0 / Q_MAX  154.07104 / Loss  100.4289321899414\n",
      "fps: 0.7961329798764166\n",
      "TIMESTEP 54851 / STATE explore / EPSILON 0.04530474999991643 / ACTION 0 / REWARD 0 / Q_MAX  134.84596 / Loss  72.66163635253906\n",
      "fps: 0.7368774087547832\n",
      "TIMESTEP 54852 / STATE explore / EPSILON 0.04530375099991643 / ACTION 3 / REWARD 13.0 / Q_MAX  144.68181 / Loss  171.624267578125\n",
      "fps: 0.7986763574847822\n",
      "TIMESTEP 54853 / STATE explore / EPSILON 0.04530275199991643 / ACTION 0 / REWARD 0 / Q_MAX  166.95311 / Loss  37.17001724243164\n",
      "fps: 0.7541045738143671\n",
      "TIMESTEP 54854 / STATE explore / EPSILON 0.04530175299991643 / ACTION 1 / REWARD 0 / Q_MAX  124.24673 / Loss  62.088226318359375\n",
      "fps: 0.7272311147745522\n",
      "TIMESTEP 54855 / STATE explore / EPSILON 0.04530075399991643 / ACTION 0 / REWARD -3.0 / Q_MAX  138.27705 / Loss  68.84989929199219\n",
      "fps: 0.7954995513548401\n",
      "TIMESTEP 54856 / STATE explore / EPSILON 0.045299754999916425 / ACTION 1 / REWARD 0 / Q_MAX  153.84056 / Loss  113.73480987548828\n",
      "fps: 0.8038126397562904\n",
      "TIMESTEP 54857 / STATE explore / EPSILON 0.045298755999916424 / ACTION 2 / REWARD 0 / Q_MAX  10.925784 / Loss  222.01756286621094\n",
      "fps: 0.7615710831682837\n",
      "TIMESTEP 54858 / STATE explore / EPSILON 0.04529775699991642 / ACTION 0 / REWARD 9.0 / Q_MAX  137.60114 / Loss  73.80480194091797\n",
      "fps: 0.809670244477982\n",
      "TIMESTEP 54859 / STATE explore / EPSILON 0.04529675799991642 / ACTION 3 / REWARD 0 / Q_MAX  153.8895 / Loss  48.449405670166016\n",
      "fps: 0.7942357133619053\n",
      "TIMESTEP 54860 / STATE explore / EPSILON 0.04529575899991642 / ACTION 2 / REWARD 0 / Q_MAX  139.91968 / Loss  120.46184539794922\n",
      "fps: 0.7674159133032903\n",
      "TIMESTEP 54861 / STATE explore / EPSILON 0.04529475999991642 / ACTION 1 / REWARD 11.0 / Q_MAX  59.91333 / Loss  172.49842834472656\n",
      "fps: 0.778771709426688\n",
      "TIMESTEP 54862 / STATE explore / EPSILON 0.045293760999916416 / ACTION 0 / REWARD 0 / Q_MAX  150.67734 / Loss  46.10015106201172\n",
      "fps: 0.7942355629650221\n",
      "TIMESTEP 54863 / STATE explore / EPSILON 0.045292761999916414 / ACTION 3 / REWARD 0 / Q_MAX  135.56195 / Loss  88.02638244628906\n",
      "fps: 0.798039028776602\n",
      "TIMESTEP 54864 / STATE explore / EPSILON 0.04529176299991641 / ACTION 2 / REWARD 0 / Q_MAX  134.11734 / Loss  37.17179870605469\n",
      "fps: 0.8038124857105569\n",
      "TIMESTEP 54865 / STATE explore / EPSILON 0.04529076399991641 / ACTION 1 / REWARD 0 / Q_MAX  125.30965 / Loss  92.66104125976562\n",
      "fps: 0.8051070106224577\n",
      "TIMESTEP 54866 / STATE explore / EPSILON 0.04528976499991641 / ACTION 1 / REWARD 0 / Q_MAX  158.61766 / Loss  40.257999420166016\n",
      "fps: 0.7721566572789729\n",
      "TIMESTEP 54867 / STATE explore / EPSILON 0.04528876599991641 / ACTION 2 / REWARD 0 / Q_MAX  160.11388 / Loss  258.2274475097656\n",
      "fps: 0.7818159145231819\n",
      "TIMESTEP 54868 / STATE explore / EPSILON 0.04528776699991641 / ACTION 3 / REWARD 0 / Q_MAX  70.09306 / Loss  518.722900390625\n",
      "fps: 0.8025221170277148\n",
      "TIMESTEP 54869 / STATE explore / EPSILON 0.045286767999916405 / ACTION 2 / REWARD 0 / Q_MAX  135.93831 / Loss  336.1700134277344\n",
      "----------Random Action----------\n",
      "fps: 0.8038126397562904\n",
      "TIMESTEP 54870 / STATE explore / EPSILON 0.045285768999916404 / ACTION 2 / REWARD 0 / Q_MAX  149.92255 / Loss  35.9339485168457\n",
      "----------Random Action----------\n",
      "fps: 0.7974027164100391\n",
      "TIMESTEP 54871 / STATE explore / EPSILON 0.0452847699999164 / ACTION 0 / REWARD 0 / Q_MAX  143.96985 / Loss  60.7988395690918\n",
      "----------Random Action----------\n",
      "fps: 0.7910937672402655\n",
      "TIMESTEP 54872 / STATE explore / EPSILON 0.0452837709999164 / ACTION 0 / REWARD 0 / Q_MAX  174.69199 / Loss  84.87474060058594\n",
      "fps: 0.7942355629650221\n",
      "TIMESTEP 54873 / STATE explore / EPSILON 0.0452827719999164 / ACTION 0 / REWARD 0 / Q_MAX  148.23143 / Loss  40.65032958984375\n",
      "fps: 0.7967671152415213\n",
      "TIMESTEP 54874 / STATE explore / EPSILON 0.0452817729999164 / ACTION 2 / REWARD 0 / Q_MAX  157.21385 / Loss  297.7962646484375\n",
      "fps: 0.7867369589136989\n",
      "TIMESTEP 54875 / STATE explore / EPSILON 0.045280773999916396 / ACTION 0 / REWARD 0 / Q_MAX  158.54852 / Loss  39.88317108154297\n",
      "fps: 0.7473415083551321\n",
      "TIMESTEP 54876 / STATE explore / EPSILON 0.045279774999916395 / ACTION 3 / REWARD 3.0 / Q_MAX  114.382065 / Loss  52.89334487915039\n",
      "fps: 0.7873566519681551\n",
      "TIMESTEP 54877 / STATE explore / EPSILON 0.04527877599991639 / ACTION 2 / REWARD 0 / Q_MAX  148.17502 / Loss  35.26103591918945\n",
      "fps: 0.7787715648293334\n",
      "TIMESTEP 54878 / STATE explore / EPSILON 0.04527777699991639 / ACTION 1 / REWARD 0 / Q_MAX  117.3445 / Loss  66.34480285644531\n",
      "fps: 0.7812053456350289\n",
      "TIMESTEP 54879 / STATE explore / EPSILON 0.04527677799991639 / ACTION 3 / REWARD 0 / Q_MAX  145.38866 / Loss  57.310203552246094\n",
      "fps: 0.8012361907094285\n",
      "TIMESTEP 54880 / STATE explore / EPSILON 0.04527577899991639 / ACTION 0 / REWARD 0 / Q_MAX  134.47578 / Loss  47.27727127075195\n",
      "fps: 0.7842689372542442\n",
      "TIMESTEP 54881 / STATE explore / EPSILON 0.04527477999991639 / ACTION 1 / REWARD 0 / Q_MAX  114.30121 / Loss  41.14002227783203\n",
      "----------Random Action----------\n",
      "fps: 0.754104709396769\n",
      "TIMESTEP 54882 / STATE explore / EPSILON 0.045273780999916385 / ACTION 1 / REWARD 0 / Q_MAX  75.87975 / Loss  29.119266510009766\n",
      "fps: 0.8025224241308694\n",
      "TIMESTEP 54883 / STATE explore / EPSILON 0.045272781999916384 / ACTION 3 / REWARD 0 / Q_MAX  146.25252 / Loss  68.6033935546875\n",
      "fps: 0.8044592273838481\n",
      "TIMESTEP 54884 / STATE explore / EPSILON 0.04527178299991638 / ACTION 1 / REWARD 0 / Q_MAX  157.701 / Loss  46.40530014038086\n",
      "fps: 0.7836543860071966\n",
      "TIMESTEP 54885 / STATE explore / EPSILON 0.04527078399991638 / ACTION 0 / REWARD 0 / Q_MAX  100.05016 / Loss  125.56507873535156\n",
      "fps: 0.8012361907094285\n",
      "TIMESTEP 54886 / STATE explore / EPSILON 0.04526978499991638 / ACTION 3 / REWARD 0 / Q_MAX  153.15544 / Loss  207.51087951660156\n",
      "fps: 0.8012361907094285\n",
      "TIMESTEP 54887 / STATE explore / EPSILON 0.04526878599991638 / ACTION 1 / REWARD 0 / Q_MAX  147.63913 / Loss  508.310302734375\n",
      "fps: 0.8149495288635212\n",
      "TIMESTEP 54888 / STATE explore / EPSILON 0.045267786999916376 / ACTION 1 / REWARD 0 / Q_MAX  143.13887 / Loss  155.3936004638672\n",
      "fps: 0.8057558379445455\n",
      "TIMESTEP 54889 / STATE explore / EPSILON 0.045266787999916375 / ACTION 1 / REWARD 0 / Q_MAX  135.14896 / Loss  87.37124633789062\n",
      "error:  could not convert string to float: '\\\\.. Ma' \n",
      "score:  \\.. Ma\n",
      "fps: 0.7363348595579046\n",
      "TIMESTEP 54890 / STATE explore / EPSILON 0.04526578899991637 / ACTION 3 / REWARD 7.0 / Q_MAX  142.45146 / Loss  48.557472229003906\n",
      "fps: 0.7879768779177125\n",
      "TIMESTEP 54891 / STATE explore / EPSILON 0.04526478999991637 / ACTION 2 / REWARD 0 / Q_MAX  165.87285 / Loss  40.888492584228516\n",
      "error:  could not convert string to float: '\\\\,. ' \n",
      "score:  \\,. \n",
      "fps: 0.7374208877172588\n",
      "TIMESTEP 54892 / STATE explore / EPSILON 0.04526379099991637 / ACTION 1 / REWARD 7.0 / Q_MAX  99.1439 / Loss  56.74346160888672\n",
      "fps: 0.8012361907094285\n",
      "TIMESTEP 54893 / STATE explore / EPSILON 0.04526279199991637 / ACTION 1 / REWARD 0 / Q_MAX  132.37596 / Loss  367.1554260253906\n",
      "fps: 0.8176147535822634\n",
      "TIMESTEP 54894 / STATE explore / EPSILON 0.04526179299991637 / ACTION 3 / REWARD 0 / Q_MAX  158.19891 / Loss  97.63870239257812\n",
      "fps: 0.8038126397562904\n",
      "TIMESTEP 54895 / STATE explore / EPSILON 0.045260793999916366 / ACTION 3 / REWARD 0 / Q_MAX  160.01784 / Loss  169.6297607421875\n",
      "fps: 0.7873565041652345\n",
      "TIMESTEP 54896 / STATE explore / EPSILON 0.045259794999916364 / ACTION 0 / REWARD 0 / Q_MAX  76.77381 / Loss  167.75694274902344\n",
      "fps: 0.7967671152415213\n",
      "TIMESTEP 54897 / STATE explore / EPSILON 0.04525879599991636 / ACTION 3 / REWARD 0 / Q_MAX  147.06262 / Loss  237.03726196289062\n",
      "fps: 0.8025222705792627\n",
      "TIMESTEP 54898 / STATE explore / EPSILON 0.04525779699991636 / ACTION 1 / REWARD 0 / Q_MAX  153.64503 / Loss  44.22468566894531\n",
      "fps: 0.7942357133619053\n",
      "TIMESTEP 54899 / STATE explore / EPSILON 0.04525679799991636 / ACTION 3 / REWARD 0 / Q_MAX  162.61472 / Loss  68.50942993164062\n",
      "fps: 0.8070561690777502\n",
      "TIMESTEP 54900 / STATE explore / EPSILON 0.04525579899991636 / ACTION 1 / REWARD 0 / Q_MAX  164.45198 / Loss  233.13014221191406\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fps: 0.781816205983387\n",
      "TIMESTEP 54901 / STATE explore / EPSILON 0.045254799999916356 / ACTION 3 / REWARD 0 / Q_MAX  135.09564 / Loss  94.45152282714844\n",
      "fps: 0.8162797273893377\n",
      "TIMESTEP 54902 / STATE explore / EPSILON 0.045253800999916355 / ACTION 1 / REWARD 0 / Q_MAX  16.623724 / Loss  66.90069580078125\n",
      "fps: 0.7942357133619053\n",
      "TIMESTEP 54903 / STATE explore / EPSILON 0.04525280199991635 / ACTION 3 / REWARD 0 / Q_MAX  176.1976 / Loss  114.14372253417969\n",
      "fps: 0.8012361907094285\n",
      "TIMESTEP 54904 / STATE explore / EPSILON 0.04525180299991635 / ACTION 1 / REWARD -0.1 / Q_MAX  164.48697 / Loss  172.69964599609375\n",
      "fps: 0.7954992496030853\n",
      "TIMESTEP 54905 / STATE explore / EPSILON 0.04525080399991635 / ACTION 1 / REWARD -0.1 / Q_MAX  167.14023 / Loss  47.32605743408203\n",
      "fps: 0.8018787916329229\n",
      "TIMESTEP 54906 / STATE explore / EPSILON 0.04524980499991635 / ACTION 3 / REWARD -0.1 / Q_MAX  149.58644 / Loss  51.6119270324707\n",
      "fps: 0.7751494096613504\n",
      "TIMESTEP 54907 / STATE explore / EPSILON 0.04524880599991635 / ACTION 3 / REWARD -0.1 / Q_MAX  165.02681 / Loss  104.35352325439453\n",
      "fps: 0.7668271999526115\n",
      "TIMESTEP 54908 / STATE explore / EPSILON 0.045247806999916346 / ACTION 2 / REWARD -0.1 / Q_MAX  135.61652 / Loss  135.10765075683594\n",
      "fps: 0.7929761846468326\n",
      "TIMESTEP 54909 / STATE explore / EPSILON 0.045246807999916344 / ACTION 3 / REWARD -0.1 / Q_MAX  169.41689 / Loss  142.22238159179688\n",
      "fps: 0.7552436924886429\n",
      "TIMESTEP 54910 / STATE explore / EPSILON 0.04524580899991634 / ACTION 3 / REWARD 0 / Q_MAX  98.676704 / Loss  30.4020938873291\n",
      "fps: 0.8149495288635212\n",
      "TIMESTEP 54911 / STATE explore / EPSILON 0.04524480999991634 / ACTION 3 / REWARD 0 / Q_MAX  144.57872 / Loss  30.398658752441406\n",
      "fps: 0.8162798862506705\n",
      "TIMESTEP 54912 / STATE explore / EPSILON 0.04524381099991634 / ACTION 0 / REWARD 0 / Q_MAX  145.42456 / Loss  132.74095153808594\n",
      "fps: 0.8243552796285992\n",
      "TIMESTEP 54913 / STATE explore / EPSILON 0.04524281199991634 / ACTION 1 / REWARD 0 / Q_MAX  159.22482 / Loss  65.01459503173828\n",
      "fps: 0.8031669368908586\n",
      "TIMESTEP 54914 / STATE explore / EPSILON 0.04524181299991634 / ACTION 3 / REWARD 0 / Q_MAX  170.49184 / Loss  84.2321548461914\n",
      "fps: 0.8070563243692563\n",
      "TIMESTEP 54915 / STATE explore / EPSILON 0.045240813999916335 / ACTION 3 / REWARD 0 / Q_MAX  157.70094 / Loss  35.042442321777344\n",
      "fps: 0.8142856588169057\n",
      "TIMESTEP 54916 / STATE explore / EPSILON 0.045239814999916333 / ACTION 3 / REWARD 0 / Q_MAX  154.44048 / Loss  81.72645568847656\n",
      "fps: 0.8031669368908586\n",
      "TIMESTEP 54917 / STATE explore / EPSILON 0.04523881599991633 / ACTION 3 / REWARD 0 / Q_MAX  132.17317 / Loss  368.855712890625\n",
      "fps: 0.7942357133619053\n",
      "TIMESTEP 54918 / STATE explore / EPSILON 0.04523781699991633 / ACTION 3 / REWARD 0 / Q_MAX  148.784 / Loss  112.62860107421875\n",
      "fps: 0.8169465359517097\n",
      "TIMESTEP 54919 / STATE explore / EPSILON 0.04523681799991633 / ACTION 0 / REWARD 0 / Q_MAX  162.94398 / Loss  67.64756774902344\n",
      "fps: 0.8243549555884916\n",
      "TIMESTEP 54920 / STATE explore / EPSILON 0.04523581899991633 / ACTION 0 / REWARD 0 / Q_MAX  31.23053 / Loss  69.71112060546875\n",
      "fps: 0.8136231851446356\n",
      "TIMESTEP 54921 / STATE explore / EPSILON 0.045234819999916326 / ACTION 0 / REWARD 0 / Q_MAX  149.95468 / Loss  187.77188110351562\n",
      "fps: 0.8044592273838481\n",
      "TIMESTEP 54922 / STATE explore / EPSILON 0.045233820999916324 / ACTION 1 / REWARD 0 / Q_MAX  166.5193 / Loss  66.45732879638672\n",
      "fps: 0.7518367488876975\n",
      "TIMESTEP 54923 / STATE explore / EPSILON 0.04523282199991632 / ACTION 2 / REWARD 11.0 / Q_MAX  144.57968 / Loss  80.76980590820312\n",
      "fps: 0.8057556831531081\n",
      "TIMESTEP 54924 / STATE explore / EPSILON 0.04523182299991632 / ACTION 1 / REWARD 0 / Q_MAX  167.68843 / Loss  159.2139892578125\n",
      "fps: 0.7757508322441119\n",
      "TIMESTEP 54925 / STATE explore / EPSILON 0.04523082399991632 / ACTION 3 / REWARD 0 / Q_MAX  161.48935 / Loss  265.74273681640625\n",
      "fps: 0.7081747906696545\n",
      "TIMESTEP 54926 / STATE explore / EPSILON 0.04522982499991632 / ACTION 2 / REWARD 9.0 / Q_MAX  143.91513 / Loss  91.77117919921875\n",
      "error:  could not convert string to float:  \n",
      "score:  \n"
     ]
    }
   ],
   "source": [
    "playGame(observe=False);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}