function gwValIteration(world,alpha,gamma,T)
%learn state values of gridworld <world> for T steps, acting randomly

global V

worldSize = size(world); %includes number of actions, which we drop in initializing V
D = length(worldSize) - 1; %dimensionality of world
V = zeros([worldSize(1:D) T]); %initialize state values; array of size <worldSize> x <T>
state = ones(1,D); %start in corner of space

for t=1:T
    action = ceil(rand*2*D); %random action
    [reward newState] = gridworldStep(world,state,action);
    TDerr = reward + gamma*getVal([newState t]) - getVal([state t]); %prediction error
    translate(t); %copy state values at time t to values at time t+1
    setVal([state t+1],getVal([state t]) + alpha*TDerr); %update value of last state
    %disp([state action reward newState TDerr getVal(V,[state t+1])])
    state = newState;
end

function v = getVal(stateTime)
global V
stateTimeString = num2str(stateTime(1)); %build text representation of state coordinates (workaround for limitations of matlab syntax)
for d=2:length(stateTime)
    stateTimeString = [stateTimeString ',' num2str(stateTime(d))];
end
v = eval(['V(' stateTimeString ')']); %read out entry from V

function newV = setVal(stateTime,v)
global V
stateTimeString = num2str(stateTime(1)); %build text representation of state coordinates (workaround for limitations of matlab syntax)
for d=2:length(stateTime)
    stateTimeString = [stateTimeString ',' num2str(stateTime(d))];
end
eval(['V(' stateTimeString ') = v;']) %replace entry in newV with v

function newV = translate(t)
global V
colonString = ':'; %build text representation
for d=2:length(size(V))-1
    colonString = [colonString ',:'];
end
eval(['V(' colonString ',t+1) = V(' colonString ',t);']) %copy entries from newV(:,...,:,t) to newV(:,...,:,t+1)