1 |
145 |
lanttu |
-------------------------------------------------------------------------------
|
2 |
|
|
-- File : mesh_fast_router.vhdl
|
3 |
|
|
-- Description : Routes packets in 2D mesh network.
|
4 |
|
|
-- Five io channels, four to neighbor routers.
|
5 |
|
|
-- All output channels have fifo buffers.
|
6 |
|
|
-- Routers are identified with two-part address
|
7 |
|
|
-- row address (top bits) and columns address (lowest bits).
|
8 |
|
|
-- Port names (N/W/S/E/Ip) refer to their _location_
|
9 |
|
|
-- on the router (instead of signal direction).
|
10 |
|
|
--
|
11 |
|
|
-- Routers wait for complete packet before forwarding it.
|
12 |
|
|
-- Packets reach first the right row and then the right column
|
13 |
|
|
-- (YX routing or dimension-order routing, such deterministic
|
14 |
|
|
-- scheme assures in-order pkt delivery).
|
15 |
|
|
--
|
16 |
|
|
-- Size of packet (including address) don't have to be
|
17 |
|
|
-- the same as Fifo_depth anymore
|
18 |
|
|
--
|
19 |
|
|
--
|
20 |
|
|
-- Author : Erno Salminen
|
21 |
|
|
-- Date : 10.06.2003
|
22 |
|
|
-------------------------------------------------------------------------------
|
23 |
|
|
-- This file is part of Transaction Generator.
|
24 |
|
|
--
|
25 |
|
|
-- Transaction Generator is free software: you can redistribute it and/or modify
|
26 |
|
|
-- it under the terms of the Lesser GNU General Public License as published by
|
27 |
|
|
-- the Free Software Foundation, either version 3 of the License, or
|
28 |
|
|
-- (at your option) any later version.
|
29 |
|
|
--
|
30 |
|
|
-- Transaction Generator is distributed in the hope that it will be useful,
|
31 |
|
|
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
|
32 |
|
|
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
33 |
|
|
-- Lesser GNU General Public License for more details.
|
34 |
|
|
--
|
35 |
|
|
-- You should have received a copy of the Lesser GNU General Public License
|
36 |
|
|
-- along with Transaction Generator. If not, see <http://www.gnu.org/licenses/>.
|
37 |
|
|
-------------------------------------------------------------------------------
|
38 |
|
|
-- Modified :
|
39 |
|
|
-- 25.07.2003 ES full signal added, finalizing
|
40 |
|
|
-- 11.08.2003 ES fifo added to router, it stores data coming from ip
|
41 |
|
|
-- ports definitions modified at the same time
|
42 |
|
|
-- 05.11.2003 ES full_ip_tx fixed
|
43 |
|
|
-- 29.04.2005 ES Internal naming changed
|
44 |
|
|
-- 21.08.2006 AK multiclk support and naming convention change
|
45 |
|
|
-- 15.10.2006 ES Fifo_size independent from pkt_len (if stfwd=0)
|
46 |
|
|
-- and routing is done in single cycle for all ports
|
47 |
|
|
-------------------------------------------------------------------------------
|
48 |
|
|
|
49 |
|
|
library ieee;
|
50 |
|
|
use ieee.std_logic_1164.all;
|
51 |
|
|
|
52 |
|
|
use ieee.std_logic_arith.all;
|
53 |
|
|
use ieee.std_logic_unsigned.all;
|
54 |
|
|
|
55 |
|
|
entity mesh_router is
|
56 |
|
|
|
57 |
|
|
generic (
|
58 |
|
|
stfwd_en_g : integer := 1; --24.08.2006 es
|
59 |
|
|
data_width_g : integer := 0;
|
60 |
|
|
addr_width_g : integer := 0; -- at least two bits, A = row & col
|
61 |
|
|
fifo_depth_g : integer := 0;
|
62 |
|
|
pkt_len_g : integer := 5;
|
63 |
|
|
len_flit_en_g : integer := 1; -- 2007/08/03 where to place a pkt_len
|
64 |
|
|
oaddr_flit_en_g : integer := 1; -- 2007/08/03 whether to send the orig address
|
65 |
|
|
|
66 |
|
|
ip_freq_g : integer := 1; -- relative IP frequency
|
67 |
|
|
mesh_freq_g : integer := 1; --relative router frequency
|
68 |
|
|
col_addr_g : integer := 0;
|
69 |
|
|
row_addr_g : integer := 0;
|
70 |
|
|
num_cols_g : integer := -1; -- if used, outer fifos are not
|
71 |
|
|
num_rows_g : integer := -1 -- mapped
|
72 |
|
|
);
|
73 |
|
|
port (
|
74 |
|
|
clk_ip : in std_logic;
|
75 |
|
|
clk_mesh : in std_logic;
|
76 |
|
|
rst_n : in std_logic;
|
77 |
|
|
|
78 |
|
|
data_n_in : in std_logic_vector (data_width_g-1 downto 0);
|
79 |
|
|
empty_n_in : in std_logic;
|
80 |
|
|
full_n_in : in std_logic;
|
81 |
|
|
re_n_in : in std_logic;
|
82 |
|
|
data_s_in : in std_logic_vector (data_width_g-1 downto 0);
|
83 |
|
|
empty_s_in : in std_logic;
|
84 |
|
|
full_s_in : in std_logic;
|
85 |
|
|
re_s_in : in std_logic;
|
86 |
|
|
data_w_in : in std_logic_vector (data_width_g-1 downto 0);
|
87 |
|
|
empty_w_in : in std_logic;
|
88 |
|
|
full_w_in : in std_logic;
|
89 |
|
|
re_w_in : in std_logic;
|
90 |
|
|
data_e_in : in std_logic_vector (data_width_g-1 downto 0);
|
91 |
|
|
empty_e_in : in std_logic;
|
92 |
|
|
full_e_in : in std_logic;
|
93 |
|
|
re_e_in : in std_logic;
|
94 |
|
|
|
95 |
|
|
-- Ip signals modified 11.08.03
|
96 |
|
|
data_ip_Tx_in : in std_logic_vector (data_width_g-1 downto 0);
|
97 |
|
|
we_ip_Tx_in : in std_logic;
|
98 |
|
|
empty_ip_Tx_Out : out std_logic;
|
99 |
|
|
full_ip_Tx_Out : out std_logic;
|
100 |
|
|
|
101 |
|
|
data_n_Out : out std_logic_vector (data_width_g-1 downto 0);
|
102 |
|
|
empty_n_Out : out std_logic;
|
103 |
|
|
full_n_Out : out std_logic;
|
104 |
|
|
re_n_Out : out std_logic;
|
105 |
|
|
data_s_Out : out std_logic_vector (data_width_g-1 downto 0);
|
106 |
|
|
empty_s_Out : out std_logic;
|
107 |
|
|
full_s_Out : out std_logic;
|
108 |
|
|
re_s_Out : out std_logic;
|
109 |
|
|
data_w_Out : out std_logic_vector (data_width_g-1 downto 0);
|
110 |
|
|
empty_w_Out : out std_logic;
|
111 |
|
|
full_w_Out : out std_logic;
|
112 |
|
|
re_w_Out : out std_logic;
|
113 |
|
|
data_e_Out : out std_logic_vector (data_width_g-1 downto 0);
|
114 |
|
|
empty_e_Out : out std_logic;
|
115 |
|
|
full_e_Out : out std_logic;
|
116 |
|
|
re_e_Out : out std_logic;
|
117 |
|
|
|
118 |
|
|
-- Ip signals modified 11.08.03
|
119 |
|
|
data_ip_Rx_Out : out std_logic_vector (data_width_g-1 downto 0);
|
120 |
|
|
re_ip_Rx_in : in std_logic;
|
121 |
|
|
full_ip_Rx_Out : out std_logic;
|
122 |
|
|
empty_ip_Rx_Out : out std_logic
|
123 |
|
|
);
|
124 |
|
|
|
125 |
|
|
end mesh_router;
|
126 |
|
|
|
127 |
|
|
|
128 |
|
|
architecture rtl of mesh_router is
|
129 |
|
|
|
130 |
|
|
-- Constants for accessing arrays (e.g. state_regs)
|
131 |
|
|
constant N : integer := 0;
|
132 |
|
|
constant W : integer := 1;
|
133 |
|
|
constant S : integer := 2;
|
134 |
|
|
constant E : integer := 3;
|
135 |
|
|
constant Ip : integer := 4;
|
136 |
|
|
constant no_dir_c : integer := 5; -- Illegal index
|
137 |
|
|
|
138 |
|
|
|
139 |
|
|
|
140 |
|
|
component fifo
|
141 |
|
|
generic (
|
142 |
|
|
data_width_g : integer := 0;
|
143 |
|
|
depth_g : integer := 0
|
144 |
|
|
);
|
145 |
|
|
port (
|
146 |
|
|
clk : in std_logic;
|
147 |
|
|
rst_n : in std_logic;
|
148 |
|
|
data_in : in std_logic_vector (data_width_g-1 downto 0);
|
149 |
|
|
we_in : in std_logic;
|
150 |
|
|
one_p_out : out std_logic;
|
151 |
|
|
full_out : out std_logic;
|
152 |
|
|
data_out : out std_logic_vector (data_width_g-1 downto 0);
|
153 |
|
|
re_in : in std_logic;
|
154 |
|
|
empty_out : out std_logic;
|
155 |
|
|
one_d_out : out std_logic
|
156 |
|
|
);
|
157 |
|
|
end component; --fifo;
|
158 |
|
|
|
159 |
|
|
component multiclk_fifo
|
160 |
|
|
generic (
|
161 |
|
|
re_freq_g : integer;
|
162 |
|
|
we_freq_g : integer;
|
163 |
|
|
depth_g : integer;
|
164 |
|
|
data_width_g : integer
|
165 |
|
|
);
|
166 |
|
|
port (
|
167 |
|
|
clk_re : in std_logic;
|
168 |
|
|
clk_we : in std_logic;
|
169 |
|
|
rst_n : in std_logic;
|
170 |
|
|
data_in : in std_logic_vector (data_width_g-1 downto 0);
|
171 |
|
|
we_in : in std_logic;
|
172 |
|
|
full_out : out std_logic;
|
173 |
|
|
one_p_out : out std_logic;
|
174 |
|
|
re_in : in std_logic;
|
175 |
|
|
data_out : out std_logic_vector (data_width_g-1 downto 0);
|
176 |
|
|
empty_out : out std_logic;
|
177 |
|
|
one_d_out : out std_logic
|
178 |
|
|
);
|
179 |
|
|
end component;
|
180 |
|
|
|
181 |
|
|
-- Arrays are easier to handle than names with direction identification
|
182 |
|
|
-- (e.g. data_arr(i) vs. data_n)
|
183 |
|
|
type data_arr_type is array (4 downto 0) of std_logic_vector (data_width_g-1 downto 0);
|
184 |
|
|
type ctrl_arr_type is array (4 downto 0) of std_logic;
|
185 |
|
|
type source_type is array (4 downto 0) of integer range 0 to 5;
|
186 |
|
|
type counter_arr_type is array (4 downto 0) of integer range 0 to pkt_len_g; -- --fifo_depth_g;
|
187 |
|
|
|
188 |
|
|
-- 2007/08/02 Start implementing variable-length packets
|
189 |
|
|
signal pkt_len_arr_r : counter_arr_type; --2007/08/02
|
190 |
|
|
|
191 |
|
|
-- Internal types and signals. Thse are mapped to router's inputs
|
192 |
|
|
signal Incoming_data : data_arr_type;
|
193 |
|
|
signal Incoming_empty : ctrl_arr_type;
|
194 |
|
|
signal Incoming_full : ctrl_arr_type;
|
195 |
|
|
|
196 |
|
|
-- From fifos. Mapped directly to outputs pins
|
197 |
|
|
signal data_from_fifo : data_arr_type;
|
198 |
|
|
signal full_from_fifo : ctrl_arr_type;
|
199 |
|
|
signal empty_from_fifo : ctrl_arr_type; -- this goes also to ctrl
|
200 |
|
|
|
201 |
|
|
-- From ctrl
|
202 |
|
|
signal data_ctrl_fifo_r : data_arr_type;
|
203 |
|
|
signal we_ctrl_fifo_r : ctrl_arr_type;
|
204 |
|
|
signal re_r : ctrl_arr_type;
|
205 |
|
|
signal send_counter_r : counter_arr_type;
|
206 |
|
|
|
207 |
|
|
-- 13.09.2006. Try wormhole, combinatorial enable signals
|
208 |
|
|
signal re_tmp : ctrl_arr_type;
|
209 |
|
|
signal we_tmp : ctrl_arr_type;
|
210 |
|
|
|
211 |
|
|
signal data_reg_valid_r : ctrl_arr_type; -- 2006/10/26
|
212 |
|
|
signal data_ctrl_fifo_dbg : data_arr_type;
|
213 |
|
|
|
214 |
|
|
|
215 |
|
|
-- State registers
|
216 |
|
|
signal State_writing_r : std_logic_vector (4 downto 0); -- 0=N,1=W,2=S,3=E,4=Ip
|
217 |
|
|
signal State_reading_r : std_logic_vector (4 downto 0); -- 0=N,1=W,2=S,3=E,4=Ip
|
218 |
|
|
|
219 |
|
|
-- state_src_r(i) tells which port the source for output port i
|
220 |
|
|
signal state_src_r : source_type; -- 0=N,1=W,2=S,3=E,4=Ip
|
221 |
|
|
|
222 |
|
|
-- state_dst_r(i) tells which port the destination for input port i
|
223 |
|
|
signal state_dst_r : source_type; -- 0=N,1=W,2=S,3=E,4=Ip
|
224 |
|
|
|
225 |
|
|
|
226 |
|
|
|
227 |
|
|
-- Marks which input is checked on current cycle
|
228 |
|
|
signal curr_src_r : integer range 0 to 4;
|
229 |
|
|
|
230 |
|
|
-- Decoded address in input port pointed by curr_src_r, _not_ a register!
|
231 |
|
|
--signal curr_dst : integer range 0 to 5;
|
232 |
|
|
-- new type 13.09.2006, ES
|
233 |
|
|
type dst_arr_type is array (5-1 downto 0) of integer range 0 to 5;
|
234 |
|
|
signal curr_dst : dst_arr_type;
|
235 |
|
|
signal curr_dst_resolved : dst_arr_type;
|
236 |
|
|
signal n_req_dbgr : dst_arr_type;
|
237 |
|
|
|
238 |
|
|
-- 2007/08/06
|
239 |
|
|
constant len_width_c : integer := 8; -- bits needed for pkt_len, will be generic someday?
|
240 |
|
|
|
241 |
|
|
|
242 |
|
|
begin -- rtl
|
243 |
|
|
|
244 |
|
|
assert addr_width_g > 1 report "Mesh addr must be at least 2 bits wide. addr = row & column" severity failure;
|
245 |
|
|
assert addr_width_g < (data_width_g+1) report "Mesh addr cannot be wider than data" severity failure;
|
246 |
|
|
|
247 |
|
|
|
248 |
|
|
|
249 |
|
|
-- Concurrent assignments
|
250 |
|
|
-- Connect fifo status signals to outputs
|
251 |
|
|
empty_n_Out <= empty_from_fifo (N);
|
252 |
|
|
empty_w_Out <= empty_from_fifo (W);
|
253 |
|
|
empty_s_Out <= empty_from_fifo (S);
|
254 |
|
|
empty_e_Out <= empty_from_fifo (E);
|
255 |
|
|
empty_ip_Rx_Out <= empty_from_fifo (Ip); -- from ip_rx_fifo
|
256 |
|
|
|
257 |
|
|
full_n_Out <= full_from_fifo (N);
|
258 |
|
|
full_w_Out <= full_from_fifo (W);
|
259 |
|
|
full_s_Out <= full_from_fifo (S);
|
260 |
|
|
full_e_Out <= full_from_fifo (E);
|
261 |
|
|
full_ip_Rx_Out <= full_from_fifo (Ip); -- -- from ip_rx_fifo
|
262 |
|
|
|
263 |
|
|
|
264 |
|
|
-- Collect inputs into array
|
265 |
|
|
Incoming_data (N) <= data_n_in;
|
266 |
|
|
Incoming_data (W) <= data_w_in;
|
267 |
|
|
Incoming_data (S) <= data_s_in;
|
268 |
|
|
Incoming_data (E) <= data_e_in;
|
269 |
|
|
|
270 |
|
|
Incoming_empty (N) <= empty_n_in;
|
271 |
|
|
Incoming_empty (W) <= empty_w_in;
|
272 |
|
|
Incoming_empty (S) <= empty_s_in;
|
273 |
|
|
Incoming_empty (E) <= empty_e_in;
|
274 |
|
|
empty_ip_Tx_Out <= Incoming_empty (Ip); -- from ip_tx_fifo
|
275 |
|
|
|
276 |
|
|
Incoming_full (N) <= full_n_in;
|
277 |
|
|
Incoming_full (W) <= full_w_in;
|
278 |
|
|
Incoming_full (S) <= full_s_in;
|
279 |
|
|
Incoming_full (E) <= full_e_in;
|
280 |
|
|
full_ip_Tx_Out <= Incoming_full (Ip); -- from ip_tx_fifo
|
281 |
|
|
|
282 |
|
|
|
283 |
|
|
-- Separate arrays signals into outputs
|
284 |
|
|
re_n_Out <= re_tmp (N); -- re_r (N);
|
285 |
|
|
re_w_Out <= re_tmp (W); -- re_r (W);
|
286 |
|
|
re_s_Out <= re_tmp (S); -- re_r (S);
|
287 |
|
|
re_e_Out <= re_tmp (E); -- re_r (E);
|
288 |
|
|
|
289 |
|
|
|
290 |
|
|
-- Debug
|
291 |
|
|
-- Easier to trace if outputs are 'Z' when output fifo is empty
|
292 |
|
|
-- (Comment out the "when part" for synthesis
|
293 |
|
|
-- e.g."data_n_Out <= data_from_fifo (N); -- when data_from_fifo and so on")
|
294 |
|
|
data_n_Out <= data_from_fifo (N); -- when empty_from_fifo (N) = '0' else (others => 'Z');
|
295 |
|
|
data_w_Out <= data_from_fifo (W); -- when empty_from_fifo (W) = '0' else (others => 'Z');
|
296 |
|
|
data_s_Out <= data_from_fifo (S); -- when empty_from_fifo (S) = '0' else (others => 'Z');
|
297 |
|
|
data_e_Out <= data_from_fifo (E); -- when empty_from_fifo (E) = '0' else (others => 'Z');
|
298 |
|
|
data_ip_Rx_Out <= data_from_fifo (Ip); -- when empty_from_fifo (Ip) = '0' else (others => '0');
|
299 |
|
|
|
300 |
|
|
-- 13.09.2006
|
301 |
|
|
read_en_tmp: process (we_ctrl_fifo_r, re_r,
|
302 |
|
|
data_reg_valid_r,
|
303 |
|
|
Incoming_empty,
|
304 |
|
|
state_dst_r, state_src_r,
|
305 |
|
|
full_from_fifo)
|
306 |
|
|
|
307 |
|
|
begin -- process read_en_tmp
|
308 |
|
|
for i in 0 to 5-1 loop
|
309 |
|
|
|
310 |
|
|
-- i viittaa kohteeseen (fifo)
|
311 |
|
|
if state_src_r (i) = no_dir_c then
|
312 |
|
|
we_tmp (i) <= '0';
|
313 |
|
|
else
|
314 |
|
|
we_tmp (i) <= we_ctrl_fifo_r (i)
|
315 |
|
|
and data_reg_valid_r (i)
|
316 |
|
|
and (not (full_from_fifo (i)));
|
317 |
|
|
end if;
|
318 |
|
|
|
319 |
|
|
|
320 |
|
|
-- i viittaa llähteeseen (=input_port)
|
321 |
|
|
if state_dst_r (i) = no_dir_c then
|
322 |
|
|
-- sisääntulossa i ei ole järkevää dataa
|
323 |
|
|
re_tmp (i) <= '0';
|
324 |
|
|
else
|
325 |
|
|
-- ei lueta sisääntulosta i ellei
|
326 |
|
|
-- a) tilakone ole lukemass sieltä (re_r pitää olla 1)
|
327 |
|
|
-- b) sisääntulossa ole validia dataa (empty pitää olla 0)
|
328 |
|
|
-- c) kohde ei ole varattu
|
329 |
|
|
re_tmp (i) <= re_r (i)
|
330 |
|
|
and (not Incoming_empty (i))
|
331 |
|
|
and (not full_from_fifo (state_dst_r (i)));
|
332 |
|
|
end if;
|
333 |
|
|
|
334 |
|
|
end loop; -- i
|
335 |
|
|
end process read_en_tmp;
|
336 |
|
|
|
337 |
|
|
|
338 |
|
|
|
339 |
|
|
|
340 |
|
|
not_map_north_row : if row_addr_g = 0 generate
|
341 |
|
|
full_from_fifo(N) <= '0';
|
342 |
|
|
data_from_fifo(N) <= (others => '0');
|
343 |
|
|
empty_from_fifo(N) <= '1';
|
344 |
|
|
end generate not_map_north_row;
|
345 |
|
|
|
346 |
|
|
map_north_row : if row_addr_g /= 0 generate
|
347 |
|
|
|
348 |
|
|
North_fifo : fifo
|
349 |
|
|
generic map (
|
350 |
|
|
data_width_g => data_width_g,
|
351 |
|
|
depth_g => fifo_depth_g
|
352 |
|
|
)
|
353 |
|
|
port map (
|
354 |
|
|
clk => clk_Mesh,
|
355 |
|
|
rst_n => rst_n,
|
356 |
|
|
data_in => data_ctrl_fifo_r (N),
|
357 |
|
|
we_in => we_tmp (N), -- we_ctrl_fifo_r (N),
|
358 |
|
|
--one_p_out => one_p_from_fifo (N),
|
359 |
|
|
full_out => full_from_fifo (N),
|
360 |
|
|
data_Out => data_from_fifo (N),
|
361 |
|
|
re_in => re_n_in, -- router input
|
362 |
|
|
empty_out => empty_from_fifo (N) --,
|
363 |
|
|
--one_d_out => one_d_from_fifo (N)
|
364 |
|
|
);
|
365 |
|
|
|
366 |
|
|
end generate map_north_row;
|
367 |
|
|
|
368 |
|
|
-- Component mappings
|
369 |
|
|
|
370 |
|
|
not_map_south_row : if row_addr_g = num_rows_g-1 generate
|
371 |
|
|
full_from_fifo(S) <= '0';
|
372 |
|
|
data_from_fifo(S) <= (others => '0');
|
373 |
|
|
empty_from_fifo(S) <= '1';
|
374 |
|
|
end generate not_map_south_row;
|
375 |
|
|
|
376 |
|
|
map_south_row : if row_addr_g /= num_rows_g-1 generate
|
377 |
|
|
|
378 |
|
|
South_fifo : fifo
|
379 |
|
|
generic map (
|
380 |
|
|
data_width_g => data_width_g,
|
381 |
|
|
depth_g => fifo_depth_g
|
382 |
|
|
)
|
383 |
|
|
port map (
|
384 |
|
|
clk => clk_Mesh,
|
385 |
|
|
rst_n => rst_n,
|
386 |
|
|
data_in => data_ctrl_fifo_r (S),
|
387 |
|
|
we_in => we_tmp (S), -- we_ctrl_fifo_r (S),
|
388 |
|
|
--one_p_out => one_p_from_fifo (S),
|
389 |
|
|
full_out => full_from_fifo (S),
|
390 |
|
|
data_Out => data_from_fifo (S),
|
391 |
|
|
re_in => re_s_in, -- router input
|
392 |
|
|
empty_out => empty_from_fifo (S) --,
|
393 |
|
|
--one_d_out => one_d_from_fifo (S)
|
394 |
|
|
);
|
395 |
|
|
end generate map_south_row;
|
396 |
|
|
|
397 |
|
|
not_map_west_col : if col_addr_g = 0 generate
|
398 |
|
|
full_from_fifo(W) <= '0';
|
399 |
|
|
data_from_fifo(W) <= (others => '0');
|
400 |
|
|
empty_from_fifo(W) <= '1';
|
401 |
|
|
end generate not_map_west_col;
|
402 |
|
|
|
403 |
|
|
map_west_col : if col_addr_g /= 0 generate
|
404 |
|
|
West_fifo : fifo
|
405 |
|
|
generic map (
|
406 |
|
|
data_width_g => data_width_g,
|
407 |
|
|
depth_g => fifo_depth_g
|
408 |
|
|
)
|
409 |
|
|
port map (
|
410 |
|
|
clk => clk_Mesh,
|
411 |
|
|
rst_n => rst_n,
|
412 |
|
|
data_in => data_ctrl_fifo_r (W),
|
413 |
|
|
we_in => we_tmp (W), --we_ctrl_fifo_r (W),
|
414 |
|
|
--one_p_out => one_p_from_fifo (W),
|
415 |
|
|
full_out => full_from_fifo (W),
|
416 |
|
|
data_Out => data_from_fifo (W),
|
417 |
|
|
re_in => re_w_in, -- router input
|
418 |
|
|
empty_out => empty_from_fifo (W) --,
|
419 |
|
|
--one_d_out => one_d_from_fifo (W)
|
420 |
|
|
);
|
421 |
|
|
end generate map_west_col;
|
422 |
|
|
|
423 |
|
|
not_map_east_col : if col_addr_g = num_cols_g-1 generate
|
424 |
|
|
full_from_fifo(E) <= '0';
|
425 |
|
|
data_from_fifo(E) <= (others => '0');
|
426 |
|
|
empty_from_fifo(E) <= '1';
|
427 |
|
|
end generate not_map_east_col;
|
428 |
|
|
|
429 |
|
|
map_east_col : if col_addr_g /= num_cols_g-1 generate
|
430 |
|
|
East_fifo : fifo
|
431 |
|
|
generic map (
|
432 |
|
|
data_width_g => data_width_g,
|
433 |
|
|
depth_g => fifo_depth_g
|
434 |
|
|
)
|
435 |
|
|
port map (
|
436 |
|
|
clk => clk_Mesh,
|
437 |
|
|
rst_n => rst_n,
|
438 |
|
|
data_in => data_ctrl_fifo_r (E),
|
439 |
|
|
we_in => we_tmp (E), --we_ctrl_fifo_r (E),
|
440 |
|
|
--one_p_out => one_p_from_fifo (E),
|
441 |
|
|
full_out => full_from_fifo (E),
|
442 |
|
|
data_Out => data_from_fifo (E),
|
443 |
|
|
re_in => re_e_in, -- router input
|
444 |
|
|
empty_out => empty_from_fifo (E) --,
|
445 |
|
|
--one_d_out => one_d_from_fifo (E)
|
446 |
|
|
);
|
447 |
|
|
end generate map_east_col;
|
448 |
|
|
|
449 |
|
|
|
450 |
|
|
-- data going to IP
|
451 |
|
|
ip_fifo_rx : multiclk_fifo
|
452 |
|
|
generic map (
|
453 |
|
|
re_freq_g => ip_freq_g,
|
454 |
|
|
we_freq_g => mesh_freq_g,
|
455 |
|
|
depth_g => fifo_depth_g,
|
456 |
|
|
data_width_g => data_width_g)
|
457 |
|
|
port map (
|
458 |
|
|
clk_re => clk_ip,
|
459 |
|
|
clk_we => clk_mesh,
|
460 |
|
|
|
461 |
|
|
rst_n => rst_n,
|
462 |
|
|
data_in => data_ctrl_fifo_r (Ip),
|
463 |
|
|
we_in => we_tmp (Ip), --we_ctrl_fifo_r (Ip),
|
464 |
|
|
--one_p_out => one_p_from_fifo (Ip),
|
465 |
|
|
full_out => full_from_fifo (Ip),
|
466 |
|
|
data_Out => data_from_fifo (Ip),
|
467 |
|
|
re_in => re_ip_Rx_in, -- router input
|
468 |
|
|
empty_out => empty_from_fifo (Ip) --,
|
469 |
|
|
--one_d_out => one_d_from_fifo (Ip)
|
470 |
|
|
);
|
471 |
|
|
|
472 |
|
|
|
473 |
|
|
|
474 |
|
|
-- Added 11.08.03
|
475 |
|
|
-- data coming from ip
|
476 |
|
|
ip_fifo_tx : multiclk_fifo
|
477 |
|
|
generic map (
|
478 |
|
|
re_freq_g => mesh_freq_g,
|
479 |
|
|
we_freq_g => ip_freq_g,
|
480 |
|
|
depth_g => fifo_depth_g,
|
481 |
|
|
data_width_g => data_width_g
|
482 |
|
|
)
|
483 |
|
|
port map (
|
484 |
|
|
clk_re => clk_mesh,
|
485 |
|
|
clk_we => clk_ip,
|
486 |
|
|
rst_n => rst_n,
|
487 |
|
|
-- Fifo inputs come straight router inputs (=from ip)
|
488 |
|
|
data_in => data_ip_Tx_in,
|
489 |
|
|
full_out => Incoming_full (Ip),
|
490 |
|
|
we_in => we_ip_Tx_in,
|
491 |
|
|
|
492 |
|
|
-- Fifo outputs go ctrl logic
|
493 |
|
|
data_Out => Incoming_data (Ip),
|
494 |
|
|
empty_out => Incoming_empty (Ip),
|
495 |
|
|
re_in => re_tmp (Ip) -- re_r (Ip)
|
496 |
|
|
);
|
497 |
|
|
|
498 |
|
|
|
499 |
|
|
Check_dst : process (State_reading_r,
|
500 |
|
|
--curr_src_r,
|
501 |
|
|
Incoming_data,
|
502 |
|
|
Incoming_empty,
|
503 |
|
|
Incoming_full
|
504 |
|
|
)
|
505 |
|
|
begin -- process Check_dst
|
506 |
|
|
|
507 |
|
|
-- 1) if (No read operation yet on the curr source port)
|
508 |
|
|
-- and (there is data (=addr) on curr src port) --possibly unnecessary check
|
509 |
|
|
-- and (Complete packet coming from curr src port)
|
510 |
|
|
--
|
511 |
|
|
-- 2) if (packet has reached target row)
|
512 |
|
|
-- 3) if (target column) => data goes to ip
|
513 |
|
|
-- 3) elsif (toward west)
|
514 |
|
|
-- 3) else (toward east)
|
515 |
|
|
--
|
516 |
|
|
-- 2) elsif toward north
|
517 |
|
|
-- 2) else (toward south)
|
518 |
|
|
-- 1) else => no packet arriving on curr src port
|
519 |
|
|
|
520 |
|
|
for src_i in 0 to 5-1 loop
|
521 |
|
|
|
522 |
|
|
-- new way
|
523 |
|
|
if State_reading_r (src_i) = '0'
|
524 |
|
|
and ((Incoming_full (src_i) = '1' and stfwd_en_g = 1)
|
525 |
|
|
or
|
526 |
|
|
(Incoming_empty (src_i) = '0'and stfwd_en_g = 0))
|
527 |
|
|
then -- 1)
|
528 |
|
|
|
529 |
|
|
--if Incoming_data (src_i) (addr_width_g-1 downto addr_width_g/2) = row_addr_g then -- 2)orig
|
530 |
|
|
if Incoming_data (src_i) (addr_width_g-len_width_c-1 downto addr_width_g/2) = row_addr_g then -- 2)2007/08/06
|
531 |
|
|
|
532 |
|
|
-- Correct row
|
533 |
|
|
if Incoming_data (src_i) (addr_width_g/2-1 downto 0) = col_addr_g then -- 3)
|
534 |
|
|
-- Correct row and column
|
535 |
|
|
curr_dst (src_i) <= Ip;
|
536 |
|
|
elsif Incoming_data (src_i) (addr_width_g/2-1 downto 0) < col_addr_g then
|
537 |
|
|
-- Packet is on the right row, going west (to left)
|
538 |
|
|
curr_dst (src_i) <= W;
|
539 |
|
|
else
|
540 |
|
|
-- Right row, packet going east (to right)
|
541 |
|
|
curr_dst (src_i) <= E;
|
542 |
|
|
end if;
|
543 |
|
|
|
544 |
|
|
-- elsif Incoming_data (src_i) (addr_width_g-1 downto addr_width_g/2) < row_addr_g then --2)orig
|
545 |
|
|
elsif Incoming_data (src_i) (addr_width_g-len_width_c-1 downto addr_width_g/2) < row_addr_g then --2)2007/08/06
|
546 |
|
|
-- Packet going north (upward)
|
547 |
|
|
curr_dst (src_i) <= N;
|
548 |
|
|
else
|
549 |
|
|
-- Packet going south (downward)
|
550 |
|
|
curr_dst (src_i) <= S;
|
551 |
|
|
end if;
|
552 |
|
|
|
553 |
|
|
|
554 |
|
|
else --1)
|
555 |
|
|
-- No packet on curr src port or read operation already started
|
556 |
|
|
curr_dst (src_i) <= no_dir_c;
|
557 |
|
|
end if;
|
558 |
|
|
|
559 |
|
|
end loop; -- i
|
560 |
|
|
|
561 |
|
|
|
562 |
|
|
|
563 |
|
|
end process Check_dst;
|
564 |
|
|
|
565 |
|
|
|
566 |
|
|
data_dbg: for i in 0 to 5-1 generate
|
567 |
|
|
data_ctrl_fifo_dbg (i) <= data_ctrl_fifo_r (i) when data_reg_valid_r (i)='1'
|
568 |
|
|
else (others => '0');
|
569 |
|
|
end generate data_dbg;
|
570 |
|
|
|
571 |
|
|
debug_proc: process (we_tmp, data_ctrl_fifo_dbg)
|
572 |
|
|
begin -- process debug_proc
|
573 |
|
|
for i in 0 to 5-1 loop
|
574 |
|
|
if we_tmp (i) = '1' then
|
575 |
|
|
assert (data_ctrl_fifo_dbg (i)(0) /= 'Z') report "" severity FAILURE;
|
576 |
|
|
end if;
|
577 |
|
|
end loop; -- i
|
578 |
|
|
end process debug_proc;
|
579 |
|
|
|
580 |
|
|
|
581 |
|
|
|
582 |
|
|
-- Why this was postponed??? -AK 13.06.2007
|
583 |
|
|
-- Intuitively, it should not be, because it is not in testbench, is combinational and
|
584 |
|
|
-- possibly would yield different results with synthesis tools
|
585 |
|
|
|
586 |
|
|
-- resolve_conflicts: postponed process (curr_dst)
|
587 |
|
|
resolve_conflicts: process (curr_dst)
|
588 |
|
|
variable n_req_v : integer; -- num of req for this outport
|
589 |
|
|
begin -- process resolve_conflicts
|
590 |
|
|
|
591 |
|
|
curr_dst_resolved <= curr_dst;
|
592 |
|
|
|
593 |
|
|
-- 1) Go through all curr_dst values
|
594 |
|
|
-- 2) Count requests for the same
|
595 |
|
|
-- 3) Give turn to inport with smallest id (simple but unfair scheme)
|
596 |
|
|
for dst_i in 0 to 5-1 loop
|
597 |
|
|
n_req_v :=0;
|
598 |
|
|
for src_i in 0 to 5-1 loop
|
599 |
|
|
|
600 |
|
|
if curr_dst (src_i) = dst_i then
|
601 |
|
|
-- src_i haluaisi lähettää outporttiin dst_i
|
602 |
|
|
|
603 |
|
|
if n_req_v > 0 then
|
604 |
|
|
-- Katsotaan onko muita halukkaita
|
605 |
|
|
curr_dst_resolved (src_i) <= no_dir_c;
|
606 |
|
|
--assert false report "Conflict for outport. Inport with smallest index will be selected" severity note;
|
607 |
|
|
end if;
|
608 |
|
|
n_req_v := n_req_v +1;
|
609 |
|
|
end if;
|
610 |
|
|
end loop; -- src_i
|
611 |
|
|
|
612 |
|
|
n_req_dbgr (dst_i) <= n_req_v;
|
613 |
|
|
end loop; -- dst_i
|
614 |
|
|
|
615 |
|
|
end process resolve_conflicts;
|
616 |
|
|
|
617 |
|
|
-- Kopioi octagonista pikkuisen reilumpi konfliktinselvityskoodi!!!
|
618 |
|
|
|
619 |
|
|
|
620 |
|
|
Main_control : process (clk_Mesh, rst_n)
|
621 |
|
|
begin -- process Main_control
|
622 |
|
|
if rst_n = '0' then -- asynchronous reset (active low)
|
623 |
|
|
for i in 0 to 4 loop
|
624 |
|
|
data_ctrl_fifo_r (i) <= (others => '0'); --'0');
|
625 |
|
|
send_counter_r (i) <= 0;
|
626 |
|
|
pkt_len_arr_r (i) <= pkt_len_g; -- 2007/08/02
|
627 |
|
|
end loop; -- i
|
628 |
|
|
|
629 |
|
|
we_ctrl_fifo_r <= (others => '0');
|
630 |
|
|
re_r <= (others => '0');
|
631 |
|
|
State_writing_r <= (others => '0');
|
632 |
|
|
State_reading_r <= (others => '0');
|
633 |
|
|
state_src_r <= (others => no_dir_c);
|
634 |
|
|
state_dst_r <= (others => no_dir_c);
|
635 |
|
|
|
636 |
|
|
data_reg_valid_r <= (others => '0');
|
637 |
|
|
|
638 |
|
|
|
639 |
|
|
elsif clk_Mesh'event and clk_Mesh = '1' then -- rising clock edge
|
640 |
|
|
|
641 |
|
|
for i in 0 to 5-1 loop
|
642 |
|
|
-- Handle all directions
|
643 |
|
|
|
644 |
|
|
-- From now on, loop variable i refers to source port
|
645 |
|
|
|
646 |
|
|
if State_reading_r (i) = '1' then
|
647 |
|
|
-- Already reading from direction i
|
648 |
|
|
|
649 |
|
|
-- 17.03.2006 use counters to enable cut-through switching.
|
650 |
|
|
-- The same thing works also with store-and-forward switching
|
651 |
|
|
-- 2007/08/02: allow variable-length packets
|
652 |
|
|
--if send_counter_r (i) = pkt_len_g --fixed-len pkt
|
653 |
|
|
if send_counter_r (i) = pkt_len_arr_r (i) --variable-_len pkt
|
654 |
|
|
then
|
655 |
|
|
-- stop
|
656 |
|
|
|
657 |
|
|
if full_from_fifo (state_dst_r (i)) = '0' then
|
658 |
|
|
|
659 |
|
|
we_ctrl_fifo_r (state_dst_r (i)) <= '0';
|
660 |
|
|
re_r (i) <= '0';
|
661 |
|
|
State_writing_r (state_dst_r (i)) <= '0';
|
662 |
|
|
State_reading_r (i) <= '0';
|
663 |
|
|
state_src_r (state_dst_r (i)) <= no_dir_c;
|
664 |
|
|
state_dst_r (i) <= no_dir_c;
|
665 |
|
|
--assert false report "Tx to north fifo completed" severity note;
|
666 |
|
|
|
667 |
|
|
send_counter_r (i) <= 0;
|
668 |
|
|
data_ctrl_fifo_r (state_dst_r (i)) <= (others => 'X'); --Z');
|
669 |
|
|
data_reg_valid_r (state_dst_r (i)) <= '0';
|
670 |
|
|
|
671 |
|
|
pkt_len_arr_r (i) <= pkt_len_g; -- 2007/08/02
|
672 |
|
|
|
673 |
|
|
--assert state_dst_r(i)/=4 report "Stop" severity note;
|
674 |
|
|
else
|
675 |
|
|
--assert state_dst_r(i)/=4 report "Wait until last data fits into fifo" severity note;
|
676 |
|
|
end if;
|
677 |
|
|
|
678 |
|
|
|
679 |
|
|
else
|
680 |
|
|
-- Packet transfer not yet complete
|
681 |
|
|
-- Continue transfer
|
682 |
|
|
we_ctrl_fifo_r (state_dst_r (i)) <= '1';
|
683 |
|
|
State_writing_r (state_dst_r (i)) <= State_writing_r (state_dst_r (i));
|
684 |
|
|
State_reading_r (i) <= State_reading_r (i);
|
685 |
|
|
state_src_r (state_dst_r (i)) <= state_src_r (state_dst_r (i));
|
686 |
|
|
state_dst_r (i) <= state_dst_r (i);
|
687 |
|
|
|
688 |
|
|
-- i = source
|
689 |
|
|
if data_reg_valid_r (state_dst_r(i)) = '0' then
|
690 |
|
|
-- data_reg empty
|
691 |
|
|
|
692 |
|
|
if re_tmp (i) = '1' then
|
693 |
|
|
data_reg_valid_r (state_dst_r(i)) <= '1'; -- read new value
|
694 |
|
|
send_counter_r (i) <= send_counter_r (i) +1;
|
695 |
|
|
data_ctrl_fifo_r (state_dst_r (i)) <= Incoming_data (i);
|
696 |
|
|
--assert i/=1 report "v=0, re=1" severity note;
|
697 |
|
|
else
|
698 |
|
|
data_reg_valid_r (state_dst_r(i)) <= '0'; -- stay empty
|
699 |
|
|
--assert i/=1 report "v=0, re=0" severity note;
|
700 |
|
|
end if;
|
701 |
|
|
|
702 |
|
|
else
|
703 |
|
|
-- data_reg full
|
704 |
|
|
|
705 |
|
|
if full_from_fifo (state_dst_r (i)) = '0' then
|
706 |
|
|
-- write old to fifo
|
707 |
|
|
if re_tmp (i) = '1' then
|
708 |
|
|
data_reg_valid_r (state_dst_r(i)) <= '1'; -- write old, read new value
|
709 |
|
|
send_counter_r (i) <= send_counter_r (i) +1;
|
710 |
|
|
data_ctrl_fifo_r (state_dst_r (i)) <= Incoming_data (i);
|
711 |
|
|
--assert i/=1 report "v = 1, re = 1, fu=0" severity note;
|
712 |
|
|
else
|
713 |
|
|
data_reg_valid_r (state_dst_r(i)) <= '0'; -- write old, now empty
|
714 |
|
|
--assert i/=1 report "v = 1, re = 0, f=0" severity note;
|
715 |
|
|
end if;
|
716 |
|
|
|
717 |
|
|
else
|
718 |
|
|
-- cannot write old
|
719 |
|
|
data_reg_valid_r (state_dst_r(i)) <= '1';
|
720 |
|
|
--assert i/=1 report "v=1, re=0, f=1" severity note;
|
721 |
|
|
assert re_tmp (i) = '0' report "reading not allowed now" severity error;
|
722 |
|
|
end if;
|
723 |
|
|
|
724 |
|
|
end if;
|
725 |
|
|
|
726 |
|
|
-- len-value in pkt does not include addr. If needed, add
|
727 |
|
|
-- + length flit itself
|
728 |
|
|
-- + orig_addr flit
|
729 |
|
|
if stfwd_en_g = 1 then
|
730 |
|
|
-- oletetaan, että tällöin paketit täytetään
|
731 |
|
|
pkt_len_arr_r (i) <= pkt_len_g;
|
732 |
|
|
-- Nythän pärjättäis pelkällä else-haaralla
|
733 |
|
|
-- => Täytynee lisätä generic "fill_pkt_en_g", koska
|
734 |
|
|
-- voi olla st-fwd mutta vaihtelevan mittaiset paketit
|
735 |
|
|
else
|
736 |
|
|
-- Len does not include len_flit itself nor network_address
|
737 |
|
|
-- => add their length to len value
|
738 |
|
|
-- Orig_address (if present) is already included own_addr_g value, it dont have to be added
|
739 |
|
|
|
740 |
|
|
if (len_flit_en_g = 0 and (re_tmp (i) = '1') and send_counter_r (i) = 0) then
|
741 |
|
|
pkt_len_arr_r (i) <= conv_integer (unsigned(Incoming_data (i)(data_width_g-1 downto data_width_g-len_width_c)))+ len_flit_en_g +1;
|
742 |
|
|
--assert false report "Read pkt_len from the pkt_header (addr- flit)" severity note;
|
743 |
|
|
|
744 |
|
|
elsif (len_flit_en_g = 1 and(re_tmp (i) = '1') and send_counter_r (i) = 1) then
|
745 |
|
|
pkt_len_arr_r (i) <= conv_integer (unsigned(Incoming_data (i)(len_width_c-1 downto 0)))+ len_flit_en_g +1;
|
746 |
|
|
--assert false report "Read pkt_len from the pkt_header (own len flit)" severity note;
|
747 |
|
|
else
|
748 |
|
|
pkt_len_arr_r (i) <= pkt_len_arr_r (i);
|
749 |
|
|
end if;
|
750 |
|
|
|
751 |
|
|
-- -- 2007/08/02 Take pkt_len from the header (2nd flit of pkt)
|
752 |
|
|
-- if (re_tmp (i) = '1') and send_counter_r (i) = 1 then
|
753 |
|
|
-- -- len-value in pkt does not include addr. If needed, add
|
754 |
|
|
-- -- + length flit itself
|
755 |
|
|
-- -- + orig_addr flit
|
756 |
|
|
-- if stfwd_en_g = 1 then
|
757 |
|
|
-- -- oletetaan, että tällöin paketit täytetään
|
758 |
|
|
-- pkt_len_arr_r (i) <= pkt_len_g;
|
759 |
|
|
-- -- Nythän pärjättäis pelkällä else-haaralla
|
760 |
|
|
-- -- => Täytynee lisätä generic "fill_pkt_en_g", koska
|
761 |
|
|
-- -- voi olla st-fwd mutta vaihtelevan mittaiset paketit
|
762 |
|
|
-- else
|
763 |
|
|
-- -- Len does not include len_flit itself nor network_address
|
764 |
|
|
-- -- => add their length to len value
|
765 |
|
|
-- -- Orig_address (if present) is already included own_addr_g value, it dont have to be added
|
766 |
|
|
-- pkt_len_arr_r (i) <= conv_integer (unsigned(Incoming_data (i)))+ len_flit_en_g +1;
|
767 |
|
|
-- --assert false report "Read pkt_len from the pkt_header" severity note;
|
768 |
|
|
-- end if;
|
769 |
|
|
|
770 |
|
|
|
771 |
|
|
end if;
|
772 |
|
|
|
773 |
|
|
if send_counter_r (i) = pkt_len_arr_r (i) -1
|
774 |
|
|
and Incoming_empty (i) = '0' -- 2006/10/24
|
775 |
|
|
and full_from_fifo (state_dst_r(i)) = '0'
|
776 |
|
|
then
|
777 |
|
|
re_r (i) <= '0';
|
778 |
|
|
else
|
779 |
|
|
re_r (i) <= '1';
|
780 |
|
|
end if;
|
781 |
|
|
|
782 |
|
|
|
783 |
|
|
end if;
|
784 |
|
|
|
785 |
|
|
else
|
786 |
|
|
-- Not yet reading from direction i
|
787 |
|
|
-- Check one direction (curr_src_r) per clock cycle
|
788 |
|
|
-- for possible new transfers
|
789 |
|
|
|
790 |
|
|
-- Direction i has to be current source,
|
791 |
|
|
-- there must be valid address on port i
|
792 |
|
|
-- and target fifo has to be empty (i.e. it is not full or reserved)
|
793 |
|
|
if
|
794 |
|
|
curr_dst_resolved (i) /= no_dir_c
|
795 |
|
|
and empty_from_fifo (curr_dst_resolved (i)) = '1'
|
796 |
|
|
and State_writing_r (curr_dst_resolved (i)) = '0' --25.07.2003
|
797 |
|
|
then
|
798 |
|
|
-- Start reading
|
799 |
|
|
|
800 |
|
|
we_ctrl_fifo_r (curr_dst_resolved (i)) <= '0';
|
801 |
|
|
-- WE not yet '1' because RE is just being asserted to one
|
802 |
|
|
-- Otherwise, the first data (i.e.) would be written twice and the
|
803 |
|
|
-- last data would be discarded
|
804 |
|
|
State_writing_r (curr_dst_resolved (i)) <= '1';
|
805 |
|
|
re_r (i) <= '1';
|
806 |
|
|
State_reading_r (i) <= '1';
|
807 |
|
|
state_src_r (curr_dst_resolved (i)) <= i;
|
808 |
|
|
state_dst_r (i) <= curr_dst_resolved (i);
|
809 |
|
|
|
810 |
|
|
send_counter_r (i) <= send_counter_r (i); --??
|
811 |
|
|
data_ctrl_fifo_r (curr_dst_resolved (i)) <= Incoming_data (i);
|
812 |
|
|
data_reg_valid_r (curr_dst_resolved (i)) <= '1';
|
813 |
|
|
|
814 |
|
|
--assert i/=1 report "Start" severity note;
|
815 |
|
|
|
816 |
|
|
else
|
817 |
|
|
-- Can't start reading from current source
|
818 |
|
|
-- Do nothing
|
819 |
|
|
end if; --State_reading_r = i
|
820 |
|
|
end if; --State_reading_r(i)=1
|
821 |
|
|
|
822 |
|
|
end loop; -- i
|
823 |
|
|
|
824 |
|
|
|
825 |
|
|
|
826 |
|
|
|
827 |
|
|
end if; --rst_n/clk'event
|
828 |
|
|
end process Main_control;
|
829 |
|
|
|
830 |
|
|
|
831 |
|
|
|
832 |
|
|
|
833 |
|
|
|
834 |
|
|
|
835 |
|
|
|
836 |
|
|
end rtl;
|